├── inputs └── .gitkeep ├── outputs ├── json │ └── .gitkeep └── sqlite │ └── .gitkeep ├── package.json ├── formats ├── json.js └── sqlite.js ├── LICENSE ├── .gitignore └── README.md /inputs/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /outputs/json/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /outputs/sqlite/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "osdb-project", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "build.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "Navinda Dissanayake", 10 | "license": "MIT", 11 | "dependencies": { 12 | "argio": "0.0.3", 13 | "csvtojson": "^2.0.10", 14 | "sqlite3": "^4.1.1" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /formats/json.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs"); 2 | 3 | const build = (data) => { 4 | try { 5 | const jsonPath = `${__dirname}/../outputs/json`; 6 | const en2sn = {}; 7 | const sn2en = {}; 8 | data.en2sn.forEach(row => { en2sn[row.word] = row.definitions }); 9 | data.sn2en.forEach(row => { sn2en[row.word] = row.definitions }); 10 | fs.writeFileSync(`${jsonPath}/en2sn.json`, JSON.stringify(en2sn)); 11 | fs.writeFileSync(`${jsonPath}/sn2en.json`, JSON.stringify(sn2en)); 12 | console.log(`Success: JSON files have been generated!.`); 13 | } catch (e) { 14 | console.log(`Error: Failed to build JSON files!. ${e}`); 15 | } 16 | } 17 | 18 | module.exports = { 19 | build 20 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Navinda Pramod Dissanayake. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | 9 | # Diagnostic reports (https://nodejs.org/api/report.html) 10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 11 | 12 | # Runtime data 13 | pids 14 | *.pid 15 | *.seed 16 | *.pid.lock 17 | 18 | # Directory for instrumented libs generated by jscoverage/JSCover 19 | lib-cov 20 | 21 | # Coverage directory used by tools like istanbul 22 | coverage 23 | *.lcov 24 | 25 | # nyc test coverage 26 | .nyc_output 27 | 28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 29 | .grunt 30 | 31 | # Bower dependency directory (https://bower.io/) 32 | bower_components 33 | 34 | # node-waf configuration 35 | .lock-wscript 36 | 37 | # Compiled binary addons (https://nodejs.org/api/addons.html) 38 | build/Release 39 | 40 | # Dependency directories 41 | node_modules/ 42 | jspm_packages/ 43 | 44 | # TypeScript v1 declaration files 45 | typings/ 46 | 47 | # TypeScript cache 48 | *.tsbuildinfo 49 | 50 | # Optional npm cache directory 51 | .npm 52 | 53 | # Optional eslint cache 54 | .eslintcache 55 | 56 | # Microbundle cache 57 | .rpt2_cache/ 58 | .rts2_cache_cjs/ 59 | .rts2_cache_es/ 60 | .rts2_cache_umd/ 61 | 62 | # Optional REPL history 63 | .node_repl_history 64 | 65 | # Output of 'npm pack' 66 | *.tgz 67 | 68 | # Yarn Integrity file 69 | .yarn-integrity 70 | 71 | # dotenv environment variables file 72 | .env 73 | .env.test 74 | 75 | # parcel-bundler cache (https://parceljs.org/) 76 | .cache 77 | 78 | # next.js build output 79 | .next 80 | 81 | # nuxt.js build output 82 | .nuxt 83 | 84 | # gatsby files 85 | .cache/ 86 | public 87 | 88 | # vuepress build output 89 | .vuepress/dist 90 | 91 | # Serverless directories 92 | .serverless/ 93 | 94 | # FuseBox cache 95 | .fusebox/ 96 | 97 | # DynamoDB Local files 98 | .dynamodb/ 99 | 100 | # Project specific 101 | inputs/* 102 | !inputs/.gitkeep 103 | 104 | outputs/json/* 105 | !outputs/json/.gitkeep 106 | 107 | outputs/sqlite/* 108 | !outputs/sqlite/.gitkeep -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OSDB Project 2 | *English-Sinhala, Sinhala-English Database with more than 100k definitions.* 3 | 4 | ## Information 5 | This is an open database anyone can use to find Sinhala definitions for English words and vice versa. This project has a build script you can use to generate databases in different formats (see "Build" section for more information). 6 | 7 | ## Files 8 | - Main input files are in ```JavaScript Object Notation (JSON)``` format and they are located in the [Open Sinhala Database Repository](https://github.com/ipmanlk/OSDB). 9 | - They will get automatically downloaded to the ```inputs``` directory after running the build script. 10 | - You can open them using a text editor such as VSCode. 11 | - ```en2sn.json``` database contains English-Sinhala definitions. 12 | - ```sn2en.json``` database contains Sinhala-English definitions. 13 | - These are the main files used by build script to generate databases. 14 | 15 | ## Download main input files (optional) 16 | This is a part of the build process. But if you want to download input files (databases) without building anything or you want use the latest databases, you can do so by running the following command. 17 | 18 | ``` 19 | node build.js --d 20 | ``` 21 | This will download input files to the ```inputs``` directory. 22 | 23 | ## Build 24 | You can use the build script in this project to automatically generate databases in various formats. 25 | 26 | ### Currently supported formats, 27 | - SQLite 28 | - JSON (Structure is different than input files) 29 | 30 | ### Building Instructions 31 | 1. Install Nodejs. 32 | 1. Fork this repository. 33 | 1. Navigate to that directory using your Terminal (or CMD), 34 | 1. Run ```npm install``` to install dependencies. 35 | 1. After that run ```node build.js -f [formats]``` to generate databases. 36 | - Generated files will be placed inside the ```outputs``` directory. 37 | 38 | #### Examples 39 | ```javascript 40 | node build.js -f json //build json files 41 | node build.js -f sqlite //build sqlite database 42 | node build.js -f json sqlite // build json and sqlite databases 43 | ``` 44 | 45 | 46 | ## Compatible Fonts (for Sinhala definitions) 47 | - [Iskoola Pota](http://www.sinhalafonts.org/fonts/13091/iskoola_potha_unicode.html) 48 | - [Malithi Web](http://www.sinhalafonts.org/fonts/13092/malithi_web.html) 49 | 50 | ## License 51 | - MIT 52 | 53 | ## Contact 54 | - io@navinda.xyz 55 | 56 | ## Download 57 | 1. You can download this project in zip format. 58 | 2. You can also clone the project with Git by running, 59 | 60 | ```git 61 | $ git clone https://github.com/ipmanlk/OSDB-Project.git 62 | ``` 63 | -------------------------------------------------------------------------------- /formats/sqlite.js: -------------------------------------------------------------------------------- 1 | const sqlite3 = require('sqlite3').verbose(); 2 | let osdb = new sqlite3.Database(`${__dirname}/../outputs/sqlite/osdb.db`); 3 | 4 | // empty table 5 | const empty = async (table) => { 6 | await createTable(table); 7 | return new Promise((resolve, reject) => { 8 | osdb.run(`DELETE FROM ${table}`, [], (err) => { 9 | if (err) { 10 | console.log(`Error: Failed to empty database tables! ${err}`); 11 | reject(); 12 | } 13 | console.log("Success: Database has been cleared!."); 14 | resolve(); 15 | }); 16 | }); 17 | } 18 | 19 | // prepare sql statements to insert data to the database 20 | const save = async (table, data) => { 21 | return new Promise(async (resolve, reject) => { 22 | let rowCount = data.length; 23 | let values = ""; 24 | let count = 1; 25 | let limit = 1000; 26 | 27 | try { 28 | for (let row of data) { 29 | values += `("${row.word}", "${row.definitions}")`; 30 | if (count < limit) values += ","; 31 | if (count == limit) { 32 | let sql = `INSERT INTO ${table}(word, definitions) VALUES${values}`; 33 | rowCount -= limit; 34 | console.log(`Status: [SQLite] Buidling ${table} table. Left: ${rowCount} rows.`); 35 | await insert(sql); 36 | count = 0; 37 | values = ""; 38 | if ((rowCount - limit) < limit) { 39 | limit = rowCount; 40 | } 41 | } 42 | count++; 43 | } 44 | console.log(`Success: SQLite ${table} has been built!.`); 45 | resolve(); 46 | } catch (e) { 47 | console.log("Error: Unable to insert data!."); 48 | reject(); 49 | } 50 | }); 51 | } 52 | 53 | // insert data to database tables 54 | const insert = (sql) => { 55 | return new Promise((resolve, reject) => { 56 | osdb.run(sql, [], (err) => { 57 | if (err) { 58 | console.log(sql); 59 | console.log(`Error: Unable to insert data to the database! ${err}`); 60 | reject(); 61 | } 62 | resolve(); 63 | }); 64 | }) 65 | } 66 | 67 | // create tables if not exists 68 | const createTable = (table) => { 69 | return new Promise((resolve, reject) => { 70 | const sql = `CREATE TABLE IF NOT EXISTS ${table}(id INTEGER PRIMARY KEY, word TEXT UNIQUE, definitions TEXT NOT NULL)`; 71 | osdb.run(sql, [], (err) => { 72 | if (err) { 73 | console.log(`Error: Unable to create tables!.${err}`); 74 | reject(); 75 | } 76 | console.log("Success: Table has been created!."); 77 | resolve(); 78 | }); 79 | }) 80 | } 81 | 82 | const build = async (data) => { 83 | await empty("en2sn"); 84 | await empty("sn2en"); 85 | await save("en2sn", data.en2sn); 86 | await save("sn2en", data.sn2en); 87 | console.log(`Success: SQLite database has been generated!.`); 88 | } 89 | 90 | module.exports = { 91 | build 92 | } --------------------------------------------------------------------------------