├── .no-sublime-package ├── rbql ├── __main__.py ├── _version.py ├── __init__.py ├── rbql_pandas.py ├── csv_utils.py ├── rbql_sqlite.py ├── rbql_ipython.py ├── rbql_main.py └── rbql_csv.py ├── Default.sublime-keymap ├── DEV_README.md ├── rbql-js ├── .eslintrc.json ├── cli_parser.js ├── csv_utils.js └── cli_rbql.js ├── Default.sublime-commands ├── Context.sublime-menu ├── LICENSE ├── Main.sublime-menu ├── other ├── tests.py ├── CSV (Rainbow).tmLanguage.old ├── convert_color_scheme.py └── make_sublime_syntax.py ├── pregenerated_grammars ├── TSV (Rainbow).sublime-syntax ├── Rainbow_CSV_hex_20_Simple.sublime-syntax ├── Rainbow_CSV_hex_21_Simple.sublime-syntax ├── Rainbow_CSV_hex_22_Simple.sublime-syntax ├── Rainbow_CSV_hex_23_Simple.sublime-syntax ├── Rainbow_CSV_hex_25_Simple.sublime-syntax ├── Rainbow_CSV_hex_26_Simple.sublime-syntax ├── Rainbow_CSV_hex_2c_Simple.sublime-syntax ├── Rainbow_CSV_hex_2d_Simple.sublime-syntax ├── Rainbow_CSV_hex_3a_Simple.sublime-syntax ├── Rainbow_CSV_hex_3b_Simple.sublime-syntax ├── Rainbow_CSV_hex_3c_Simple.sublime-syntax ├── Rainbow_CSV_hex_3d_Simple.sublime-syntax ├── Rainbow_CSV_hex_3e_Simple.sublime-syntax ├── Rainbow_CSV_hex_40_Simple.sublime-syntax ├── Rainbow_CSV_hex_5f_Simple.sublime-syntax ├── Rainbow_CSV_hex_60_Simple.sublime-syntax ├── Rainbow_CSV_hex_7b_Simple.sublime-syntax ├── Rainbow_CSV_hex_7d_Simple.sublime-syntax ├── Rainbow_CSV_hex_7e_Simple.sublime-syntax ├── Rainbow_CSV_hex_24_Simple.sublime-syntax ├── Rainbow_CSV_hex_28_Simple.sublime-syntax ├── Rainbow_CSV_hex_29_Simple.sublime-syntax ├── Rainbow_CSV_hex_2a_Simple.sublime-syntax ├── Rainbow_CSV_hex_2b_Simple.sublime-syntax ├── Rainbow_CSV_hex_2e_Simple.sublime-syntax ├── Rainbow_CSV_hex_2f_Simple.sublime-syntax ├── Rainbow_CSV_hex_3f_Simple.sublime-syntax ├── Rainbow_CSV_hex_5b_Simple.sublime-syntax ├── Rainbow_CSV_hex_5c_Simple.sublime-syntax ├── Rainbow_CSV_hex_5d_Simple.sublime-syntax ├── Rainbow_CSV_hex_5e_Simple.sublime-syntax ├── Rainbow_CSV_hex_27_Simple.sublime-syntax ├── Rainbow_CSV_hex_7c_Simple.sublime-syntax ├── Rainbow_CSV_hex_2c_quoted_rfc.sublime-syntax ├── Rainbow_CSV_hex_3b_quoted_rfc.sublime-syntax ├── CSV (Rainbow).sublime-syntax └── Rainbow_CSV_hex_3b_Standard.sublime-syntax ├── RainbowCSV.sublime-color-scheme ├── RainbowCSV.sublime-settings ├── sublime_rbql.py ├── auto_syntax.py └── README.md /.no-sublime-package: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rbql/__main__.py: -------------------------------------------------------------------------------- 1 | from .rbql_main import main 2 | main() 3 | -------------------------------------------------------------------------------- /rbql/_version.py: -------------------------------------------------------------------------------- 1 | # Explanation of this file purpose: https://stackoverflow.com/a/16084844/2898283 2 | __version__ = '0.27.0' 3 | 4 | -------------------------------------------------------------------------------- /Default.sublime-keymap: -------------------------------------------------------------------------------- 1 | [ 2 | {"keys": ["f5"], "command": "run_query", "context": [{"key": "setting.rainbow_mode", "operator": "equal", "operand": true}]} 3 | ] 4 | -------------------------------------------------------------------------------- /DEV_README.md: -------------------------------------------------------------------------------- 1 | # Debugging the extension 2 | 3 | Copy the extension to the sublime text packages dir, e.g.: 4 | ``` 5 | rm -rf "/mnt/c/Users/mecha/AppData/Roaming/Sublime Text 3/Packages/rainbow_csv" && cp -r sublime_rainbow_csv "/mnt/c/Users/mecha/AppData/Roaming/Sublime Text 3/Packages/rainbow_csv" 6 | ``` 7 | -------------------------------------------------------------------------------- /rbql/__init__.py: -------------------------------------------------------------------------------- 1 | from .rbql_engine import query 2 | from .rbql_engine import query_table 3 | from .rbql_engine import exception_to_error_info 4 | 5 | from ._version import __version__ 6 | 7 | from .rbql_csv import query_csv 8 | 9 | from .rbql_pandas import query_dataframe as query_pandas_dataframe 10 | 11 | from .rbql_ipython import load_ipython_extension 12 | -------------------------------------------------------------------------------- /rbql-js/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": false, 4 | "commonjs": true, 5 | "es6": true, 6 | "node": true 7 | }, 8 | "parserOptions": { 9 | "ecmaFeatures": { 10 | "jsx": true 11 | }, 12 | "sourceType": "module", 13 | "ecmaVersion": 2018 14 | }, 15 | "rules": { 16 | "no-const-assign": "warn", 17 | "no-this-before-super": "warn", 18 | "no-undef": "warn", 19 | "semi": [2, "always"], 20 | "no-unreachable": "warn", 21 | "no-unused-vars": "warn", 22 | "constructor-super": "warn", 23 | "no-trailing-spaces": "error", 24 | "valid-typeof": "warn" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Default.sublime-commands: -------------------------------------------------------------------------------- 1 | [ 2 | { "caption": "Rainbow CSV: Run RBQL query", "command": "run_query" }, 3 | { "caption": "Rainbow CSV: Align CSV columns with spaces", "command": "align" }, 4 | { "caption": "Rainbow CSV: Shrink CSV table by trimming spaces from fields", "command": "shrink" }, 5 | { "caption": "Rainbow CSV: CSVLint", "command": "csv_lint" }, 6 | { "caption": "Rainbow CSV: Enable Simple", "command": "enable_simple" }, 7 | { "caption": "Rainbow CSV: Enable Quoted", "command": "enable_quoted" }, 8 | { "caption": "Rainbow CSV: Enable RFC", "command": "enable_rfc" }, 9 | { 10 | "caption": "Rainbow CSV Settings", 11 | "command": "edit_settings", 12 | "args": { 13 | "base_file": "${packages}/rainbow_csv/RainbowCSV.sublime-settings", 14 | "default": "// Settings in here override those in \"RainbowCSV.sublime-settings\"\n\n{\n\t$0\n}\n" 15 | } 16 | }, 17 | ] 18 | -------------------------------------------------------------------------------- /Context.sublime-menu: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "caption": "Rainbow CSV", 4 | "id": "rainbow_csv", 5 | "children": 6 | [ 7 | { "command": "enable_auto", "caption": "Enable: set selected character or text as separator" }, 8 | { "command": "disable", "caption": "Disable" }, 9 | { "command": "run_query", "caption": "Run RBQL query (F5)" }, 10 | { "command": "set_table_name", "caption": "Set table name for RBQL" }, 11 | { "command": "align", "caption": "Align (with trailing spaces)" }, 12 | { "command": "shrink", "caption": "Shrink (trim spaces from fields)" }, 13 | { "command": "csv_lint", "caption": "CSVLint" }, 14 | { "command": "edit_settings", "caption": "Settings", 15 | "args": { 16 | "base_file": "${packages}/rainbow_csv/RainbowCSV.sublime-settings", 17 | "default": "// Settings in here override those in \"RainbowCSV.sublime-settings\"\n\n{\n\t$0\n}\n" 18 | } 19 | } 20 | ] 21 | } 22 | ] 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Dmitry Ignatovich 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Main.sublime-menu: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "caption": "Preferences", 4 | "children": [ 5 | { 6 | "caption": "Package Settings", 7 | "children": [ 8 | { 9 | "caption": "Rainbow CSV", 10 | "children": [ 11 | { 12 | "caption": "Settings", 13 | "command": "edit_settings", 14 | "args": { 15 | "base_file": "${packages}/rainbow_csv/RainbowCSV.sublime-settings", 16 | "default": "// Settings in here override those in \"RainbowCSV.sublime-settings\"\n\n{\n\t$0\n}\n" 17 | } 18 | }, 19 | ], 20 | "id": "rainbowcsv" 21 | } 22 | ], 23 | "id": "package-settings", 24 | "mnemonic": "P" 25 | } 26 | ], 27 | "id": "preferences", 28 | "mnemonic": "n" 29 | } 30 | ] 31 | -------------------------------------------------------------------------------- /other/tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import os 5 | import argparse 6 | import random 7 | import unittest 8 | 9 | plugin_directory = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 10 | sys.path.append(plugin_directory) 11 | 12 | import rainbow_utils 13 | 14 | 15 | class TestStatuslineGeneration(unittest.TestCase): 16 | def test_run1(self): 17 | # 10,a,b,20000,5 18 | # a1 a2 a3 a4 a5 19 | test_stln = rainbow_utils.generate_tab_statusline(1, ['10', 'a', 'b', '20000', '5']) 20 | test_stln_str = ''.join(test_stln) 21 | canonic_stln = 'a1 a2 a3 a4 a5' 22 | self.assertEqual(test_stln_str, canonic_stln) 23 | 24 | def test_run2(self): 25 | # 10 a b 20000 5 26 | # a1 a2 a3 a4 a5 27 | test_stln = rainbow_utils.generate_tab_statusline(4, ['10', 'a', 'b', '20000', '5']) 28 | test_stln_str = ''.join(test_stln) 29 | canonic_stln = 'a1 a2 a3 a4 a5' 30 | self.assertEqual(test_stln_str, canonic_stln) 31 | 32 | def test_run3(self): 33 | # 10 a b 20000 5 34 | # a1 a2 35 | test_stln = rainbow_utils.generate_tab_statusline(4, ['10', 'a', 'b', '20000', '5'], 9) 36 | test_stln_str = ''.join(test_stln) 37 | canonic_stln = 'a1 a2' 38 | self.assertEqual(test_stln_str, canonic_stln) 39 | 40 | 41 | def test_run4(self): 42 | # 10 a b 20000 5 43 | # a1 a2 a3 44 | test_stln = rainbow_utils.generate_tab_statusline(4, ['10', 'a', 'b', '20000', '5'], 10) 45 | test_stln_str = ''.join(test_stln) 46 | canonic_stln = 'a1 a2 a3' 47 | self.assertEqual(test_stln_str, canonic_stln) 48 | -------------------------------------------------------------------------------- /pregenerated_grammars/TSV (Rainbow).sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'TSV (Rainbow)' 4 | file_extensions: [tsv] 5 | scope: text.csv.rbcsmn9 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '\t' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '\t' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '\t' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '\t' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '\t' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '\t' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '\t' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '\t' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '\t' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '\t' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_20_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV Simple' 4 | file_extensions: [rbcsmn32] 5 | scope: text.csv.rbcsmn32 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: ' ' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: ' ' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: ' ' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: ' ' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: ' ' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: ' ' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: ' ' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: ' ' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: ' ' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: ' ' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_21_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV ! Simple' 4 | file_extensions: [rbcsmn33] 5 | scope: text.csv.rbcsmn33 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '!' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '!' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '!' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '!' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '!' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '!' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '!' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '!' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '!' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '!' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_22_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV " Simple' 4 | file_extensions: [rbcsmn34] 5 | scope: text.csv.rbcsmn34 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '"' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '"' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '"' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '"' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '"' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '"' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '"' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '"' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '"' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '"' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_23_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV # Simple' 4 | file_extensions: [rbcsmn35] 5 | scope: text.csv.rbcsmn35 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '#' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '#' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '#' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '#' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '#' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '#' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '#' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '#' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '#' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '#' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_25_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV % Simple' 4 | file_extensions: [rbcsmn37] 5 | scope: text.csv.rbcsmn37 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '%' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '%' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '%' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '%' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '%' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '%' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '%' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '%' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '%' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '%' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_26_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV & Simple' 4 | file_extensions: [rbcsmn38] 5 | scope: text.csv.rbcsmn38 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '&' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '&' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '&' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '&' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '&' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '&' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '&' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '&' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '&' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '&' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_2c_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV , Simple' 4 | file_extensions: [rbcsmn44] 5 | scope: text.csv.rbcsmn44 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: ',' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: ',' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: ',' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: ',' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: ',' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: ',' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: ',' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: ',' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: ',' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: ',' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_2d_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV - Simple' 4 | file_extensions: [rbcsmn45] 5 | scope: text.csv.rbcsmn45 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '-' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '-' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '-' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '-' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '-' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '-' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '-' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '-' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '-' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '-' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_3a_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV : Simple' 4 | file_extensions: [rbcsmn58] 5 | scope: text.csv.rbcsmn58 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: ':' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: ':' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: ':' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: ':' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: ':' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: ':' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: ':' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: ':' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: ':' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: ':' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_3b_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV ; Simple' 4 | file_extensions: [rbcsmn59] 5 | scope: text.csv.rbcsmn59 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: ';' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: ';' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: ';' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: ';' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: ';' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: ';' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: ';' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: ';' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: ';' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: ';' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_3c_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV < Simple' 4 | file_extensions: [rbcsmn60] 5 | scope: text.csv.rbcsmn60 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '<' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '<' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '<' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '<' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '<' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '<' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '<' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '<' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '<' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '<' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_3d_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV = Simple' 4 | file_extensions: [rbcsmn61] 5 | scope: text.csv.rbcsmn61 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '=' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '=' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '=' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '=' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '=' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '=' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '=' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '=' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '=' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '=' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_3e_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV > Simple' 4 | file_extensions: [rbcsmn62] 5 | scope: text.csv.rbcsmn62 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '>' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '>' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '>' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '>' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '>' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '>' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '>' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '>' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '>' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '>' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_40_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV @ Simple' 4 | file_extensions: [rbcsmn64] 5 | scope: text.csv.rbcsmn64 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '@' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '@' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '@' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '@' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '@' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '@' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '@' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '@' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '@' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '@' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_5f_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV _ Simple' 4 | file_extensions: [rbcsmn95] 5 | scope: text.csv.rbcsmn95 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '_' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '_' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '_' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '_' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '_' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '_' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '_' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '_' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '_' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '_' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_60_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV ` Simple' 4 | file_extensions: [rbcsmn96] 5 | scope: text.csv.rbcsmn96 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '`' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '`' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '`' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '`' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '`' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '`' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '`' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '`' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '`' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '`' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_7b_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV { Simple' 4 | file_extensions: [rbcsmn123] 5 | scope: text.csv.rbcsmn123 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '{' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '{' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '{' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '{' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '{' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '{' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '{' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '{' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '{' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '{' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_7d_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV } Simple' 4 | file_extensions: [rbcsmn125] 5 | scope: text.csv.rbcsmn125 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '}' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '}' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '}' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '}' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '}' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '}' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '}' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '}' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '}' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '}' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_7e_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV ~ Simple' 4 | file_extensions: [rbcsmn126] 5 | scope: text.csv.rbcsmn126 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '~' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '~' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '~' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '~' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '~' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '~' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '~' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '~' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '~' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '~' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_24_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV $ Simple' 4 | file_extensions: [rbcsmn36] 5 | scope: text.csv.rbcsmn36 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '\$' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '\$' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '\$' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '\$' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '\$' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '\$' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '\$' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '\$' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '\$' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '\$' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_28_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV ( Simple' 4 | file_extensions: [rbcsmn40] 5 | scope: text.csv.rbcsmn40 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '\(' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '\(' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '\(' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '\(' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '\(' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '\(' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '\(' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '\(' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '\(' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '\(' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_29_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV ) Simple' 4 | file_extensions: [rbcsmn41] 5 | scope: text.csv.rbcsmn41 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '\)' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '\)' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '\)' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '\)' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '\)' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '\)' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '\)' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '\)' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '\)' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '\)' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_2a_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV * Simple' 4 | file_extensions: [rbcsmn42] 5 | scope: text.csv.rbcsmn42 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '\*' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '\*' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '\*' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '\*' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '\*' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '\*' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '\*' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '\*' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '\*' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '\*' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_2b_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV + Simple' 4 | file_extensions: [rbcsmn43] 5 | scope: text.csv.rbcsmn43 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '\+' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '\+' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '\+' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '\+' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '\+' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '\+' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '\+' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '\+' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '\+' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '\+' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_2e_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV . Simple' 4 | file_extensions: [rbcsmn46] 5 | scope: text.csv.rbcsmn46 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '\.' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '\.' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '\.' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '\.' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '\.' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '\.' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '\.' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '\.' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '\.' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '\.' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_2f_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV / Simple' 4 | file_extensions: [rbcsmn47] 5 | scope: text.csv.rbcsmn47 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '\/' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '\/' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '\/' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '\/' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '\/' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '\/' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '\/' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '\/' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '\/' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '\/' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_3f_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV ? Simple' 4 | file_extensions: [rbcsmn63] 5 | scope: text.csv.rbcsmn63 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '\?' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '\?' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '\?' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '\?' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '\?' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '\?' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '\?' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '\?' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '\?' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '\?' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_5b_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV [ Simple' 4 | file_extensions: [rbcsmn91] 5 | scope: text.csv.rbcsmn91 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '\[' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '\[' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '\[' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '\[' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '\[' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '\[' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '\[' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '\[' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '\[' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '\[' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_5c_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV \ Simple' 4 | file_extensions: [rbcsmn92] 5 | scope: text.csv.rbcsmn92 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '\\' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '\\' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '\\' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '\\' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '\\' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '\\' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '\\' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '\\' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '\\' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '\\' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_5d_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV ] Simple' 4 | file_extensions: [rbcsmn93] 5 | scope: text.csv.rbcsmn93 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '\]' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '\]' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '\]' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '\]' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '\]' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '\]' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '\]' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '\]' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '\]' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '\]' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_5e_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV ^ Simple' 4 | file_extensions: [rbcsmn94] 5 | scope: text.csv.rbcsmn94 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '\^' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '\^' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '\^' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '\^' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '\^' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '\^' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '\^' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '\^' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '\^' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '\^' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_27_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV '' Simple' 4 | file_extensions: [rbcsmn39] 5 | scope: text.csv.rbcsmn39 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '''' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '''' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '''' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '''' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '''' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '''' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '''' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '''' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '''' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '''' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_7c_Simple.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV | Simple' 4 | file_extensions: [rbcsmn124] 5 | scope: text.csv.rbcsmn124 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | rainbow1: 14 | - meta_content_scope: rainbow1 15 | - match: '\|' 16 | set: rainbow2 17 | - match: '$' 18 | pop: true 19 | 20 | rainbow2: 21 | - meta_content_scope: keyword.rainbow2 22 | - match: '\|' 23 | set: rainbow3 24 | - match: '$' 25 | pop: true 26 | 27 | rainbow3: 28 | - meta_content_scope: entity.name.rainbow3 29 | - match: '\|' 30 | set: rainbow4 31 | - match: '$' 32 | pop: true 33 | 34 | rainbow4: 35 | - meta_content_scope: comment.rainbow4 36 | - match: '\|' 37 | set: rainbow5 38 | - match: '$' 39 | pop: true 40 | 41 | rainbow5: 42 | - meta_content_scope: string.rainbow5 43 | - match: '\|' 44 | set: rainbow6 45 | - match: '$' 46 | pop: true 47 | 48 | rainbow6: 49 | - meta_content_scope: entity.name.tag.rainbow6 50 | - match: '\|' 51 | set: rainbow7 52 | - match: '$' 53 | pop: true 54 | 55 | rainbow7: 56 | - meta_content_scope: storage.type.rainbow7 57 | - match: '\|' 58 | set: rainbow8 59 | - match: '$' 60 | pop: true 61 | 62 | rainbow8: 63 | - meta_content_scope: support.rainbow8 64 | - match: '\|' 65 | set: rainbow9 66 | - match: '$' 67 | pop: true 68 | 69 | rainbow9: 70 | - meta_content_scope: constant.language.rainbow9 71 | - match: '\|' 72 | set: rainbow10 73 | - match: '$' 74 | pop: true 75 | 76 | rainbow10: 77 | - meta_content_scope: variable.language.rainbow10 78 | - match: '\|' 79 | set: rainbow1 80 | - match: '$' 81 | pop: true 82 | -------------------------------------------------------------------------------- /other/CSV (Rainbow).tmLanguage.old: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | fileTypes 6 | 7 | rbcst44 8 | 9 | name 10 | CSV (Rainbow) 11 | patterns 12 | 13 | 14 | captures 15 | 16 | 1 17 | 18 | name 19 | rainbow1 20 | 21 | 2 22 | 23 | name 24 | keyword.rainbow2 25 | 26 | 3 27 | 28 | name 29 | entity.name.rainbow3 30 | 31 | 4 32 | 33 | name 34 | comment.rainbow4 35 | 36 | 5 37 | 38 | name 39 | string.rainbow5 40 | 41 | 6 42 | 43 | name 44 | entity.name.tag.rainbow6 45 | 46 | 7 47 | 48 | name 49 | storage.type.rainbow7 50 | 51 | 8 52 | 53 | name 54 | support.rainbow8 55 | 56 | 9 57 | 58 | name 59 | constant.language.rainbow9 60 | 61 | 10 62 | 63 | name 64 | variable.language.rainbow10 65 | 66 | 67 | match 68 | ((?:"(?:[^"]*"")*[^"]*",?)|(?:[^,]*,?))?((?:"(?:[^"]*"")*[^"]*",?)|(?:[^,]*,?))?((?:"(?:[^"]*"")*[^"]*",?)|(?:[^,]*,?))?((?:"(?:[^"]*"")*[^"]*",?)|(?:[^,]*,?))?((?:"(?:[^"]*"")*[^"]*",?)|(?:[^,]*,?))?((?:"(?:[^"]*"")*[^"]*",?)|(?:[^,]*,?))?((?:"(?:[^"]*"")*[^"]*",?)|(?:[^,]*,?))?((?:"(?:[^"]*"")*[^"]*",?)|(?:[^,]*,?))?((?:"(?:[^"]*"")*[^"]*",?)|(?:[^,]*,?))?((?:"(?:[^"]*"")*[^"]*",?)|(?:[^,]*,?))? 69 | name 70 | rainbowline 71 | 72 | 73 | scopeName 74 | text.rbcst44 75 | uuid 76 | ca03e352-04ef-4340-9a6b-9b99aae1b044 77 | 78 | 79 | -------------------------------------------------------------------------------- /rbql-js/cli_parser.js: -------------------------------------------------------------------------------- 1 | function die(error_msg) { 2 | console.error('Error: ' + error_msg); 3 | process.exit(1); 4 | } 5 | 6 | 7 | function normalize_cli_key(cli_key) { 8 | return cli_key.replace(/^-*/, ''); 9 | } 10 | 11 | 12 | function show_help(scheme, description, epilog) { 13 | if (description) 14 | console.log(description); 15 | console.log('Options:'); 16 | for (var k in scheme) { 17 | if (scheme[k].hasOwnProperty('hidden')) { 18 | continue; 19 | } 20 | let metavar = ''; 21 | if (scheme[k].hasOwnProperty('metavar')) 22 | metavar = ' ' + scheme[k]['metavar']; 23 | console.log(' ' + k + metavar); 24 | if (scheme[k].hasOwnProperty('default')) { 25 | console.log(' Default: "' + scheme[k]['default'] + '"'); 26 | } 27 | console.log(' ' + scheme[k]['help']); 28 | console.log(); 29 | } 30 | if (epilog) 31 | console.log(epilog); 32 | } 33 | 34 | 35 | function parse_cmd_args(cmd_args, scheme, description=null, epilog=null) { 36 | var result = {}; 37 | for (var arg_key in scheme) { 38 | var arg_info = scheme[arg_key]; 39 | if (arg_info.hasOwnProperty('default')) 40 | result[normalize_cli_key(arg_key)] = arg_info['default']; 41 | if (arg_info.hasOwnProperty('boolean')) 42 | result[normalize_cli_key(arg_key)] = false; 43 | } 44 | cmd_args = cmd_args.slice(2); 45 | var i = 0; 46 | while(i < cmd_args.length) { 47 | var arg_key = cmd_args[i]; 48 | if (arg_key == '--help' || arg_key == '-h') { 49 | show_help(scheme, description, epilog); 50 | process.exit(0); 51 | } 52 | i += 1; 53 | if (!scheme.hasOwnProperty(arg_key)) { 54 | die(`unknown argument: ${arg_key}`); 55 | } 56 | var arg_info = scheme[arg_key]; 57 | var normalized_key = normalize_cli_key(arg_key); 58 | if (arg_info['boolean']) { 59 | result[normalized_key] = true; 60 | continue; 61 | } 62 | if (i >= cmd_args.length) { 63 | die(`no CLI value for key: ${arg_key}`); 64 | } 65 | var arg_value = cmd_args[i]; 66 | i += 1; 67 | result[normalized_key] = arg_value; 68 | } 69 | return result; 70 | } 71 | 72 | 73 | 74 | module.exports.parse_cmd_args = parse_cmd_args; 75 | -------------------------------------------------------------------------------- /RainbowCSV.sublime-color-scheme: -------------------------------------------------------------------------------- 1 | { 2 | "globals": 3 | { 4 | "background": "#272822", 5 | "caret": "#F8F8F0", 6 | "foreground": "#F8F8F2", 7 | "invisibles": "#3B3A32", 8 | "line_highlight": "#3E3D32", 9 | "selection": "#49483E", 10 | "find_highlight": "#FFE792", 11 | "find_highlight_foreground": "#000000", 12 | "selection_border": "#222218", 13 | "active_guide": "#9D550FB0", 14 | "misspelling": "#F92672", 15 | "brackets_foreground": "#F8F8F2A5", 16 | "brackets_options": "underline", 17 | "bracket_contents_foreground": "#F8F8F2A5", 18 | "bracket_contents_options": "underline", 19 | "tags_options": "stippled_underline" 20 | }, 21 | "rules": 22 | [ 23 | { 24 | "name": "rainbow csv rainbow1", 25 | "scope": "rainbow1", 26 | "foreground": "#E6194B" 27 | }, 28 | { 29 | "name": "rainbow csv rainbow2", 30 | "scope": "keyword.rainbow2", 31 | "foreground": "#3CB44B" 32 | }, 33 | { 34 | "name": "rainbow csv rainbow3", 35 | "scope": "entity.name.rainbow3", 36 | "foreground": "#FFE119" 37 | }, 38 | { 39 | "name": "rainbow csv rainbow4", 40 | "scope": "comment.rainbow4", 41 | "foreground": "#0082C8" 42 | }, 43 | { 44 | "name": "rainbow csv rainbow5", 45 | "scope": "string.rainbow5", 46 | "foreground": "#FABEBE" 47 | }, 48 | { 49 | "name": "rainbow csv rainbow6", 50 | "scope": "entity.name.tag.rainbow6", 51 | "foreground": "#46F0F0" 52 | }, 53 | { 54 | "name": "rainbow csv rainbow7", 55 | "scope": "storage.type.rainbow7", 56 | "foreground": "#F032E6" 57 | }, 58 | { 59 | "name": "rainbow csv rainbow8", 60 | "scope": "support.rainbow8", 61 | "foreground": "#008080" 62 | }, 63 | { 64 | "name": "rainbow csv rainbow9", 65 | "scope": "constant.language.rainbow9", 66 | "foreground": "#F58231" 67 | }, 68 | { 69 | "name": "rainbow csv rainbow10", 70 | "scope": "variable.language.rainbow10", 71 | "foreground": "#FFFFFF" 72 | }, 73 | { 74 | "name": "rainbow csv rainbowerror", 75 | "scope": "invalid.rainbowerror", 76 | "foreground": "#FFFFFF", 77 | "background": "#FF0000" 78 | } 79 | ] 80 | } 81 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_2c_quoted_rfc.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV , RFC' 4 | file_extensions: [rbcstn44] 5 | scope: text.csv.rbcstn44 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | quoted_field: 14 | - match: '""' 15 | scope: meta.rainbow.double-quote-escaped 16 | - match: '"' 17 | pop: true 18 | 19 | rainbow1: 20 | - meta_content_scope: rainbow1 21 | - match: ',' 22 | set: rainbow2 23 | - match: '$' 24 | pop: true 25 | - match: '"' 26 | push: quoted_field 27 | 28 | rainbow2: 29 | - meta_content_scope: keyword.rainbow2 30 | - match: ',' 31 | set: rainbow3 32 | - match: '$' 33 | pop: true 34 | - match: '"' 35 | push: quoted_field 36 | 37 | rainbow3: 38 | - meta_content_scope: entity.name.rainbow3 39 | - match: ',' 40 | set: rainbow4 41 | - match: '$' 42 | pop: true 43 | - match: '"' 44 | push: quoted_field 45 | 46 | rainbow4: 47 | - meta_content_scope: comment.rainbow4 48 | - match: ',' 49 | set: rainbow5 50 | - match: '$' 51 | pop: true 52 | - match: '"' 53 | push: quoted_field 54 | 55 | rainbow5: 56 | - meta_content_scope: string.rainbow5 57 | - match: ',' 58 | set: rainbow6 59 | - match: '$' 60 | pop: true 61 | - match: '"' 62 | push: quoted_field 63 | 64 | rainbow6: 65 | - meta_content_scope: entity.name.tag.rainbow6 66 | - match: ',' 67 | set: rainbow7 68 | - match: '$' 69 | pop: true 70 | - match: '"' 71 | push: quoted_field 72 | 73 | rainbow7: 74 | - meta_content_scope: storage.type.rainbow7 75 | - match: ',' 76 | set: rainbow8 77 | - match: '$' 78 | pop: true 79 | - match: '"' 80 | push: quoted_field 81 | 82 | rainbow8: 83 | - meta_content_scope: support.rainbow8 84 | - match: ',' 85 | set: rainbow9 86 | - match: '$' 87 | pop: true 88 | - match: '"' 89 | push: quoted_field 90 | 91 | rainbow9: 92 | - meta_content_scope: constant.language.rainbow9 93 | - match: ',' 94 | set: rainbow10 95 | - match: '$' 96 | pop: true 97 | - match: '"' 98 | push: quoted_field 99 | 100 | rainbow10: 101 | - meta_content_scope: variable.language.rainbow10 102 | - match: ',' 103 | set: rainbow1 104 | - match: '$' 105 | pop: true 106 | - match: '"' 107 | push: quoted_field 108 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_3b_quoted_rfc.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV ; RFC' 4 | file_extensions: [rbcstn59] 5 | scope: text.csv.rbcstn59 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | quoted_field: 14 | - match: '""' 15 | scope: meta.rainbow.double-quote-escaped 16 | - match: '"' 17 | pop: true 18 | 19 | rainbow1: 20 | - meta_content_scope: rainbow1 21 | - match: ';' 22 | set: rainbow2 23 | - match: '$' 24 | pop: true 25 | - match: '"' 26 | push: quoted_field 27 | 28 | rainbow2: 29 | - meta_content_scope: keyword.rainbow2 30 | - match: ';' 31 | set: rainbow3 32 | - match: '$' 33 | pop: true 34 | - match: '"' 35 | push: quoted_field 36 | 37 | rainbow3: 38 | - meta_content_scope: entity.name.rainbow3 39 | - match: ';' 40 | set: rainbow4 41 | - match: '$' 42 | pop: true 43 | - match: '"' 44 | push: quoted_field 45 | 46 | rainbow4: 47 | - meta_content_scope: comment.rainbow4 48 | - match: ';' 49 | set: rainbow5 50 | - match: '$' 51 | pop: true 52 | - match: '"' 53 | push: quoted_field 54 | 55 | rainbow5: 56 | - meta_content_scope: string.rainbow5 57 | - match: ';' 58 | set: rainbow6 59 | - match: '$' 60 | pop: true 61 | - match: '"' 62 | push: quoted_field 63 | 64 | rainbow6: 65 | - meta_content_scope: entity.name.tag.rainbow6 66 | - match: ';' 67 | set: rainbow7 68 | - match: '$' 69 | pop: true 70 | - match: '"' 71 | push: quoted_field 72 | 73 | rainbow7: 74 | - meta_content_scope: storage.type.rainbow7 75 | - match: ';' 76 | set: rainbow8 77 | - match: '$' 78 | pop: true 79 | - match: '"' 80 | push: quoted_field 81 | 82 | rainbow8: 83 | - meta_content_scope: support.rainbow8 84 | - match: ';' 85 | set: rainbow9 86 | - match: '$' 87 | pop: true 88 | - match: '"' 89 | push: quoted_field 90 | 91 | rainbow9: 92 | - meta_content_scope: constant.language.rainbow9 93 | - match: ';' 94 | set: rainbow10 95 | - match: '$' 96 | pop: true 97 | - match: '"' 98 | push: quoted_field 99 | 100 | rainbow10: 101 | - meta_content_scope: variable.language.rainbow10 102 | - match: ';' 103 | set: rainbow1 104 | - match: '$' 105 | pop: true 106 | - match: '"' 107 | push: quoted_field 108 | -------------------------------------------------------------------------------- /pregenerated_grammars/CSV (Rainbow).sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'CSV (Rainbow)' 4 | file_extensions: [csv] 5 | scope: text.csv.rbcstn44 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | quoted_field: 14 | - match: '""' 15 | scope: meta.rainbow.double-quote-escaped 16 | - match: '"' 17 | pop: true 18 | - match: '$' 19 | pop: true 20 | 21 | rainbow1: 22 | - meta_content_scope: rainbow1 23 | - match: ',' 24 | set: rainbow2 25 | - match: '$' 26 | pop: true 27 | - match: '"' 28 | push: quoted_field 29 | 30 | rainbow2: 31 | - meta_content_scope: keyword.rainbow2 32 | - match: ',' 33 | set: rainbow3 34 | - match: '$' 35 | pop: true 36 | - match: '"' 37 | push: quoted_field 38 | 39 | rainbow3: 40 | - meta_content_scope: entity.name.rainbow3 41 | - match: ',' 42 | set: rainbow4 43 | - match: '$' 44 | pop: true 45 | - match: '"' 46 | push: quoted_field 47 | 48 | rainbow4: 49 | - meta_content_scope: comment.rainbow4 50 | - match: ',' 51 | set: rainbow5 52 | - match: '$' 53 | pop: true 54 | - match: '"' 55 | push: quoted_field 56 | 57 | rainbow5: 58 | - meta_content_scope: string.rainbow5 59 | - match: ',' 60 | set: rainbow6 61 | - match: '$' 62 | pop: true 63 | - match: '"' 64 | push: quoted_field 65 | 66 | rainbow6: 67 | - meta_content_scope: entity.name.tag.rainbow6 68 | - match: ',' 69 | set: rainbow7 70 | - match: '$' 71 | pop: true 72 | - match: '"' 73 | push: quoted_field 74 | 75 | rainbow7: 76 | - meta_content_scope: storage.type.rainbow7 77 | - match: ',' 78 | set: rainbow8 79 | - match: '$' 80 | pop: true 81 | - match: '"' 82 | push: quoted_field 83 | 84 | rainbow8: 85 | - meta_content_scope: support.rainbow8 86 | - match: ',' 87 | set: rainbow9 88 | - match: '$' 89 | pop: true 90 | - match: '"' 91 | push: quoted_field 92 | 93 | rainbow9: 94 | - meta_content_scope: constant.language.rainbow9 95 | - match: ',' 96 | set: rainbow10 97 | - match: '$' 98 | pop: true 99 | - match: '"' 100 | push: quoted_field 101 | 102 | rainbow10: 103 | - meta_content_scope: variable.language.rainbow10 104 | - match: ',' 105 | set: rainbow1 106 | - match: '$' 107 | pop: true 108 | - match: '"' 109 | push: quoted_field 110 | -------------------------------------------------------------------------------- /pregenerated_grammars/Rainbow_CSV_hex_3b_Standard.sublime-syntax: -------------------------------------------------------------------------------- 1 | %YAML 1.2 2 | --- 3 | name: 'Rainbow CSV ; Standard' 4 | file_extensions: [rbcstn59] 5 | scope: text.csv.rbcstn59 6 | 7 | 8 | contexts: 9 | main: 10 | - match: '^' 11 | push: rainbow1 12 | 13 | quoted_field: 14 | - match: '""' 15 | scope: meta.rainbow.double-quote-escaped 16 | - match: '"' 17 | pop: true 18 | - match: '$' 19 | pop: true 20 | 21 | rainbow1: 22 | - meta_content_scope: rainbow1 23 | - match: ';' 24 | set: rainbow2 25 | - match: '$' 26 | pop: true 27 | - match: '"' 28 | push: quoted_field 29 | 30 | rainbow2: 31 | - meta_content_scope: keyword.rainbow2 32 | - match: ';' 33 | set: rainbow3 34 | - match: '$' 35 | pop: true 36 | - match: '"' 37 | push: quoted_field 38 | 39 | rainbow3: 40 | - meta_content_scope: entity.name.rainbow3 41 | - match: ';' 42 | set: rainbow4 43 | - match: '$' 44 | pop: true 45 | - match: '"' 46 | push: quoted_field 47 | 48 | rainbow4: 49 | - meta_content_scope: comment.rainbow4 50 | - match: ';' 51 | set: rainbow5 52 | - match: '$' 53 | pop: true 54 | - match: '"' 55 | push: quoted_field 56 | 57 | rainbow5: 58 | - meta_content_scope: string.rainbow5 59 | - match: ';' 60 | set: rainbow6 61 | - match: '$' 62 | pop: true 63 | - match: '"' 64 | push: quoted_field 65 | 66 | rainbow6: 67 | - meta_content_scope: entity.name.tag.rainbow6 68 | - match: ';' 69 | set: rainbow7 70 | - match: '$' 71 | pop: true 72 | - match: '"' 73 | push: quoted_field 74 | 75 | rainbow7: 76 | - meta_content_scope: storage.type.rainbow7 77 | - match: ';' 78 | set: rainbow8 79 | - match: '$' 80 | pop: true 81 | - match: '"' 82 | push: quoted_field 83 | 84 | rainbow8: 85 | - meta_content_scope: support.rainbow8 86 | - match: ';' 87 | set: rainbow9 88 | - match: '$' 89 | pop: true 90 | - match: '"' 91 | push: quoted_field 92 | 93 | rainbow9: 94 | - meta_content_scope: constant.language.rainbow9 95 | - match: ';' 96 | set: rainbow10 97 | - match: '$' 98 | pop: true 99 | - match: '"' 100 | push: quoted_field 101 | 102 | rainbow10: 103 | - meta_content_scope: variable.language.rainbow10 104 | - match: ';' 105 | set: rainbow1 106 | - match: '$' 107 | pop: true 108 | - match: '"' 109 | push: quoted_field 110 | -------------------------------------------------------------------------------- /other/convert_color_scheme.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import os 5 | import argparse 6 | import random 7 | import xml.etree.ElementTree as ET 8 | import copy 9 | import xml.dom.minidom 10 | 11 | plugin_directory = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 12 | sys.path.append(plugin_directory) 13 | 14 | color_entries = list() 15 | color_entries.append(('rainbow1', '#E6194B', None)) 16 | color_entries.append(('keyword.rainbow2', '#3CB44B', None)) 17 | color_entries.append(('entity.name.rainbow3', '#FFE119', None)) 18 | color_entries.append(('comment.rainbow4', '#0082C8', None)) 19 | color_entries.append(('string.rainbow5', '#FABEBE', None)) 20 | color_entries.append(('entity.name.tag.rainbow6', '#46F0F0', None)) 21 | color_entries.append(('storage.type.rainbow7', '#F032E6', None)) 22 | color_entries.append(('support.rainbow8', '#008080', None)) 23 | color_entries.append(('constant.language.rainbow9', '#F58231', None)) 24 | color_entries.append(('variable.language.rainbow10', '#FFFFFF', None)) 25 | 26 | 27 | def remove_blanks_from_xml(xml_str): 28 | lines = xml_str.split('\n') 29 | lines = [l.strip() for l in lines] 30 | lines = [l for l in lines if len(l)] 31 | return ''.join(lines) 32 | 33 | 34 | def parse_xml_str(xml_str): 35 | xml_str = remove_blanks_from_xml(xml_str) 36 | return ET.fromstring(xml_str) 37 | 38 | 39 | def pretty_print(root): 40 | xml_str = ET.tostring(root) 41 | return xml.dom.minidom.parseString(xml_str).toprettyxml() 42 | 43 | 44 | def add_plist_entry(root, key, value): 45 | cur = ET.Element('key') 46 | cur.text = key 47 | root.append(cur) 48 | cur = ET.Element('string') 49 | cur.text = value 50 | root.append(cur) 51 | 52 | 53 | def make_color_entry(scope, color, font): 54 | result = ET.Element('dict') 55 | add_plist_entry(result, 'name', 'rainbow csv ' + scope) 56 | add_plist_entry(result, 'scope', scope) 57 | 58 | cur = ET.Element('key') 59 | cur.text = 'settings' 60 | result.append(cur) 61 | settings = ET.Element('dict') 62 | 63 | if font != None: 64 | add_plist_entry(settings, 'fontStyle', font) 65 | add_plist_entry(settings, 'foreground', color) 66 | result.append(settings) 67 | return result 68 | 69 | 70 | def convert_colorscheme(root): 71 | root = copy.deepcopy(root) 72 | array_elem = root.findall('./dict/array') 73 | if len(array_elem) != 1: 74 | return None 75 | array_elem = array_elem[0] 76 | for entry in color_entries: 77 | array_elem.append(make_color_entry(*entry)) 78 | return root 79 | 80 | 81 | 82 | def main(): 83 | parser = argparse.ArgumentParser() 84 | #parser.add_argument('--verbose', action='store_true', help='Run in verbose mode') 85 | #parser.add_argument('--num_iter', type=int, help='number of iterations option') 86 | parser.add_argument('src_path', help='example of positional argument') 87 | args = parser.parse_args() 88 | 89 | #num_iter = args.num_iter 90 | src_path = args.src_path 91 | root = parse_xml_str(open(src_path).read()) 92 | converted = convert_colorscheme(root) 93 | #print converted 94 | #print ET.tostring(converted) 95 | print pretty_print(converted) 96 | 97 | #for line in sys.stdin: 98 | # line = line.rstrip('\n') 99 | # fields = line.split('\t') 100 | 101 | if __name__ == '__main__': 102 | main() 103 | -------------------------------------------------------------------------------- /other/make_sublime_syntax.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import os 5 | import argparse 6 | import random 7 | import re 8 | 9 | parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 10 | sys.path.insert(0, parent_dir) 11 | 12 | import auto_syntax 13 | 14 | 15 | def name_normalize(delim): 16 | if delim == '<': 17 | return 'less-than' 18 | if delim == '>': 19 | return 'greater-than' 20 | if delim == ':': 21 | return 'colon' 22 | if delim == '"': 23 | return 'double-quote' 24 | if delim == '/': 25 | return 'slash' 26 | if delim == '\\': 27 | return 'backslash' 28 | if delim == '|': 29 | return 'pipe' 30 | if delim == '?': 31 | return 'question-mark' 32 | if delim == '*': 33 | return 'asterisk' 34 | if delim == '\t': 35 | return 'tab' 36 | if delim == ' ': 37 | return 'space' 38 | return '[{}]'.format(delim) 39 | 40 | 41 | def get_syntax_file_name_old(delim, policy): 42 | policy = auto_syntax.filename_policy_map[policy] 43 | if delim == '\t' and policy == 'Simple': 44 | return 'TSV (Rainbow)' 45 | if delim == ',' and policy == 'Standard': 46 | return 'CSV (Rainbow)' 47 | return 'Rainbow CSV {} {}'.format(name_normalize(delim), policy) 48 | 49 | 50 | def write_sublime_syntax(delim, policy, dst_dir, old_names): 51 | # TODO get rid of this 52 | if old_names: 53 | syntax_file_name = get_syntax_file_name_old(delim, policy) + '.sublime-syntax' 54 | else: 55 | syntax_file_name = auto_syntax.get_syntax_file_basename(delim, policy) 56 | syntax_path = os.path.join(dst_dir, syntax_file_name) 57 | syntax_text = auto_syntax.make_sublime_syntax(delim, policy) 58 | with open(syntax_path, 'w') as dst: 59 | dst.write(syntax_text) 60 | 61 | 62 | def main(): 63 | parser = argparse.ArgumentParser() 64 | parser.add_argument('--make_grammars_prod', help='make and put grammars into DIR') 65 | parser.add_argument('--make_grammars_old', help='make and put grammars into DIR') 66 | parser.add_argument('--dbg_delim', help='Run in debug mode: print single grammar with delim') 67 | parser.add_argument('--dbg_policy', help='Run in debug mode: print single grammar with policy') 68 | args = parser.parse_args() 69 | 70 | if args.make_grammars_old: 71 | dst_dir = args.make_grammars_old 72 | delims = auto_syntax.get_pregenerated_delims() 73 | standard_delims = '\t|,;' 74 | for delim in delims: 75 | if standard_delims.find(delim) != -1: 76 | write_sublime_syntax(delim, 'quoted', dst_dir, old_names=True) 77 | write_sublime_syntax(delim, 'simple', dst_dir, old_names=True) 78 | return 79 | 80 | if args.make_grammars_prod: 81 | dst_dir = args.make_grammars_prod 82 | delims = auto_syntax.get_pregenerated_delims() 83 | standard_delims = ',;' 84 | for delim in delims: 85 | if standard_delims.find(delim) != -1: 86 | write_sublime_syntax(delim, 'quoted', dst_dir, old_names=False) 87 | write_sublime_syntax(delim, 'quoted_rfc', dst_dir, old_names=False) 88 | write_sublime_syntax(delim, 'simple', dst_dir, old_names=False) 89 | return 90 | 91 | delim = args.dbg_delim 92 | policy = args.dbg_policy 93 | 94 | grammar = auto_syntax.make_sublime_syntax(delim, policy) 95 | print(grammar) 96 | 97 | 98 | 99 | if __name__ == '__main__': 100 | main() 101 | -------------------------------------------------------------------------------- /RainbowCSV.sublime-settings: -------------------------------------------------------------------------------- 1 | { 2 | // Disable Rainbow CSV for files bigger than the specified size. 3 | // This can be helpful to prevent poor performance and crashes with very large files. 4 | // Manual separator selection will override this setting for the current file. 5 | "rainbow_csv_max_file_size_bytes": 5000000, 6 | 7 | 8 | // Enable content-based separator autodetection. 9 | // Files with ".csv" and ".tsv" extensions are always highlighted no matter what is the value of this option. 10 | "enable_rainbow_csv_autodetect": true, 11 | 12 | // Enable the hover popup that shows additional Rainbow CSV context. 13 | "show_rainbow_hover": true, 14 | 15 | // List of CSV dialects to autodetect. 16 | // "simple" - separators CAN NOT be escaped in a double quoted field, double quotes are ignored 17 | // "quoted" - separators CAN be escaped in a double quoted field. 18 | // Separators can consist of more than one character, e.g. ": " or "~#~" 19 | // If "enable_rainbow_csv_autodetect" is set to false this setting is ignored 20 | "rainbow_csv_autodetect_dialects": [["\t", "simple"], [",", "quoted"], [";", "quoted"], ["|", "simple"]], 21 | 22 | // Allow quoted multiline fields as defined in RFC-4180: https://tools.ietf.org/html/rfc4180 23 | "allow_newlines_in_fields": false, 24 | 25 | // Use zero-based column indices in hover column info. 26 | // This doesn't affect column naming in RBQL queries. 27 | "use_zero_based_column_indices": false, 28 | 29 | // Use custom high-contrast rainbow colors instead of colors provided by your current color scheme. 30 | // When you enable this option, "auto_adjust_rainbow_colors" also gets enabled by default. 31 | "use_custom_rainbow_colors": false, 32 | 33 | 34 | // Auto adjust rainbow colors for Packages/User/RainbowCSV.sublime-color-scheme 35 | // Rainbow CSV will auto-generate color theme with high-contrast colors to make CSV columns more distinguishable. 36 | // You can disable this setting and manually customize Rainbow CSV color scheme at `Packages/User/RainbowCSV.sublime-color-scheme`, you can use the following [RainbowCSV.sublime-color-scheme](https://github.com/mechatroner/sublime_rainbow_csv/blob/master/RainbowCSV.sublime-color-scheme) file as a starting point for your customizations. 37 | // Do NOT manually customize Packages/User/RainbowCSV.sublime-color-scheme without disabling this setting, the plugin will just rewrite it in that case. 38 | // This option has effect only if "use_custom_rainbow_colors" is set to true 39 | "auto_adjust_rainbow_colors": true, 40 | 41 | 42 | // RBQL backend language. 43 | // Supported values: "Python", "JS" 44 | // In order to use RBQL with JavaScript (JS) you need to have Node JS installed and added to your system path. 45 | "rbql_backend_language": "Python", 46 | 47 | // RBQL will treat first records in all input and join files as headers. 48 | // You can set this value to true if most of the CSV files you deal with have headers. 49 | // You can override this setting on the query level by either adding `WITH (header)` or `WITH (noheader)` to the end of the query. 50 | "rbql_with_headers": false, 51 | 52 | // RBQL encoding for files and queries. 53 | // Supported values: "latin-1", "utf-8" 54 | "rbql_encoding": "utf-8", 55 | 56 | 57 | // Format of RBQL result set tables. 58 | // Supported values: "input", "tsv", "csv" 59 | // * input: same format as the input table 60 | // * tsv: tab separated values. 61 | // * csv: is Excel-compatible and allows quoted commas. 62 | "rbql_output_format": "input", 63 | 64 | 65 | // Enable logging to debug the extension. To view the output you can either 66 | // 1. Click "View" -> "Show Console" 67 | // 2. Click "Preferences" -> "Browse Packages...", And then open "User/rainbow_csv_debug.log" 68 | "enable_debug_logging": false, 69 | } 70 | -------------------------------------------------------------------------------- /rbql/rbql_pandas.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | from __future__ import print_function 4 | 5 | from . import rbql_engine 6 | 7 | 8 | def get_dataframe_column_names_for_rbql(dataframe): 9 | import pandas 10 | if isinstance(dataframe.columns, pandas.RangeIndex) or not len(dataframe.columns): 11 | return None 12 | return [str(v) for v in list(dataframe.columns)] 13 | 14 | 15 | class DataframeIterator(rbql_engine.RBQLInputIterator): 16 | def __init__(self, table, normalize_column_names=True, variable_prefix='a'): 17 | self.table = table 18 | self.normalize_column_names = normalize_column_names 19 | self.variable_prefix = variable_prefix 20 | self.NR = 0 21 | # TODO include `Index` into the list of addressable variable names. 22 | self.column_names = get_dataframe_column_names_for_rbql(table) 23 | self.table_itertuples = self.table.itertuples(index=False) 24 | 25 | def get_variables_map(self, query_text): 26 | variable_map = dict() 27 | rbql_engine.parse_basic_variables(query_text, self.variable_prefix, variable_map) 28 | rbql_engine.parse_array_variables(query_text, self.variable_prefix, variable_map) 29 | if self.column_names is not None: 30 | if self.normalize_column_names: 31 | rbql_engine.parse_dictionary_variables(query_text, self.variable_prefix, self.column_names, variable_map) 32 | rbql_engine.parse_attribute_variables(query_text, self.variable_prefix, self.column_names, 'column names list', variable_map) 33 | else: 34 | rbql_engine.map_variables_directly(query_text, self.column_names, variable_map) 35 | return variable_map 36 | 37 | def get_record(self): 38 | try: 39 | record = next(self.table_itertuples) 40 | except StopIteration: 41 | return None 42 | self.NR += 1 43 | # Convert to list because `record` has `Pandas` type. 44 | return list(record) 45 | 46 | def get_warnings(self): 47 | return [] 48 | 49 | def get_header(self): 50 | return self.column_names 51 | 52 | 53 | class DataframeWriter(rbql_engine.RBQLOutputWriter): 54 | def __init__(self): 55 | self.header = None 56 | self.output_rows = [] 57 | self.result = None 58 | 59 | def write(self, fields): 60 | self.output_rows.append(fields) 61 | return True 62 | 63 | def set_header(self, header): 64 | self.header = header 65 | 66 | def finish(self): 67 | import pandas as pd 68 | self.result = pd.DataFrame(self.output_rows, columns=self.header) 69 | 70 | 71 | class SingleDataframeRegistry(rbql_engine.RBQLTableRegistry): 72 | def __init__(self, table, table_name, normalize_column_names=True): 73 | self.table = table 74 | self.normalize_column_names = normalize_column_names 75 | self.table_name = table_name 76 | 77 | def get_iterator_by_table_id(self, table_id, single_char_alias): 78 | if table_id.lower() != self.table_name: 79 | raise rbql_engine.RbqlParsingError('Unable to find join table: "{}"'.format(table_id)) 80 | return DataframeIterator(self.table, self.normalize_column_names, single_char_alias) 81 | 82 | 83 | def query_dataframe(query_text, input_dataframe, output_warnings=None, join_dataframe=None, normalize_column_names=True, user_init_code=''): 84 | if output_warnings is None: 85 | # Ignore output warnings if the output_warnings container hasn't been provided. 86 | output_warnings = [] 87 | if not normalize_column_names and join_dataframe is not None: 88 | input_columns = get_dataframe_column_names_for_rbql(input_dataframe) 89 | join_columns = get_dataframe_column_names_for_rbql(join_dataframe) 90 | if input_columns is not None and join_columns is not None: 91 | rbql_engine.ensure_no_ambiguous_variables(query_text, input_columns, join_columns) 92 | input_iterator = DataframeIterator(input_dataframe, normalize_column_names) 93 | output_writer = DataframeWriter() 94 | join_tables_registry = None if join_dataframe is None else SingleDataframeRegistry(join_dataframe, 'b', normalize_column_names) 95 | rbql_engine.query(query_text, input_iterator, output_writer, output_warnings, join_tables_registry, user_init_code=user_init_code) 96 | return output_writer.result 97 | -------------------------------------------------------------------------------- /rbql/csv_utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | from __future__ import print_function 3 | import re 4 | 5 | 6 | newline_rgx = re.compile('(?:\r\n)|\r|\n') 7 | 8 | field_regular_expression = '"((?:[^"]*"")*[^"]*)"' 9 | field_rgx = re.compile(field_regular_expression) 10 | field_rgx_external_whitespaces = re.compile(' *' + field_regular_expression + ' *') 11 | 12 | 13 | def extract_next_field(src, dlm, preserve_quotes_and_whitespaces, allow_external_whitespaces, cidx, result): 14 | warning = False 15 | rgx = field_rgx_external_whitespaces if allow_external_whitespaces else field_rgx 16 | match_obj = rgx.match(src, cidx) 17 | if match_obj is not None: 18 | match_end = match_obj.span()[1] 19 | if match_end == len(src) or src[match_end] == dlm: 20 | if preserve_quotes_and_whitespaces: 21 | result.append(match_obj.group(0)) 22 | else: 23 | result.append(match_obj.group(1).replace('""', '"')) 24 | return (match_end + 1, False) 25 | warning = True 26 | uidx = src.find(dlm, cidx) 27 | if uidx == -1: 28 | uidx = len(src) 29 | field = src[cidx:uidx] 30 | warning = warning or field.find('"') != -1 31 | result.append(field) 32 | return (uidx + 1, warning) 33 | 34 | 35 | 36 | def split_quoted_str(src, dlm, preserve_quotes_and_whitespaces=False): 37 | # This function is newline-agnostic i.e. it can also split records with multiline fields. 38 | assert dlm != '"' 39 | if src.find('"') == -1: # Optimization for most common case 40 | return (src.split(dlm), False) 41 | result = list() 42 | cidx = 0 43 | warning = False 44 | allow_external_whitespaces = dlm != ' ' 45 | while cidx < len(src): 46 | extraction_report = extract_next_field(src, dlm, preserve_quotes_and_whitespaces, allow_external_whitespaces, cidx, result) 47 | cidx = extraction_report[0] 48 | warning = warning or extraction_report[1] 49 | 50 | if src[-1] == dlm: 51 | result.append('') 52 | return (result, warning) 53 | 54 | 55 | def split_whitespace_separated_str(src, preserve_whitespaces=False): 56 | rgxp = re.compile(" *[^ ]+ *") if preserve_whitespaces else re.compile("[^ ]+") 57 | result = [] 58 | for m in rgxp.finditer(src): 59 | result.append(m.group()) 60 | if preserve_whitespaces and len(result) > 1: 61 | for i in range(len(result) - 1): 62 | result[i] = result[i][:-1] 63 | return result 64 | 65 | 66 | def smart_split(src, dlm, policy, preserve_quotes_and_whitespaces): 67 | if policy == 'simple': 68 | return (src.split(dlm), False) 69 | if policy == 'whitespace': 70 | return (split_whitespace_separated_str(src, preserve_quotes_and_whitespaces), False) 71 | if policy == 'monocolumn': 72 | return ([src], False) 73 | return split_quoted_str(src, dlm, preserve_quotes_and_whitespaces) 74 | 75 | 76 | def extract_line_from_data(data): 77 | mobj = newline_rgx.search(data) 78 | if mobj is None: 79 | return (None, None, data) 80 | pos_start, pos_end = mobj.span() 81 | str_before = data[:pos_start] 82 | str_after = data[pos_end:] 83 | return (str_before, mobj.group(0), str_after) 84 | 85 | 86 | def quote_field(src, delim): 87 | if src.find('"') != -1: 88 | return '"{}"'.format(src.replace('"', '""')) 89 | if src.find(delim) != -1: 90 | return '"{}"'.format(src) 91 | return src 92 | 93 | 94 | def rfc_quote_field(src, delim): 95 | # A single regexp can be used to find all 4 characters simultaneously, but this approach doesn't significantly improve performance according to my tests. 96 | if src.find('"') != -1: 97 | return '"{}"'.format(src.replace('"', '""')) 98 | if src.find(delim) != -1 or src.find('\n') != -1 or src.find('\r') != -1: 99 | return '"{}"'.format(src) 100 | return src 101 | 102 | 103 | def unquote_field(field): 104 | field_rgx_external_whitespaces_full = re.compile('^ *'+ field_regular_expression + ' *$') 105 | match_obj = field_rgx_external_whitespaces_full.match(field) 106 | if match_obj is not None: 107 | return match_obj.group(1).replace('""', '"') 108 | return field 109 | 110 | 111 | def unquote_fields(fields): 112 | return [unquote_field(f) for f in fields] 113 | 114 | 115 | -------------------------------------------------------------------------------- /sublime_rbql.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import codecs 4 | import traceback 5 | import subprocess 6 | import tempfile 7 | import time 8 | 9 | import rainbow_csv.rbql as rbql 10 | 11 | 12 | def system_has_node_js(): 13 | exit_code = 0 14 | out_data = '' 15 | try: 16 | cmd = ['node', '--version'] 17 | pobj = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 18 | out_data, err_data = pobj.communicate() 19 | exit_code = pobj.returncode 20 | except OSError as e: 21 | if e.errno == 2: 22 | return False 23 | raise 24 | return exit_code == 0 and len(out_data) and len(err_data) == 0 25 | 26 | 27 | def execute_python(src_table_path, encoding, query, input_delim, input_policy, out_delim, out_policy, dst_table_path, with_headers): 28 | try: 29 | warnings = [] 30 | rbql.query_csv(query, src_table_path, input_delim, input_policy, dst_table_path, out_delim, out_policy, encoding, warnings, with_headers) 31 | return (None, None, warnings) 32 | except Exception as e: 33 | error_type, error_msg = rbql.exception_to_error_info(e) 34 | return (error_type, error_msg, []) 35 | 36 | 37 | def execute_js(src_table_path, encoding, query, input_delim, input_policy, out_delim, out_policy, dst_table_path, with_headers): 38 | import json 39 | if not system_has_node_js(): 40 | return ('Execution Error', 'Node.js is not found in your OS, test command: "node --version"', []) 41 | script_dir = os.path.dirname(os.path.realpath(__file__)) 42 | rbql_js_script_path = os.path.join(script_dir, 'rbql-js', 'cli_rbql.js') 43 | cmd = ['node', rbql_js_script_path, '--query', query, '--delim', input_delim, '--policy', input_policy, '--out-delim', out_delim, '--out-policy', out_policy, '--input', src_table_path, '--output', dst_table_path, '--encoding', encoding, '--error-format', 'json'] 44 | if with_headers: 45 | cmd.append('--with-headers') 46 | if os.name == 'nt': 47 | # Without the startupinfo magic Windows will show console window for a longer period. 48 | si = subprocess.STARTUPINFO() 49 | si.dwFlags |= subprocess.STARTF_USESHOWWINDOW 50 | pobj = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, startupinfo=si) 51 | else: 52 | pobj = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 53 | out_data, err_data = pobj.communicate() 54 | error_code = pobj.returncode 55 | error_type = None 56 | error_msg = None 57 | warnings = [] 58 | err_data = err_data.strip() 59 | if len(err_data): 60 | try: 61 | err_data = err_data.decode('utf-8') 62 | json_err = json.loads(err_data) 63 | error_type = json_err.get('error_type', None) 64 | error_msg = json_err.get('error', None) 65 | warnings = json_err.get('warnings', []) 66 | except Exception as e: 67 | error_type = 'Unexpected Error' 68 | error_msg = 'Unable to parse rbql-js error: {}, report: "{}"'.format(e, err_data) 69 | if error_type is None and error_code: 70 | error_type = 'Unexpected Error' 71 | error_msg = 'rbq-js failed with exit code: {}'.format(error_code) 72 | return (error_type, error_msg, warnings) 73 | 74 | 75 | def converged_execute(meta_language, src_table_path, query, input_delim, input_policy, out_delim, out_policy, encoding, with_headers): 76 | try: 77 | tmp_dir = tempfile.gettempdir() 78 | table_name = os.path.basename(src_table_path) 79 | delim_ext_map = {'\t': '.tsv', ',': '.csv'} 80 | dst_extension = delim_ext_map[out_delim] if out_delim in delim_ext_map else '.txt' 81 | dst_table_name = table_name + dst_extension 82 | dst_table_path = os.path.join(tmp_dir, dst_table_name) 83 | assert meta_language in ['python', 'js'], 'Meta language must be "python" or "js"' 84 | if meta_language == 'python': 85 | exec_result = execute_python(src_table_path, encoding, query, input_delim, input_policy, out_delim, out_policy, dst_table_path, with_headers) 86 | else: 87 | exec_result = execute_js(src_table_path, encoding, query, input_delim, input_policy, out_delim, out_policy, dst_table_path, with_headers) 88 | error_type, error_details, warnings = exec_result 89 | if error_type is not None: 90 | dst_table_path = None 91 | return (error_type, error_details, warnings, dst_table_path) 92 | except Exception as e: 93 | return ('Execution Error', str(e), [], None) 94 | 95 | 96 | -------------------------------------------------------------------------------- /rbql/rbql_sqlite.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # This module allows to query sqlite databases using RBQL 4 | 5 | from __future__ import unicode_literals 6 | from __future__ import print_function 7 | 8 | 9 | # TODO consider to support table names in "FROM" section of the query, making table_name param of SqliteRecordIterator optional 10 | # TODO consider adding support for multiple variable_prefixes i.e. "a" and or "b" and to alias input and join tables 11 | 12 | 13 | import re 14 | import os 15 | import sys 16 | 17 | from . import rbql_engine 18 | from . import rbql_csv 19 | 20 | 21 | class SqliteRecordIterator(rbql_engine.RBQLInputIterator): 22 | def __init__(self, db_connection, table_name, variable_prefix='a'): 23 | self.db_connection = db_connection 24 | self.table_name = table_name 25 | self.variable_prefix = variable_prefix 26 | self.cursor = self.db_connection.cursor() 27 | import sqlite3 28 | if re.match('^[a-zA-Z0-9_]*$', table_name) is None: 29 | raise rbql_engine.RbqlIOHandlingError('Unable to use "{}": input table name can contain only alphanumeric characters and underscore'.format(table_name)) 30 | try: 31 | self.cursor.execute('SELECT * FROM {};'.format(table_name)) 32 | except sqlite3.OperationalError as e: 33 | if str(e).find('no such table') != -1: 34 | raise rbql_engine.RbqlIOHandlingError('no such table "{}"'.format(table_name)) 35 | raise 36 | 37 | def get_header(self): 38 | column_names = [description[0] for description in self.cursor.description] 39 | return column_names 40 | 41 | def get_variables_map(self, query_text): 42 | variable_map = dict() 43 | rbql_engine.parse_basic_variables(query_text, self.variable_prefix, variable_map) 44 | rbql_engine.parse_array_variables(query_text, self.variable_prefix, variable_map) 45 | rbql_engine.parse_dictionary_variables(query_text, self.variable_prefix, self.get_header(), variable_map) 46 | rbql_engine.parse_attribute_variables(query_text, self.variable_prefix, self.get_header(), 'table column names', variable_map) 47 | return variable_map 48 | 49 | def get_record(self): 50 | record_tuple = self.cursor.fetchone() 51 | if record_tuple is None: 52 | return None 53 | # We need to convert tuple to list here because otherwise we won't be able to concatinate lists in expressions with star `*` operator 54 | return list(record_tuple) 55 | 56 | def get_all_records(self, num_rows=None): 57 | # TODO consider to use TOP in the sqlite query when num_rows is not None 58 | if num_rows is None: 59 | return self.cursor.fetchall() 60 | result = [] 61 | for i in range(num_rows): 62 | row = self.cursor.fetchone() 63 | if row is None: 64 | break 65 | result.append(row) 66 | return result 67 | 68 | def get_warnings(self): 69 | return [] 70 | 71 | 72 | class SqliteDbRegistry(rbql_engine.RBQLTableRegistry): 73 | def __init__(self, db_connection): 74 | self.db_connection = db_connection 75 | 76 | def get_iterator_by_table_id(self, table_id, single_char_alias): 77 | self.record_iterator = SqliteRecordIterator(self.db_connection, table_id, single_char_alias) 78 | return self.record_iterator 79 | 80 | 81 | def query_sqlite_to_csv(query_text, db_connection, input_table_name, output_path, output_delim, output_policy, output_csv_encoding, output_warnings, user_init_code='', colorize_output=False): 82 | output_stream, close_output_on_finish = (None, False) 83 | join_tables_registry = None 84 | try: 85 | output_stream, close_output_on_finish = (sys.stdout, False) if output_path is None else (open(output_path, 'wb'), True) 86 | 87 | if not rbql_csv.is_ascii(query_text) and output_csv_encoding == 'latin-1': 88 | raise rbql_engine.RbqlIOHandlingError('To use non-ascii characters in query enable UTF-8 encoding instead of latin-1/binary') 89 | 90 | if not rbql_csv.is_ascii(output_delim) and output_csv_encoding == 'latin-1': 91 | raise rbql_engine.RbqlIOHandlingError('To use non-ascii separators enable UTF-8 encoding instead of latin-1/binary') 92 | 93 | default_init_source_path = os.path.join(os.path.expanduser('~'), '.rbql_init_source.py') 94 | if user_init_code == '' and os.path.exists(default_init_source_path): 95 | user_init_code = rbql_csv.read_user_init_code(default_init_source_path) 96 | 97 | join_tables_registry = SqliteDbRegistry(db_connection) 98 | input_iterator = SqliteRecordIterator(db_connection, input_table_name) 99 | output_writer = rbql_csv.CSVWriter(output_stream, close_output_on_finish, output_csv_encoding, output_delim, output_policy, colorize_output=colorize_output) 100 | rbql_engine.query(query_text, input_iterator, output_writer, output_warnings, join_tables_registry, user_init_code) 101 | finally: 102 | if close_output_on_finish: 103 | output_stream.close() 104 | 105 | 106 | -------------------------------------------------------------------------------- /rbql-js/csv_utils.js: -------------------------------------------------------------------------------- 1 | let field_regular_expression = '"((?:[^"]*"")*[^"]*)"'; 2 | let field_rgx = new RegExp('^' + field_regular_expression); 3 | let field_rgx_external_whitespaces = new RegExp('^ *' + field_regular_expression + ' *'); 4 | 5 | 6 | // TODO consider making this file (and rbql.js) both node and browser compatible: https://caolan.org/posts/writing_for_node_and_the_browser.html 7 | 8 | 9 | function split_lines(text) { 10 | return text.split(/\r\n|\r|\n/); 11 | } 12 | 13 | 14 | function extract_next_field(src, dlm, preserve_quotes_and_whitespaces, allow_external_whitespaces, cidx, result) { 15 | var warning = false; 16 | let src_cur = src.substring(cidx); 17 | let rgx = allow_external_whitespaces ? field_rgx_external_whitespaces : field_rgx; 18 | let match_obj = rgx.exec(src_cur); 19 | if (match_obj !== null) { 20 | let match_end = match_obj[0].length; 21 | if (cidx + match_end == src.length || src[cidx + match_end] == dlm) { 22 | if (preserve_quotes_and_whitespaces) { 23 | result.push(match_obj[0]); 24 | } else { 25 | result.push(match_obj[1].replace(/""/g, '"')); 26 | } 27 | return [cidx + match_end + 1, false]; 28 | } 29 | warning = true; 30 | } 31 | var uidx = src.indexOf(dlm, cidx); 32 | if (uidx == -1) 33 | uidx = src.length; 34 | var field = src.substring(cidx, uidx); 35 | warning = warning || field.indexOf('"') != -1; 36 | result.push(field); 37 | return [uidx + 1, warning]; 38 | } 39 | 40 | 41 | function split_quoted_str(src, dlm, preserve_quotes_and_whitespaces=false) { 42 | // This function is newline-agnostic i.e. it can also split records with multiline fields. 43 | if (src.indexOf('"') == -1) // Optimization for most common case 44 | return [src.split(dlm), false]; 45 | var result = []; 46 | var cidx = 0; 47 | var warning = false; 48 | let allow_external_whitespaces = dlm != ' '; 49 | while (cidx < src.length) { 50 | var extraction_report = extract_next_field(src, dlm, preserve_quotes_and_whitespaces, allow_external_whitespaces, cidx, result); 51 | cidx = extraction_report[0]; 52 | warning = warning || extraction_report[1]; 53 | } 54 | if (src.charAt(src.length - 1) == dlm) 55 | result.push(''); 56 | return [result, warning]; 57 | } 58 | 59 | 60 | function quote_field(src, delim) { 61 | if (src.indexOf(delim) != -1 || src.indexOf('"') != -1) { 62 | var escaped = src.replace(/"/g, '""'); 63 | return `"${escaped}"`; 64 | } 65 | return src; 66 | } 67 | 68 | 69 | function rfc_quote_field(src, delim) { 70 | if (src.indexOf(delim) != -1 || src.indexOf('"') != -1 || src.indexOf('\n') != -1 || src.indexOf('\r') != -1) { 71 | var escaped = src.replace(/"/g, '""'); 72 | return `"${escaped}"`; 73 | } 74 | return src; 75 | } 76 | 77 | 78 | function unquote_field(field) { 79 | let rgx = new RegExp('^' + ' *' + field_regular_expression + ' *$'); 80 | let match_obj = rgx.exec(field); 81 | if (match_obj !== null) { 82 | return match_obj[1].replace(/""/g, '"'); 83 | } 84 | return field; 85 | } 86 | 87 | 88 | function unquote_fields(fields) { 89 | return fields.map(unquote_field); 90 | } 91 | 92 | 93 | function split_whitespace_separated_str(src, preserve_whitespaces=false) { 94 | var rgxp = preserve_whitespaces ? new RegExp(' *[^ ]+ *', 'g') : new RegExp('[^ ]+', 'g'); 95 | let result = []; 96 | let match_obj = null; 97 | while((match_obj = rgxp.exec(src)) !== null) { 98 | result.push(match_obj[0]); 99 | } 100 | if (preserve_whitespaces) { 101 | for (let i = 0; i < result.length - 1; i++) { 102 | result[i] = result[i].slice(0, -1); 103 | } 104 | } 105 | return result; 106 | } 107 | 108 | 109 | function smart_split(src, dlm, policy, preserve_quotes_and_whitespaces) { 110 | if (policy === 'simple') 111 | return [src.split(dlm), false]; 112 | if (policy === 'whitespace') 113 | return [split_whitespace_separated_str(src, preserve_quotes_and_whitespaces), false]; 114 | if (policy === 'monocolumn') 115 | return [[src], false]; 116 | return split_quoted_str(src, dlm, preserve_quotes_and_whitespaces); 117 | } 118 | 119 | 120 | class MultilineRecordAggregator { 121 | constructor(comment_prefix) { 122 | this.comment_prefix = comment_prefix; 123 | this.reset(); 124 | } 125 | add_line(line_text) { 126 | if (this.has_full_record || this.has_comment_line) { 127 | throw new Error('Invalid usage - record aggregator must be reset before adding new lines'); 128 | } 129 | if (this.comment_prefix && this.rfc_line_buffer.length == 0 && line_text.startsWith(this.comment_prefix)) { 130 | this.has_comment_line = true; 131 | return false; 132 | } 133 | let match_list = line_text.match(/"/g); 134 | let has_unbalanced_double_quote = match_list && match_list.length % 2 == 1; 135 | this.rfc_line_buffer.push(line_text); 136 | this.has_full_record = (!has_unbalanced_double_quote && this.rfc_line_buffer.length == 1) || (has_unbalanced_double_quote && this.rfc_line_buffer.length > 1); 137 | return this.has_full_record; 138 | } 139 | is_inside_multiline_record() { 140 | return this.rfc_line_buffer.length && !this.has_full_record; 141 | } 142 | get_full_line(line_separator) { 143 | return this.rfc_line_buffer.join(line_separator); 144 | } 145 | get_num_lines_in_record() { 146 | return this.rfc_line_buffer.length; 147 | } 148 | reset() { 149 | this.rfc_line_buffer = []; 150 | this.has_full_record = false; 151 | this.has_comment_line = false; 152 | } 153 | } 154 | 155 | 156 | module.exports.split_quoted_str = split_quoted_str; 157 | module.exports.split_whitespace_separated_str = split_whitespace_separated_str; 158 | module.exports.smart_split = smart_split; 159 | module.exports.quote_field = quote_field; 160 | module.exports.rfc_quote_field = rfc_quote_field; 161 | module.exports.unquote_field = unquote_field; 162 | module.exports.unquote_fields = unquote_fields; 163 | module.exports.split_lines = split_lines; 164 | module.exports.MultilineRecordAggregator = MultilineRecordAggregator; 165 | -------------------------------------------------------------------------------- /rbql/rbql_ipython.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | from __future__ import print_function 4 | 5 | from . import rbql_engine 6 | from . import rbql_pandas 7 | 8 | # TODO figure out how to implement at least basic autocomplete for the magic command. 9 | 10 | import re 11 | from_autocomplete_matcher = re.compile(r'(?:^| )from +([_a-zA-Z0-9]+)(?:$| )', flags=re.IGNORECASE) 12 | join_autocomplete_matcher = re.compile(r'(?:^| )join +([_a-zA-Z0-9]+)(?:$| )', flags=re.IGNORECASE) 13 | 14 | 15 | class IPythonDataframeRegistry(rbql_engine.RBQLTableRegistry): 16 | # TODO consider making this class nested under load_ipython_extension to avoid redundant `import pandas`. 17 | def __init__(self, all_ns_refs): 18 | self.all_ns_refs = all_ns_refs 19 | 20 | def get_iterator_by_table_id(self, table_id, single_char_alias): 21 | import pandas 22 | # It seems to be the first namespace is "user" namespace, at least according to this code: 23 | # https://github.com/google/picatrix/blob/a2f39766ad4b007b125dc8f84916e18fb3dc5478/picatrix/lib/utils.py 24 | for ns in self.all_ns_refs: 25 | if table_id in ns and isinstance(ns[table_id], pandas.DataFrame): 26 | return rbql_pandas.DataframeIterator(ns[table_id], normalize_column_names=True, variable_prefix=single_char_alias) 27 | return None 28 | 29 | 30 | def eprint(*args, **kwargs): 31 | import sys 32 | print(*args, file=sys.stderr, **kwargs) 33 | 34 | 35 | class AttrDict(dict): 36 | # Helper class to convert dict keys to attributes. See explanation here: https://stackoverflow.com/a/14620633/2898283 37 | def __init__(self, *args, **kwargs): 38 | super(AttrDict, self).__init__(*args, **kwargs) 39 | self.__dict__ = self 40 | 41 | 42 | def load_ipython_extension(ipython): 43 | from IPython.core.magic import register_line_magic 44 | from IPython.core.getipython import get_ipython 45 | import pandas 46 | 47 | ipython = ipython or get_ipython() # The pattern taken from here: https://github.com/pydoit/doit/blob/9efe141a5dc96d4912143561695af7fc4a076490/doit/tools.py 48 | # ipython is interactiveshell. Docs: https://ipython.readthedocs.io/en/stable/api/generated/IPython.core.interactiveshell.html 49 | 50 | 51 | def get_table_column_names(table_id): 52 | user_namespace = ipython.all_ns_refs[0] if len(ipython.all_ns_refs) else dict() 53 | if table_id not in user_namespace or not isinstance(user_namespace[table_id], pandas.DataFrame): 54 | return [] 55 | input_df = user_namespace[table_id] 56 | if isinstance(input_df.columns, pandas.RangeIndex) or not len(input_df.columns): 57 | return [] 58 | return [str(v) for v in list(input_df.columns)] 59 | 60 | 61 | def rbql_completers(self, event): 62 | # This should return a list of strings with possible completions. 63 | # Note that all the included strings that don't start with event.symbol 64 | # are removed, in order to not confuse readline. 65 | 66 | # eg Typing %%rbql foo then hitting tab would yield an event like so: namespace(command='%%rbql', line='%%rbql foo', symbol='foo', text_until_cursor='%%rbql foo') 67 | # https://stackoverflow.com/questions/36479197/ipython-custom-tab-completion-for-user-magic-function 68 | # https://github.com/ipython/ipython/issues/11878 69 | 70 | simple_sql_keys_lower_case = ['update', 'select', 'where', 'limit', 'from', 'group by', 'order by'] 71 | simple_sql_keys_upper_case = [sk.upper() for sk in simple_sql_keys_lower_case] 72 | autocomplete_suggestions = simple_sql_keys_lower_case + simple_sql_keys_upper_case 73 | 74 | if event.symbol and event.symbol.startswith('a.'): 75 | from_match = from_autocomplete_matcher.search(event.line) 76 | if from_match is not None: 77 | table_id = from_match.group(1) 78 | table_column_names = get_table_column_names(table_id) 79 | autocomplete_suggestions += ['a.' + cn for cn in table_column_names] 80 | 81 | if event.symbol and event.symbol.startswith('b.'): 82 | from_match = join_autocomplete_matcher.search(event.line) 83 | if from_match is not None: 84 | table_id = from_match.group(1) 85 | table_column_names = get_table_column_names(table_id) 86 | autocomplete_suggestions += ['b.' + cn for cn in table_column_names] 87 | 88 | return autocomplete_suggestions 89 | 90 | ipython.set_hook('complete_command', rbql_completers, str_key='%rbql') 91 | 92 | 93 | # The difference between line and cell magic is described here: https://jakevdp.github.io/PythonDataScienceHandbook/01.03-magic-commands.html. 94 | # In short: line magic only accepts one line of input whereas cell magic supports multiline input as magic command argument. 95 | # Both line and cell magic would make sense for RBQL queries but for MVP it should be enough to implement just the cell magic. 96 | @register_line_magic("rbql") 97 | def run_rbql_query(query_text): 98 | # Unfortunately globals() and locals() called from here won't contain user variables defined in the notebook. 99 | 100 | tables_registry = IPythonDataframeRegistry(ipython.all_ns_refs) 101 | output_writer = rbql_pandas.DataframeWriter() 102 | # Ignore warnings because pandas dataframes can't cause them. 103 | output_warnings = [] 104 | # TODO make it possible to specify user_init_code in code cells. 105 | error_type, error_msg = None, None 106 | user_namespace = None 107 | if len(ipython.all_ns_refs) > 0: 108 | user_namespace = AttrDict(ipython.all_ns_refs[0]) 109 | try: 110 | rbql_engine.query(query_text, input_iterator=None, output_writer=output_writer, output_warnings=output_warnings, join_tables_registry=tables_registry, user_init_code='', user_namespace=user_namespace) 111 | except Exception as e: 112 | error_type, error_msg = rbql_engine.exception_to_error_info(e) 113 | if error_type is None: 114 | return output_writer.result 115 | else: 116 | # TODO use IPython.display to print error in red color, see https://stackoverflow.com/questions/16816013/is-it-possible-to-print-using-different-colors-in-ipythons-notebook 117 | eprint('Error [{}]: {}'.format(error_type, error_msg)) 118 | -------------------------------------------------------------------------------- /auto_syntax.py: -------------------------------------------------------------------------------- 1 | import binascii 2 | import re 3 | 4 | 5 | legacy_syntax_names = { 6 | ('\t', 'simple'): ('TSV (Rainbow)', 'tsv'), 7 | (',', 'quoted'): ('CSV (Rainbow)', 'csv'), 8 | } 9 | 10 | 11 | filename_policy_map = {'simple': 'Simple', 'quoted': 'Standard', 'quoted_rfc': 'quoted_rfc'} 12 | 13 | 14 | def encode_delim(delim): 15 | return binascii.hexlify(delim.encode('utf-8')).decode('ascii') 16 | 17 | 18 | def decode_delim(delim): 19 | return binascii.unhexlify(delim.encode('ascii')).decode('utf-8') 20 | 21 | 22 | def get_syntax_file_basename(delim, policy): 23 | for k, (v, _ext) in legacy_syntax_names.items(): 24 | if (delim, policy) == k: 25 | return v + '.sublime-syntax' 26 | return 'Rainbow_CSV_hex_{}_{}.sublime-syntax'.format(encode_delim(delim), filename_policy_map[policy]) 27 | 28 | 29 | def get_syntax_file_ext(delim, policy): 30 | for k, (_v, ext) in legacy_syntax_names.items(): 31 | if k == (delim, policy): 32 | return ext 33 | return None 34 | 35 | 36 | simple_header_template = '''%YAML 1.2 37 | --- 38 | name: '{}' 39 | file_extensions: [{}] 40 | scope: text.csv.{} 41 | 42 | 43 | contexts: 44 | main: 45 | - match: '^' 46 | push: rainbow1 47 | ''' 48 | 49 | 50 | standard_header_template = '''%YAML 1.2 51 | --- 52 | name: '{}' 53 | file_extensions: [{}] 54 | scope: text.csv.{} 55 | 56 | 57 | contexts: 58 | main: 59 | - match: '^' 60 | push: rainbow1 61 | 62 | quoted_field: 63 | - match: '""' 64 | scope: meta.rainbow.double-quote-escaped 65 | - match: '"' 66 | pop: true 67 | ''' 68 | 69 | non_rfc_endline_rule = ''' - match: '$'\n pop: true\n''' 70 | 71 | 72 | rainbow_scope_names = [ 73 | 'rainbow1', 74 | 'keyword.rainbow2', 75 | 'entity.name.rainbow3', 76 | 'comment.rainbow4', 77 | 'string.rainbow5', 78 | 'entity.name.tag.rainbow6', 79 | 'storage.type.rainbow7', 80 | 'support.rainbow8', 81 | 'constant.language.rainbow9', 82 | 'variable.language.rainbow10' 83 | ] 84 | 85 | 86 | def oniguruma_regular_escape_single_char(delim_char): 87 | single_escape_chars = r'\/|.$^*()[]+?' 88 | if single_escape_chars.find(delim_char) != -1: 89 | return r'\{}'.format(delim_char) 90 | if delim_char == '\t': 91 | return r'\t' 92 | return delim_char 93 | 94 | 95 | def oniguruma_regular_escape(delim): 96 | return ''.join([oniguruma_regular_escape_single_char(d) for d in delim]) 97 | 98 | 99 | def get_syntax_name(delim, policy): 100 | for k, (v, _ext) in legacy_syntax_names.items(): 101 | if (delim, policy) == k: 102 | return v 103 | ui_delim = delim.replace('\t', 'tab') 104 | 105 | hr_policy_map = {'simple': 'Simple', 'quoted': 'Standard', 'quoted_rfc': 'RFC'} 106 | return 'Rainbow CSV {} {}'.format(ui_delim, hr_policy_map[policy]) 107 | 108 | 109 | def yaml_escape(data): 110 | return data.replace("'", "''") 111 | 112 | 113 | def get_context_name(context_id): 114 | return "rainbow{}".format(context_id + 1) 115 | 116 | 117 | def make_simple_context(delim, context_id, num_contexts, indent=' '): 118 | result_lines = [] 119 | next_context_id = (context_id + 1) % num_contexts 120 | context_header = "{}:".format(get_context_name(context_id)) 121 | # We use `meta_content_scope` instead of `meta_scope` to prevent wrong separator color bug, see https://github.com/mechatroner/sublime_rainbow_csv/issues/31 122 | result_lines.append("- meta_content_scope: {}".format(rainbow_scope_names[context_id])) 123 | result_lines.append("- match: '{}'".format(yaml_escape(oniguruma_regular_escape(delim)))) 124 | result_lines.append(" set: {}".format(get_context_name(next_context_id))) 125 | result_lines.append("- match: '$'") 126 | result_lines.append(" pop: true") 127 | result_lines = [indent + v for v in result_lines] 128 | result_lines = [context_header] + result_lines 129 | result_lines = [indent + v for v in result_lines] 130 | return '\n'.join(result_lines) + '\n' 131 | 132 | 133 | def make_standard_context(delim, context_id, num_contexts, indent=' '): 134 | result_lines = [] 135 | next_context_id = (context_id + 1) % num_contexts 136 | context_header = "{}:".format(get_context_name(context_id)) 137 | # We use `meta_content_scope` instead of `meta_scope` to prevent wrong separator color bug, see https://github.com/mechatroner/sublime_rainbow_csv/issues/31 138 | result_lines.append("- meta_content_scope: {}".format(rainbow_scope_names[context_id])) 139 | result_lines.append("- match: '{}'".format(yaml_escape(oniguruma_regular_escape(delim)))) 140 | result_lines.append(" set: {}".format(get_context_name(next_context_id))) 141 | result_lines.append("- match: '$'") 142 | result_lines.append(" pop: true") 143 | result_lines.append("- match: '\"'") 144 | result_lines.append(" push: quoted_field") 145 | result_lines = [indent + v for v in result_lines] 146 | result_lines = [context_header] + result_lines 147 | result_lines = [indent + v for v in result_lines] 148 | return '\n'.join(result_lines) + '\n' 149 | 150 | 151 | def make_sublime_syntax_simple(delim): 152 | scope = 'rbcsmn' + ''.join([str(ord(d)) for d in delim]) 153 | name = get_syntax_name(delim, 'simple') 154 | ext = get_syntax_file_ext(delim, 'simple') or scope 155 | result = simple_header_template.format(yaml_escape(name), ext, scope) 156 | num_contexts = len(rainbow_scope_names) 157 | for context_id in range(num_contexts): 158 | result += '\n' 159 | result += make_simple_context(delim, context_id, num_contexts) 160 | return result 161 | 162 | 163 | def make_sublime_syntax_standard(delim, policy): 164 | assert policy in ['quoted', 'quoted_rfc'] 165 | scope = 'rbcstn' + ''.join([str(ord(d)) for d in delim]) 166 | name = get_syntax_name(delim, policy) 167 | ext = get_syntax_file_ext(delim, policy) or scope 168 | result = standard_header_template.format(yaml_escape(name), ext, scope) 169 | if policy == 'quoted': 170 | result += non_rfc_endline_rule 171 | num_contexts = len(rainbow_scope_names) 172 | for context_id in range(num_contexts): 173 | result += '\n' 174 | result += make_standard_context(delim, context_id, num_contexts) 175 | return result 176 | 177 | 178 | def make_sublime_syntax(delim, policy): 179 | assert policy in filename_policy_map.keys() 180 | if policy == 'quoted': 181 | return make_sublime_syntax_standard(delim, policy) 182 | elif policy == 'quoted_rfc': 183 | return make_sublime_syntax_standard(delim, policy) 184 | else: 185 | return make_sublime_syntax_simple(delim) 186 | 187 | 188 | def get_pregenerated_delims(): 189 | delims = [chr(i) for i in range(32, 127)] 190 | delims.append('\t') 191 | delims = [delim for delim in delims if re.match('^[a-zA-Z0-9]$', delim) is None] 192 | return delims 193 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rainbow CSV 2 | ![rainbow_csv](https://i.imgur.com/EhV2niB.png) 3 | 4 | ### Main features 5 | 6 | * Highlight columns in *.csv, *.tsv and other separated files in different rainbow colors. 7 | * Provide info about columns on mouse hover. 8 | * Check consistency of CSV files (CSVLint) 9 | * Align columns with spaces and Shrink (trim spaces from fields) 10 | * Execute SQL-like RBQL queries. 11 | 12 | ![screenshot](https://i.imgur.com/UtGKbEg.png) 13 | 14 | ### Usage 15 | Rainbow CSV has content-based csv/tsv autodetection mechanism. This means that the package will analyze plain text files even if they do not have "*.csv" or "*.tsv" extension. 16 | 17 | Rainbow highlighting can also be manually enabled from Sublime context menu (see the demo gif below): 18 | 1. Select a character (or sequence of characters) that you want to use as a delimiter with the cursor 19 | 2. Right mouse click: context menu -> Rainbow CSV -> Enable ... 20 | 21 | You can also disable rainbow highlighting and go back to the original file highlighting using the same context menu. 22 | This feature can be used to temporarily rainbow-highlight even non-table files. 23 | 24 | Manual Rainbow Enabling/Disabling demo gif: 25 | ![demo](https://i.imgur.com/7lSSMst.gif) 26 | 27 | Rainbow CSV also lets you execute SQL-like queries in RBQL language, see the demo gif below: 28 | ![demo gif](https://i.imgur.com/UF6zM4i.gif) 29 | 30 | To Run RBQL query press **F5** or select "Rainbow CSV" -> "Run RBQL query" option from the file context menu. 31 | 32 | 33 | ### Key mappings 34 | 35 | |Key | Action | 36 | |--------------------------|----------------------------------------------------| 37 | |**F5** | Start query editing for the current CSV file | 38 | 39 | ### Commands 40 | 41 | #### Rainbow CSV: Enable Simple 42 | Before running the command you need to select the separator (single or multiple characters) with your cursor 43 | Set the selected character as the separator and enables syntax highlighting. Sublime will generate the syntax file if it doesn't exist. 44 | Simple dialect completely ignores double quotes: i.e. separators can not be escaped in double quoted fields 45 | 46 | #### Rainbow CSV: Enable Standard 47 | Same as the _Enable Simple_ command, but separators can be escaped in double quoted fields. 48 | 49 | #### Rainbow CSV: CSVLint 50 | The linter checks the following: 51 | * consistency of double quotes usage in CSV rows 52 | * consistency of number of fields per CSV row 53 | 54 | #### Rainbow CSV: Run RBQL query 55 | Run RBQL query for the current file. 56 | Unlike F5 button it will work even if the current file is not a CSV table: in this case only 2 variables "a1" and "NR" will be available. 57 | 58 | #### Rainbow CSV: Align CSV columns with spaces 59 | Align CSV columns with spaces in the current file 60 | 61 | #### Rainbow CSV: Shrink CSV table 62 | Remove leading and trailing spaces from all fields in the current file 63 | 64 | 65 | ### Configuration 66 | 67 | To adjust plugin configuration: 68 | 1. Go to "Preferences" -> "Package Settings" -> "Rainbow CSV" -> "Settings". 69 | 2. On the right side change the settings like you'd like. 70 | 71 | 72 | ### Configuration parameters 73 | 74 | To configure the extension, click "Preferences" -> "Package Settings" -> "Rainbow CSV" -> "Settings" 75 | 76 | #### "allow_newlines_in_fields" 77 | Allow quoted multiline fields as defined in [RFC-4180](https://tools.ietf.org/html/rfc4180) 78 | 79 | #### "enable_rainbow_csv_autodetect" 80 | Enable content-based separator autodetection. 81 | Files with ".csv" and ".tsv" extensions are always highlighted no matter what is the value of this option. 82 | 83 | #### "rainbow_csv_autodetect_dialects" 84 | List of CSV dialects to autodetect. 85 | If "enable_rainbow_csv_autodetect" is set to false this setting is ignored 86 | 87 | #### "rainbow_csv_max_file_size_bytes" 88 | Disable Rainbow CSV for files bigger than the specified size. This can be helpful to prevent poor performance and crashes with very large files. 89 | Manual separator selection will override this setting for the current file. 90 | E.g. to disable on files larger than 100 MB, set `"rainbow_csv_max_file_size_bytes": 100000000` 91 | 92 | #### "use_custom_rainbow_colors" 93 | Use custom high-contrast rainbow colors instead of colors provided by your current color scheme. 94 | When you enable this option, "auto_adjust_rainbow_colors" also gets enabled by default. 95 | 96 | #### "auto_adjust_rainbow_colors" 97 | Auto adjust rainbow colors for Packages/User/RainbowCSV.sublime-color-scheme 98 | Rainbow CSV will auto-generate color theme with high-contrast colors to make CSV columns more distinguishable. 99 | You can disable this setting and manually customize Rainbow CSV color scheme at `Packages/User/RainbowCSV.sublime-color-scheme`, you can use the following [RainbowCSV.sublime-color-scheme](https://github.com/mechatroner/sublime_rainbow_csv/blob/master/RainbowCSV.sublime-color-scheme) file as a starting point for your customizations. 100 | Do NOT manually customize Packages/User/RainbowCSV.sublime-color-scheme without disabling this setting, the plugin will just rewrite it in that case. 101 | This option has effect only if "use_custom_rainbow_colors" is set to true 102 | 103 | #### "rbql_backend_language" 104 | RBQL backend language. 105 | Supported values: _"Python"_, _"JS"_ 106 | To use RBQL with JavaScript (JS) you need to have Node JS installed and added to your system path. 107 | 108 | #### "rbql_with_headers" 109 | Default: false 110 | RBQL will treat first records in all input and join files as headers. 111 | You can set this value to true if most of the CSV files you deal with have headers. 112 | You can override this setting on the query level by either adding `WITH (header)` or `WITH (noheader)` to the end of the query. 113 | 114 | #### "rbql_output_format" 115 | Format of RBQL result set tables. 116 | Supported values: _"tsv"_, _"csv"_, _"input"_ 117 | * input: same format as the input table 118 | * tsv: tab separated values. 119 | * csv: is Excel-compatible and allows quoted commas. 120 | 121 | Example: to always use "tsv" as output format add this line to your settings file: `"rbql_output_format": "tsv",` 122 | 123 | #### "rbql_encoding" 124 | RBQL encoding for files and queries. 125 | Supported values: _"latin-1"_, _"utf-8"_ 126 | 127 | 128 | ### References 129 | 130 | * This Sublime Text plugin is an adaptation of Vim's rainbow_csv [plugin](https://github.com/mechatroner/rainbow_csv) 131 | 132 | 133 | # RBQL (Rainbow Query Language) Description 134 | 135 | RBQL is an eval-based SQL-like query engine for (not only) CSV file processing. It provides SQL-like language that supports SELECT queries with Python or JavaScript expressions. 136 | RBQL is best suited for data transformation, data cleaning, and analytical queries. 137 | RBQL is distributed with CLI apps, text editor plugins, Python and JS libraries. 138 | 139 | [Official Site](https://rbql.org/) 140 | 141 | ### Main Features 142 | 143 | * Use Python or JavaScript expressions inside _SELECT_, _UPDATE_, _WHERE_ and _ORDER BY_ statements 144 | * Supports multiple input formats 145 | * Result set of any query immediately becomes a first-class table on its own 146 | * No need to provide FROM statement in the query when the input table is defined by the current context. 147 | * Supports all main SQL keywords 148 | * Supports aggregate functions and GROUP BY queries 149 | * Supports user-defined functions (UDF) 150 | * Provides some new useful query modes which traditional SQL engines do not have 151 | * Lightweight, dependency-free, works out of the box 152 | 153 | #### Limitations: 154 | 155 | * RBQL doesn't support nested queries, but they can be emulated with consecutive queries 156 | * Number of tables in all JOIN queries is always 2 (input table and join table), use consecutive queries to join 3 or more tables 157 | 158 | ### Supported SQL Keywords (Keywords are case insensitive) 159 | 160 | * SELECT 161 | * UPDATE 162 | * WHERE 163 | * ORDER BY ... [ DESC | ASC ] 164 | * [ LEFT | INNER ] JOIN 165 | * DISTINCT 166 | * GROUP BY 167 | * TOP _N_ 168 | * LIMIT _N_ 169 | * AS 170 | 171 | All keywords have the same meaning as in SQL queries. You can check them [online](https://www.w3schools.com/sql/default.asp) 172 | 173 | 174 | ### RBQL variables 175 | RBQL for CSV files provides the following variables which you can use in your queries: 176 | 177 | * _a1_, _a2_,..., _a{N}_ 178 | Variable type: **string** 179 | Description: value of i-th field in the current record in input table 180 | * _b1_, _b2_,..., _b{N}_ 181 | Variable type: **string** 182 | Description: value of i-th field in the current record in join table B 183 | * _NR_ 184 | Variable type: **integer** 185 | Description: Record number (1-based) 186 | * _NF_ 187 | Variable type: **integer** 188 | Description: Number of fields in the current record 189 | * _a.name_, _b.Person_age_, ... _a.{Good_alphanumeric_column_name}_ 190 | Variable type: **string** 191 | Description: Value of the field referenced by it's "name". You can use this notation if the field in the header has a "good" alphanumeric name 192 | * _a["object id"]_, _a['9.12341234']_, _b["%$ !! 10 20"]_ ... _a["Arbitrary column name!"]_ 193 | Variable type: **string** 194 | Description: Value of the field referenced by it's "name". You can use this notation to reference fields by arbitrary values in the header 195 | 196 | 197 | ### UPDATE statement 198 | 199 | _UPDATE_ query produces a new table where original values are replaced according to the UPDATE expression, so it can also be considered a special type of SELECT query. 200 | 201 | ### Aggregate functions and queries 202 | 203 | RBQL supports the following aggregate functions, which can also be used with _GROUP BY_ keyword: 204 | _COUNT_, _ARRAY_AGG_, _MIN_, _MAX_, _SUM_, _AVG_, _VARIANCE_, _MEDIAN_ 205 | 206 | Limitation: aggregate functions inside Python (or JS) expressions are not supported. Although you can use expressions inside aggregate functions. 207 | E.g. `MAX(float(a1) / 1000)` - valid; `MAX(a1) / 1000` - invalid. 208 | There is a workaround for the limitation above for _ARRAY_AGG_ function which supports an optional parameter - a callback function that can do something with the aggregated array. Example: 209 | `SELECT a2, ARRAY_AGG(a1, lambda v: sorted(v)[:5]) GROUP BY a2` - Python; `SELECT a2, ARRAY_AGG(a1, v => v.sort().slice(0, 5)) GROUP BY a2` - JS 210 | 211 | 212 | ### JOIN statements 213 | 214 | Join table B can be referenced either by its file path or by its name - an arbitrary string which the user should provide before executing the JOIN query. 215 | RBQL supports _STRICT LEFT JOIN_ which is like _LEFT JOIN_, but generates an error if any key in the left table "A" doesn't have exactly one matching key in the right table "B". 216 | Table B path can be either relative to the working dir, relative to the main table or absolute. 217 | Limitation: _JOIN_ statements can't contain Python/JS expressions and must have the following form: _ (/path/to/table.tsv | table_name ) ON a... == b... [AND a... == b... [AND ... ]]_ 218 | 219 | ### SELECT EXCEPT statement 220 | 221 | SELECT EXCEPT can be used to select everything except specific columns. E.g. to select everything but columns 2 and 4, run: `SELECT * EXCEPT a2, a4` 222 | Traditional SQL engines do not support this query mode. 223 | 224 | 225 | ### UNNEST() operator 226 | UNNEST(list) takes a list/array as an argument and repeats the output record multiple times - one time for each value from the list argument. 227 | Example: `SELECT a1, UNNEST(a2.split(';'))` 228 | 229 | 230 | ### LIKE() function 231 | RBQL does not support LIKE operator, instead it provides "like()" function which can be used like this: 232 | `SELECT * where like(a1, 'foo%bar')` 233 | 234 | 235 | ### WITH (header) and WITH (noheader) statements 236 | You can set whether the input (and join) CSV file has a header or not using the environment configuration parameters which could be `--with_headers` CLI flag or GUI checkbox or something else. 237 | But it is also possible to override this selection directly in the query by adding either `WITH (header)` or `WITH (noheader)` statement at the end of the query. 238 | Example: `select top 5 NR, * with (header)` 239 | 240 | 241 | ### User Defined Functions (UDF) 242 | 243 | RBQL supports User Defined Functions 244 | You can define custom functions and/or import libraries in two special files: 245 | * `~/.rbql_init_source.py` - for Python 246 | * `~/.rbql_init_source.js` - for JavaScript 247 | 248 | 249 | ## Examples of RBQL queries 250 | 251 | #### With Python expressions 252 | 253 | * `SELECT TOP 100 a1, int(a2) * 10, len(a4) WHERE a1 == "Buy" ORDER BY int(a2) DESC` 254 | * `SELECT a.id, a.weight / 1000 AS weight_kg` 255 | * `SELECT * ORDER BY random.random()` - random sort 256 | * `SELECT len(a.vehicle_price) / 10, a2 WHERE int(a.vehicle_price) < 500 and a['Vehicle type'] in ["car", "plane", "boat"] limit 20` - referencing columns by names from header and using Python's "in" to emulate SQL's "in" 257 | * `UPDATE SET a3 = 'NPC' WHERE a3.find('Non-playable character') != -1` 258 | * `SELECT NR, *` - enumerate records, NR is 1-based 259 | * `SELECT * WHERE re.match(".*ab.*", a1) is not None` - select entries where first column has "ab" pattern 260 | * `SELECT a1, b1, b2 INNER JOIN ./countries.txt ON a2 == b1 ORDER BY a1, a3` - example of join query 261 | * `SELECT MAX(a1), MIN(a1) WHERE a.Name != 'John' GROUP BY a2, a3` - example of aggregate query 262 | * `SELECT *a1.split(':')` - Using Python3 unpack operator to split one column into many. Do not try this with other SQL engines! 263 | 264 | #### With JavaScript expressions 265 | 266 | * `SELECT TOP 100 a1, a2 * 10, a4.length WHERE a1 == "Buy" ORDER BY parseInt(a2) DESC` 267 | * `SELECT a.id, a.weight / 1000 AS weight_kg` 268 | * `SELECT * ORDER BY Math.random()` - random sort 269 | * `SELECT TOP 20 a.vehicle_price.length / 10, a2 WHERE parseInt(a.vehicle_price) < 500 && ["car", "plane", "boat"].indexOf(a['Vehicle type']) > -1 limit 20` - referencing columns by names from header 270 | * `UPDATE SET a3 = 'NPC' WHERE a3.indexOf('Non-playable character') != -1` 271 | * `SELECT NR, *` - enumerate records, NR is 1-based 272 | * `SELECT a1, b1, b2 INNER JOIN ./countries.txt ON a2 == b1 ORDER BY a1, a3` - example of join query 273 | * `SELECT MAX(a1), MIN(a1) WHERE a.Name != 'John' GROUP BY a2, a3` - example of aggregate query 274 | * `SELECT ...a1.split(':')` - Using JS "destructuring assignment" syntax to split one column into many. Do not try this with other SQL engines! 275 | 276 | 277 | ### References 278 | 279 | #### Rainbow CSV and similar plugins in other editors: 280 | 281 | * Rainbow CSV extension in [Vim](https://github.com/mechatroner/rainbow_csv) 282 | * Rainbow CSV extension in [Visual Studio Code](https://marketplace.visualstudio.com/items?itemName=mechatroner.rainbow-csv) 283 | * rainbow-csv package in [Atom](https://atom.io/packages/rainbow-csv) 284 | * rainbow_csv plugin in [gedit](https://github.com/mechatroner/gtk_gedit_rainbow_csv) - doesn't support quoted commas in csv 285 | * rainbow_csv_4_nedit in [NEdit](https://github.com/DmitTrix/rainbow_csv_4_nedit) 286 | * CSV highlighting in [Nano](https://github.com/scopatz/nanorc) 287 | * Rainbow CSV in [IntelliJ IDEA](https://plugins.jetbrains.com/plugin/12896-rainbow-csv/) 288 | 289 | #### RBQL: 290 | 291 | * [RBQL: Official Site](https://rbql.org/) 292 | * RBQL is integrated with Rainbow CSV extensions in [Vim](https://github.com/mechatroner/rainbow_csv), [VSCode](https://marketplace.visualstudio.com/items?itemName=mechatroner.rainbow-csv), [Sublime Text](https://packagecontrol.io/packages/rainbow_csv) and [Atom](https://atom.io/packages/rainbow-csv) editors. 293 | * [RBQL in npm](https://www.npmjs.com/package/rbql): `$ npm install -g rbql` 294 | * [RBQL in PyPI](https://pypi.org/project/rbql/): `$ pip install rbql` 295 | 296 | -------------------------------------------------------------------------------- /rbql-js/cli_rbql.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const fs = require('fs'); 4 | const readline = require('readline'); 5 | 6 | const rbql = require('./rbql.js'); 7 | const rbql_csv = require('./rbql_csv.js'); 8 | const csv_utils = require('./csv_utils.js'); 9 | const cli_parser = require('./cli_parser.js'); 10 | 11 | let out_format_names = ['csv', 'tsv', 'monocolumn', 'input']; 12 | 13 | var error_format = 'hr'; 14 | var interactive_mode = false; 15 | 16 | 17 | // TODO implement colored output like in Python version 18 | // TODO implement query history like in Python version. "readline" modules allows to do that, see "completer" parameter. 19 | 20 | // FIXME test readline on Win: disable interactive mode? 21 | 22 | // FIXME handle broken pipe error and add tests. See Python version. 23 | 24 | 25 | class RbqlParsingError extends Error {} 26 | class GenericError extends Error {} 27 | 28 | 29 | function show_error_plain_text(error_type, error_msg) { 30 | if (interactive_mode) { 31 | console.log(`\x1b[31;1mError [${error_type}]:\x1b[0m ${error_msg}`); 32 | } else { 33 | console.error(`Error [${error_type}]: ${error_msg}`); 34 | } 35 | } 36 | 37 | 38 | function report_error_json(error_type, error_msg) { 39 | let report = new Object(); 40 | report.error_type = error_type; 41 | report.error = error_msg; 42 | process.stderr.write(JSON.stringify(report)); 43 | } 44 | 45 | 46 | function show_exception(e) { 47 | let [error_type, error_msg] = rbql.exception_to_error_info(e); 48 | if (error_format == 'hr') { 49 | show_error_plain_text(error_type, error_msg); 50 | } else { 51 | report_error_json(error_type, error_msg); 52 | } 53 | } 54 | 55 | 56 | function show_warning(msg) { 57 | if (interactive_mode) { 58 | console.log('\x1b[33;1mWarning:\x1b[0m ' + msg); 59 | } else { 60 | console.error('Warning: ' + msg); 61 | } 62 | } 63 | 64 | 65 | function normalize_delim(delim) { 66 | if (delim == 'TAB') 67 | return '\t'; 68 | if (delim == '\\t') 69 | return '\t'; 70 | return delim; 71 | } 72 | 73 | 74 | function get_default(src, key, default_val) { 75 | return src.hasOwnProperty(key) ? src[key] : default_val; 76 | } 77 | 78 | 79 | async function read_user_query(user_input_reader) { 80 | let finish_promise = new Promise(function(resolve, reject) { 81 | user_input_reader.question('Input SQL-like RBQL query and press Enter:\n> ', (query) => { 82 | resolve(query); 83 | }); 84 | }); 85 | let query = await finish_promise; 86 | return query; 87 | } 88 | 89 | 90 | function get_default_policy(delim) { 91 | if ([';', ','].indexOf(delim) != -1) { 92 | return 'quoted'; 93 | } else if (delim == ' ') { 94 | return 'whitespace'; 95 | } else { 96 | return 'simple'; 97 | } 98 | } 99 | 100 | 101 | function is_delimited_table(sampled_lines, delim, policy) { 102 | if (sampled_lines.length < 10) 103 | return false; 104 | let num_fields = null; 105 | for (var i = 0; i < sampled_lines.length; i++) { 106 | let [fields, warning] = csv_utils.smart_split(sampled_lines[i], delim, policy, true); 107 | if (warning) 108 | return false; 109 | if (num_fields === null) 110 | num_fields = fields.length; 111 | if (num_fields < 2 || num_fields != fields.length) 112 | return false; 113 | } 114 | return true; 115 | } 116 | 117 | 118 | async function sample_lines(table_path) { 119 | let finish_promise = new Promise(function(resolve, reject) { 120 | let input_reader = readline.createInterface({ input: fs.createReadStream(table_path) }); 121 | let sampled_lines = []; 122 | input_reader.on('line', line => { 123 | if (sampled_lines.length < 10) { 124 | sampled_lines.push(line); 125 | } else { 126 | input_reader.close(); 127 | } 128 | }); 129 | input_reader.on('close', () => { resolve(sampled_lines); }); 130 | }); 131 | let sampled_lines = await finish_promise; 132 | return sampled_lines; 133 | } 134 | 135 | 136 | async function sample_records(table_path, encoding, delim, policy) { 137 | let table_stream = fs.createReadStream(table_path); 138 | let sampling_iterator = new rbql_csv.CSVRecordIterator(table_stream, null, encoding, delim, policy); 139 | let sampled_records = await sampling_iterator.get_all_records(10); 140 | let warnings = sampling_iterator.get_warnings(); 141 | return [sampled_records, warnings]; 142 | } 143 | 144 | 145 | async function autodetect_delim_policy(table_path) { 146 | let sampled_lines = await sample_lines(table_path); 147 | let autodetection_dialects = [['\t', 'simple'], [',', 'quoted'], [';', 'quoted'], ['|', 'simple']]; 148 | for (var i = 0; i < autodetection_dialects.length; i++) { 149 | let [delim, policy] = autodetection_dialects[i]; 150 | if (is_delimited_table(sampled_lines, delim, policy)) 151 | return [delim, policy]; 152 | } 153 | if (table_path.endsWith('.csv')) 154 | return [',', 'quoted']; 155 | if (table_path.endsWith('.tsv')) 156 | return ['\t', 'simple']; 157 | return [null, null]; 158 | } 159 | 160 | 161 | function print_colorized(records, delim, show_column_names, with_headers) { 162 | let reset_color_code = '\x1b[0m'; 163 | let color_codes = ['\x1b[0m', '\x1b[31m', '\x1b[32m', '\x1b[33m', '\x1b[34m', '\x1b[35m', '\x1b[36m', '\x1b[31;1m', '\x1b[32;1m', '\x1b[33;1m']; 164 | for (let r = 0; r < records.length; r++) { 165 | let out_fields = []; 166 | for (let c = 0; c < records[r].length; c++) { 167 | let color_code = color_codes[c % color_codes.length]; 168 | let field = records[r][c]; 169 | let colored_field = (!show_column_names || (with_headers && r == 0)) ? color_code + field : `${color_code}a${c + 1}:${field}`; 170 | out_fields.push(colored_field); 171 | } 172 | let out_line = out_fields.join(delim) + reset_color_code; 173 | console.log(out_line); 174 | } 175 | } 176 | 177 | 178 | async function handle_query_success(warnings, output_path, encoding, delim, policy) { 179 | if (error_format == 'hr') { 180 | if (warnings !== null) { 181 | for (let i = 0; i < warnings.length; i++) { 182 | show_warning(warnings[i]); 183 | } 184 | } 185 | if (interactive_mode) { 186 | let [records, _warnings] = await sample_records(output_path, encoding, delim, policy); 187 | console.log('\nOutput table preview:'); 188 | console.log('===================================='); 189 | print_colorized(records, delim, false, false); 190 | console.log('===================================='); 191 | console.log('Success! Result table was saved to: ' + output_path); 192 | } 193 | } else { 194 | if (warnings !== null && warnings.length) { 195 | var warnings_report = JSON.stringify({'warnings': warnings}); 196 | process.stderr.write(warnings_report); 197 | } 198 | } 199 | } 200 | 201 | 202 | async function run_with_js(args) { 203 | var delim = normalize_delim(args['delim']); 204 | var policy = args['policy'] ? args['policy'] : get_default_policy(delim); 205 | var query = args['query']; 206 | if (!query) 207 | throw new RbqlParsingError('RBQL query is empty'); 208 | var input_path = get_default(args, 'input', null); 209 | var output_path = get_default(args, 'output', null); 210 | var csv_encoding = args['encoding']; 211 | var with_headers = args['with-headers']; 212 | var comment_prefix = args['comment-prefix']; 213 | var output_delim = get_default(args, 'out-delim', null); 214 | var output_policy = get_default(args, 'out-policy', null); 215 | let init_source_file = get_default(args, 'init-source-file', null); 216 | let output_format = args['out-format']; 217 | if (output_delim === null) { 218 | [output_delim, output_policy] = output_format == 'input' ? [delim, policy] : rbql_csv.interpret_named_csv_format(output_format); 219 | } 220 | 221 | let user_init_code = ''; 222 | if (init_source_file !== null) 223 | user_init_code = rbql_csv.read_user_init_code(init_source_file); 224 | try { 225 | let warnings = []; 226 | // Do not use bulk_read mode here because: 227 | // * Bulk read can't handle large file since node unable to read the whole file into a string, see https://github.com/mechatroner/rainbow_csv/issues/19 228 | // * In case of stdin read we would have to use the util.TextDecoder anyway 229 | // * binary/latin-1 do not require the decoder anyway 230 | // * This is CLI so no way we are in the Electron environment which can't use the TextDecoder 231 | // * Streaming mode works a little faster (since we don't need to do the manual validation) 232 | // TODO check if the current node installation doesn't have ICU enabled (which is typicaly provided by Node.js by default, see https://nodejs.org/api/intl.html) and report a user-friendly error with an option to use latin-1 encoding or switch the interpreter 233 | await rbql_csv.query_csv(query, input_path, delim, policy, output_path, output_delim, output_policy, csv_encoding, warnings, with_headers, comment_prefix, user_init_code/*, {'bulk_read': true}*/); 234 | await handle_query_success(warnings, output_path, csv_encoding, output_delim, output_policy); 235 | return true; 236 | } catch (e) { 237 | if (!interactive_mode) 238 | throw e; 239 | show_exception(e); 240 | return false; 241 | } 242 | } 243 | 244 | 245 | function get_default_output_path(input_path, delim) { 246 | let well_known_extensions = {',': '.csv', '\t': '.tsv'}; 247 | if (well_known_extensions.hasOwnProperty(delim)) 248 | return input_path + well_known_extensions[delim]; 249 | return input_path + '.txt'; 250 | } 251 | 252 | 253 | async function show_preview(input_path, encoding, delim, policy, with_headers) { 254 | let [records, warnings] = await sample_records(input_path, encoding, delim, policy); 255 | console.log('Input table preview:'); 256 | console.log('===================================='); 257 | print_colorized(records, delim, true, with_headers); 258 | console.log('====================================\n'); 259 | for (let warning of warnings) { 260 | show_warning(warning); 261 | } 262 | } 263 | 264 | 265 | async function run_interactive_loop(args) { 266 | let input_path = get_default(args, 'input', null); 267 | if (!input_path) 268 | throw new GenericError('Input file must be provided in interactive mode. You can use stdin input only in non-interactive mode'); 269 | if (error_format != 'hr') 270 | throw new GenericError('Only default "hr" error format is supported in interactive mode'); 271 | 272 | 273 | let delim = get_default(args, 'delim', null); 274 | let policy = null; 275 | if (delim !== null) { 276 | delim = normalize_delim(delim); 277 | policy = args['policy'] ? args['policy'] : get_default_policy(delim); 278 | } else { 279 | [delim, policy] = await autodetect_delim_policy(input_path); 280 | if (!delim) 281 | throw new GenericError('Unable to autodetect table delimiter. Provide column separator explicitly with "--delim" option'); 282 | } 283 | await show_preview(input_path, args['encoding'], delim, policy, args['with-headers']); 284 | args.delim = delim; 285 | args.policy = policy; 286 | if (!args.output) { 287 | args.output = get_default_output_path(input_path, delim); 288 | show_warning('Output path was not provided. Result set will be saved as: ' + args.output); 289 | } 290 | 291 | let user_input_reader = readline.createInterface({ input: process.stdin, output: process.stdout }); 292 | try { 293 | while (true) { 294 | let query = await read_user_query(user_input_reader); 295 | args.query = query; 296 | let success = await run_with_js(args); 297 | if (success) 298 | break; 299 | } 300 | } finally { 301 | user_input_reader.close(); 302 | } 303 | } 304 | 305 | 306 | let tool_description = `rbql-js 307 | 308 | Run RBQL queries against CSV files and data streams 309 | 310 | rbql-js supports two modes: non-interactive (with "--query" option) and interactive (without "--query" option) 311 | Interactive mode shows source table preview which makes query editing much easier. Usage example: 312 | $ rbql-js --input input.csv 313 | Non-interactive mode supports source tables in stdin. Usage example: 314 | $ rbql-js --query "select a1, a2 order by a1" --delim , < input.csv 315 | `; 316 | 317 | let epilog = ` 318 | Description of the available CSV split policies: 319 | * "simple" - RBQL uses simple split() function and doesn't perform special handling of double quote characters 320 | * "quoted" - Separator can be escaped inside double-quoted fields. Double quotes inside double-quoted fields must be doubled 321 | * "quoted_rfc" - Same as "quoted", but also allows newlines inside double-quoted fields, see RFC-4180: https://tools.ietf.org/html/rfc4180 322 | * "whitespace" - Works only with whitespace separator, multiple consecutive whitespaces are treated as a single whitespace 323 | * "monocolumn" - RBQL doesn't perform any split at all, each line is a single-element record, i.e. only "a1" and "NR" are available 324 | `; 325 | 326 | 327 | async function do_main(args) { 328 | 329 | if (args['version']) { 330 | console.log(rbql.version); 331 | process.exit(0); 332 | } 333 | 334 | if (args.hasOwnProperty('policy') && args['policy'] === 'monocolumn') 335 | args['delim'] = ''; 336 | 337 | if (args.hasOwnProperty('policy') && !args.hasOwnProperty('delim')) 338 | throw new GenericError('Using "--policy" without "--delim" is not allowed'); 339 | 340 | if (args.encoding == 'latin-1') 341 | args.encoding = 'binary'; 342 | 343 | error_format = args['error-format']; 344 | 345 | if (args.hasOwnProperty('query')) { 346 | interactive_mode = false; 347 | if (!args.hasOwnProperty('delim')) { 348 | throw new GenericError('Separator must be provided with "--delim" option in non-interactive mode'); 349 | } 350 | await run_with_js(args); 351 | } else { 352 | interactive_mode = true; 353 | if (error_format == 'json') { 354 | throw new GenericError('json error format is not compatible with interactive mode'); 355 | } 356 | await run_interactive_loop(args); 357 | } 358 | } 359 | 360 | 361 | function main() { 362 | var scheme = { 363 | '--input': {'help': 'Read csv table from FILE instead of stdin. Required in interactive mode', 'metavar': 'FILE'}, 364 | '--query': {'help': 'Query string in rbql. Run in interactive mode if empty', 'metavar': 'QUERY'}, 365 | '--output': {'help': 'Write output table to FILE instead of stdout', 'metavar': 'FILE'}, 366 | '--delim': {'help': 'Delimiter character or multicharacter string, e.g. "," or "###". Can be autodetected in interactive mode', 'metavar': 'DELIM'}, 367 | '--policy': {'help': 'Split policy, see the explanation below. Supported values: "simple", "quoted", "quoted_rfc", "whitespace", "monocolumn". Can be autodetected in interactive mode', 'metavar': 'POLICY'}, 368 | '--with-headers': {'boolean': true, 'help': 'Indicates that input (and join) table has header'}, 369 | '--comment-prefix': {'help': 'Ignore lines in input and join tables that start with the comment PREFIX, e.g. "#" or ">>"', 'metavar': 'PREFIX'}, 370 | '--encoding': {'default': 'utf-8', 'help': 'Manually set csv encoding', 'metavar': 'ENCODING'}, 371 | '--out-format': {'default': 'input', 'help': 'Output format. Supported values: ' + out_format_names.map(v => `"${v}"`).join(', '), 'metavar': 'FORMAT'}, 372 | '--out-delim': {'help': 'Output delim. Use with "out-policy". Overrides out-format', 'metavar': 'DELIM'}, 373 | '--out-policy': {'help': 'Output policy. Use with "out-delim". Overrides out-format', 'metavar': 'POLICY'}, 374 | '--error-format': {'default': 'hr', 'help': 'Errors and warnings format. [hr|json]', 'hidden': true}, 375 | '--version': {'boolean': true, 'help': 'Print RBQL version and exit'}, 376 | '--init-source-file': {'help': 'Path to init source file to use instead of ~/.rbql_init_source.js', 'hidden': true} 377 | }; 378 | let args = cli_parser.parse_cmd_args(process.argv, scheme, tool_description, epilog); 379 | do_main(args).then(() => {}).catch(error_info => { show_exception(error_info); process.exit(1); }); 380 | } 381 | 382 | 383 | if (require.main === module) { 384 | main(); 385 | } 386 | 387 | 388 | -------------------------------------------------------------------------------- /rbql/rbql_main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import unicode_literals 3 | from __future__ import print_function 4 | 5 | import sys 6 | import os 7 | import argparse 8 | 9 | from . import csv_utils 10 | from . import rbql_csv 11 | from . import rbql_sqlite 12 | from . import rbql_engine 13 | from . import _version 14 | 15 | # TODO support sqlite input join on both sqlite and csv tables - pass 2 join registries 16 | # TODO add demo gif to python package README.md for pypi website 17 | 18 | # TODO add --output_header param 19 | # TODO add option to write to other sqlite dbs 20 | 21 | # TODO add an option to align columns for content preview. This would be especially useful for Windows which doesn't support terminal colors 22 | 23 | 24 | PY3 = sys.version_info[0] == 3 25 | 26 | 27 | history_path = os.path.join(os.path.expanduser("~"), ".rbql_py_query_history") 28 | 29 | polymorphic_input = input if PY3 else raw_input 30 | 31 | 32 | def eprint(*args, **kwargs): 33 | print(*args, file=sys.stderr, **kwargs) 34 | 35 | 36 | policy_names = ['quoted', 'simple', 'whitespace', 'quoted_rfc', 'monocolumn'] 37 | out_format_names = ['csv', 'tsv', 'input'] 38 | 39 | 40 | polymorphic_xrange = range if PY3 else xrange 41 | 42 | 43 | def get_default_policy(delim): 44 | if delim in [';', ',']: 45 | return 'quoted' 46 | elif delim == ' ': 47 | return 'whitespace' 48 | else: 49 | return 'simple' 50 | 51 | 52 | def show_error(error_type, error_msg, is_interactive): 53 | if is_interactive: 54 | if os.name == 'nt': # Windows does not support terminal colors 55 | print('Error [{}]: {}'.format(error_type, error_msg)) 56 | else: 57 | print('{}Error [{}]:{} {}'.format('\u001b[31;1m', error_type, '\u001b[0m', error_msg)) 58 | else: 59 | eprint('Error [{}]: {}'.format(error_type, error_msg)) 60 | 61 | 62 | def show_warning(msg, is_interactive): 63 | if is_interactive: 64 | if os.name == 'nt': # Windows does not support terminal colors 65 | print('Warning: ' + msg) 66 | else: 67 | print('{}Warning:{} {}'.format('\u001b[33;1m', '\u001b[0m', msg)) 68 | else: 69 | eprint('Warning: ' + msg) 70 | 71 | 72 | def run_with_python_csv(args, is_interactive): 73 | if args.debug_mode: 74 | rbql_csv.set_debug_mode() 75 | delim = rbql_csv.normalize_delim(args.delim) 76 | policy = args.policy if args.policy is not None else get_default_policy(delim) 77 | query = args.query 78 | with_headers = args.with_headers 79 | input_path = args.input 80 | output_path = args.output 81 | csv_encoding = args.encoding 82 | args.output_delim, args.output_policy = (delim, policy) if args.out_format == 'input' else rbql_csv.interpret_named_csv_format(args.out_format) 83 | out_delim, out_policy = args.output_delim, args.output_policy 84 | 85 | user_init_code = rbql_csv.read_user_init_code(args.init_source_file) if args.init_source_file is not None else '' 86 | 87 | warnings = [] 88 | error_type, error_msg = None, None 89 | try: 90 | rbql_csv.query_csv(query, input_path, delim, policy, output_path, out_delim, out_policy, csv_encoding, warnings, with_headers, args.comment_prefix, user_init_code, args.color) 91 | except Exception as e: 92 | if args.debug_mode: 93 | raise 94 | error_type, error_msg = rbql_engine.exception_to_error_info(e) 95 | 96 | if error_type is None: 97 | success = True 98 | for warning in warnings: 99 | show_warning(warning, is_interactive) 100 | else: 101 | success = False 102 | show_error(error_type, error_msg, is_interactive) 103 | 104 | return success 105 | 106 | 107 | def run_with_python_sqlite(args, is_interactive): 108 | import sqlite3 109 | user_init_code = rbql_csv.read_user_init_code(args.init_source_file) if args.init_source_file is not None else '' 110 | 111 | warnings = [] 112 | error_type, error_msg = None, None 113 | try: 114 | # TODO open in readonly mode 115 | db_connection = sqlite3.connect(args.database) 116 | if args.debug_mode: 117 | rbql_engine.set_debug_mode() 118 | rbql_sqlite.query_sqlite_to_csv(args.query, db_connection, args.input, args.output, args.output_delim, args.output_policy, args.encoding, warnings, user_init_code, args.color) 119 | except Exception as e: 120 | if args.debug_mode: 121 | raise 122 | error_type, error_msg = rbql_engine.exception_to_error_info(e) 123 | finally: 124 | db_connection.close() 125 | 126 | if error_type is None: 127 | success = True 128 | for warning in warnings: 129 | show_warning(warning, is_interactive) 130 | else: 131 | success = False 132 | show_error(error_type, error_msg, is_interactive) 133 | 134 | return success 135 | 136 | 137 | def is_delimited_table(sampled_lines, delim, policy): 138 | if len(sampled_lines) < 2: 139 | return False 140 | num_fields = None 141 | for sl in sampled_lines: 142 | fields, warning = csv_utils.smart_split(sl, delim, policy, True) 143 | if warning or len(fields) < 2: 144 | return False 145 | if num_fields is None: 146 | num_fields = len(fields) 147 | if num_fields != len(fields): 148 | return False 149 | return True 150 | 151 | 152 | def sample_lines(src_path, encoding, delim, policy, comment_prefix=None): 153 | # TODO this should be a dependency-free function, remove sample line functionality from CSVRecordIterator 154 | result = [] 155 | with open(src_path, 'rb') as source: 156 | line_iterator = rbql_csv.CSVRecordIterator(source, encoding, delim=delim, policy=policy, line_mode=True, comment_prefix=comment_prefix) 157 | for _i in polymorphic_xrange(10): 158 | line = line_iterator.polymorphic_get_row() 159 | if line is None: 160 | break 161 | result.append(line) 162 | return result 163 | 164 | 165 | def autodetect_delim_policy(input_path, encoding, comment_prefix=None): 166 | sampled_lines = sample_lines(input_path, encoding, None, None, comment_prefix) 167 | autodetection_dialects = [('\t', 'simple'), (',', 'quoted'), (';', 'quoted'), ('|', 'simple')] 168 | for delim, policy in autodetection_dialects: 169 | if is_delimited_table(sampled_lines, delim, policy): 170 | return (delim, policy) 171 | if input_path.endswith('.csv'): 172 | return (',', 'quoted') 173 | if input_path.endswith('.tsv'): 174 | return ('\t', 'simple') 175 | return (None, None) 176 | 177 | 178 | def sample_records(input_path, delim, policy, encoding, comment_prefix=None): 179 | with open(input_path, 'rb') as source: 180 | record_iterator = rbql_csv.CSVRecordIterator(source, encoding, delim=delim, policy=policy, comment_prefix=comment_prefix) 181 | sampled_records = record_iterator.get_all_records(num_rows=10); 182 | warnings = record_iterator.get_warnings() 183 | return (sampled_records, warnings) 184 | 185 | 186 | def print_colorized(records, delim, encoding, show_column_names, with_headers): 187 | # TODO consider colorizing a1,a2,... in different default color 188 | if os.name == 'nt': # Windows does not support terminal colors 189 | reset_color_code = '' 190 | color_codes = [''] 191 | else: 192 | reset_color_code = '\u001b[0m' 193 | color_codes = ['\u001b[0m', '\u001b[31m', '\u001b[32m', '\u001b[33m', '\u001b[34m', '\u001b[35m', '\u001b[36m', '\u001b[31;1m', '\u001b[32;1m', '\u001b[33;1m'] 194 | for rnum, record in enumerate(records): 195 | out_fields = [] 196 | for i, field in enumerate(record): 197 | color_code = color_codes[i % len(color_codes)] 198 | if not show_column_names or (with_headers and rnum == 0): 199 | colored_field = '{}{}'.format(color_code, field) 200 | else: 201 | colored_field = '{}a{}:{}'.format(color_code, i + 1, field) 202 | out_fields.append(colored_field) 203 | out_line = delim.join(out_fields) + reset_color_code 204 | if PY3: 205 | sys.stdout.buffer.write(out_line.encode(encoding)) 206 | else: 207 | sys.stdout.write(out_line.encode(encoding)) 208 | sys.stdout.write('\n') 209 | sys.stdout.flush() 210 | 211 | 212 | def get_default_output_path(input_path, delim): 213 | well_known_extensions = {',': '.csv', '\t': '.tsv'} 214 | if delim in well_known_extensions: 215 | return input_path + well_known_extensions[delim] 216 | return input_path + '.txt' 217 | 218 | 219 | def run_interactive_loop(mode, args): 220 | assert mode in ['csv', 'sqlite'] 221 | try: 222 | import readline # Module readline is not available on Windows 223 | if os.path.exists(history_path): 224 | readline.read_history_file(history_path) 225 | readline.set_history_length(100) 226 | except Exception: 227 | pass 228 | while True: 229 | try: 230 | query = polymorphic_input('Input SQL-like RBQL query and press Enter:\n> ') 231 | query = query.strip() 232 | except EOFError: 233 | print() 234 | break # Ctrl-D 235 | if not len(query): 236 | break 237 | try: 238 | readline.write_history_file(history_path) # This can fail sometimes for no valid reason 239 | except Exception: 240 | pass 241 | args.query = query 242 | if mode == 'csv': 243 | success = run_with_python_csv(args, is_interactive=True) 244 | else: 245 | success = run_with_python_sqlite(args, is_interactive=True) 246 | if success: 247 | print('\nOutput table preview:') 248 | print('====================================') 249 | records, _warnings = sample_records(args.output, args.output_delim, args.output_policy, args.encoding, comment_prefix=None) 250 | print_colorized(records, args.output_delim, args.encoding, show_column_names=False, with_headers=False) 251 | print('====================================') 252 | print('Success! Result table was saved to: ' + args.output) 253 | break 254 | 255 | 256 | def sample_records_sqlite(db_connection, table_name): 257 | import sqlite3 258 | record_iterator = rbql_sqlite.SqliteRecordIterator(db_connection, table_name) 259 | records = [] 260 | records.append(record_iterator.get_column_names()) 261 | records += record_iterator.get_all_records(num_rows=10) 262 | return records 263 | 264 | 265 | def read_table_names(db_connection): 266 | cursor = db_connection.cursor() 267 | cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") 268 | table_names = [r[0] for r in cursor.fetchall()] 269 | return table_names 270 | 271 | 272 | def select_table_name_by_user_choice(db_connection): 273 | table_names = read_table_names(db_connection) 274 | max_to_show = 20 275 | if len(table_names) > max_to_show: 276 | print('Database has {} tables, showing top {}:'.format(len(table_names), max_to_show)) 277 | else: 278 | print('Showing database tables:') 279 | print(', '.join(table_names[:max_to_show])) 280 | table_name = polymorphic_input('No input table was provided as a CLI argument, please type in the table name to use:\n> ') 281 | table_name = table_name.strip() 282 | while table_name not in table_names: 283 | table_name = polymorphic_input('"{}" is not a valid table name. Please enter a valid table name:\n> '.format(table_name)) 284 | table_name = table_name.strip() 285 | return table_name 286 | 287 | 288 | def start_preview_mode_sqlite(args): 289 | import sqlite3 290 | db_path = args.database 291 | db_connection = sqlite3.connect(db_path) 292 | if not args.input: 293 | args.input = select_table_name_by_user_choice(db_connection) 294 | try: 295 | records = sample_records_sqlite(db_connection, table_name=args.input) 296 | except Exception as e: 297 | if args.debug_mode: 298 | raise 299 | error_type, error_msg = rbql_engine.exception_to_error_info(e) 300 | show_error(error_type, 'Unable to sample preview records: {}'.format(error_msg), is_interactive=True) 301 | sys.exit(1) 302 | db_connection.close() 303 | 304 | print('Input table preview:') 305 | print('====================================') 306 | print_colorized(records, '|', args.encoding, show_column_names=True, with_headers=False) 307 | print('====================================\n') 308 | if args.output is None: 309 | args.output = get_default_output_path('rbql_sqlite_rs', args.output_delim) 310 | show_warning('Output path was not provided. Result set will be saved as: ' + args.output, is_interactive=True) 311 | try: 312 | run_interactive_loop('sqlite', args) 313 | except KeyboardInterrupt: 314 | print() 315 | 316 | 317 | def start_preview_mode_csv(args): 318 | input_path = args.input 319 | if not input_path: 320 | show_error('generic', 'Input file must be provided in interactive mode. You can use stdin input only in non-interactive mode', is_interactive=True) 321 | return 322 | if not os.path.exists(input_path): 323 | show_error('generic', 'Input file {} does not exist'.format(input_path), is_interactive=True) 324 | return 325 | if args.delim is not None: 326 | delim = rbql_csv.normalize_delim(args.delim) 327 | policy = args.policy if args.policy is not None else get_default_policy(delim) 328 | else: 329 | delim, policy = autodetect_delim_policy(input_path, args.encoding, args.comment_prefix) 330 | if delim is None: 331 | show_error('generic', 'Unable to autodetect table delimiter. Provide column separator explicitly with "--delim" option', is_interactive=True) 332 | return 333 | args.delim = delim 334 | args.policy = policy 335 | records, warnings = sample_records(input_path, delim, policy, args.encoding, args.comment_prefix) 336 | print('Input table preview:') 337 | print('====================================') 338 | print_colorized(records, delim, args.encoding, show_column_names=True, with_headers=args.with_headers) 339 | print('====================================\n') 340 | for warning in warnings: 341 | show_warning(warning, is_interactive=True) 342 | if args.output is None: 343 | args.output = get_default_output_path(input_path, delim) 344 | show_warning('Output path was not provided. Result set will be saved as: ' + args.output, is_interactive=True) 345 | try: 346 | run_interactive_loop('csv', args) 347 | except KeyboardInterrupt: 348 | print() 349 | 350 | 351 | csv_tool_description = ''' 352 | Run RBQL queries against CSV files, sqlite databases 353 | 354 | rbql supports two modes: non-interactive (with "--query" option) and interactive (without "--query" option) 355 | 356 | Interactive mode shows source table preview which makes query editing much easier. Usage example: 357 | $ rbql --input input.csv 358 | 359 | Non-interactive mode supports reading input tables from stdin and writing output to stdout. Usage example: 360 | $ rbql --query "select a1, a2 order by a1" --delim , < input.csv 361 | 362 | By default rbql works with CSV input files. 363 | To learn how to use rbql to query an sqlite database, run this command: 364 | 365 | $ rbql sqlite --help 366 | 367 | ''' 368 | 369 | csv_epilog = ''' 370 | Description of the available CSV split policies: 371 | * "simple" - RBQL uses simple split() function and doesn't perform special handling of double quote characters 372 | * "quoted" - Separator can be escaped inside double-quoted fields. Double quotes inside double-quoted fields must be doubled 373 | * "quoted_rfc" - Same as "quoted", but also allows newlines inside double-quoted fields, see RFC-4180: https://tools.ietf.org/html/rfc4180 374 | * "whitespace" - Works only with whitespace separator, multiple consecutive whitespaces are treated as a single whitespace 375 | * "monocolumn" - RBQL doesn't perform any split at all, each line is a single-element record, i.e. only "a1" and "NR" column variables are available 376 | ''' 377 | 378 | 379 | def csv_main(): 380 | parser = argparse.ArgumentParser(prog='rbql [csv]', formatter_class=argparse.RawDescriptionHelpFormatter, description=csv_tool_description, epilog=csv_epilog) 381 | parser.add_argument('--input', metavar='FILE', help='read csv table from FILE instead of stdin. Required in interactive mode') 382 | parser.add_argument('--delim', help='delimiter character or multicharacter string, e.g. "," or "###". Can be autodetected in interactive mode') 383 | parser.add_argument('--policy', help='CSV split policy, see the explanation below. Can be autodetected in interactive mode', choices=policy_names) 384 | parser.add_argument('--with-headers', action='store_true', help='indicates that input (and join) table has header') 385 | parser.add_argument('--comment-prefix', metavar='PREFIX', help='ignore lines in input and join tables that start with the comment PREFIX, e.g. "#" or ">>"') 386 | parser.add_argument('--query', help='query string in rbql. Run in interactive mode if empty') 387 | parser.add_argument('--out-format', help='output format', default='input', choices=out_format_names) 388 | parser.add_argument('--encoding', help='manually set csv encoding', default=rbql_csv.default_csv_encoding, choices=['latin-1', 'utf-8']) 389 | parser.add_argument('--output', metavar='FILE', help='write output table to FILE instead of stdout') 390 | parser.add_argument('--color', action='store_true', help='colorize columns in output in non-interactive mode') 391 | parser.add_argument('--version', action='store_true', help='print RBQL version and exit') 392 | parser.add_argument('--init-source-file', metavar='FILE', help=argparse.SUPPRESS) # Path to init source file to use instead of ~/.rbql_init_source.py 393 | parser.add_argument('--debug-mode', action='store_true', help=argparse.SUPPRESS) # Run in debug mode 394 | args = parser.parse_args() 395 | 396 | if args.version: 397 | print(_version.__version__) 398 | return 399 | 400 | if args.color and os.name == 'nt': 401 | show_error('generic', '--color option is not supported for Windows terminals', is_interactive=False) 402 | sys.exit(1) 403 | 404 | if args.output is not None and args.color: 405 | show_error('generic', '"--output" is not compatible with "--color" option', is_interactive=False) 406 | sys.exit(1) 407 | 408 | if args.policy == 'monocolumn': 409 | args.delim = '' 410 | 411 | if args.delim is None and args.policy is not None: 412 | show_error('generic', 'Using "--policy" without "--delim" is not allowed', is_interactive=False) 413 | sys.exit(1) 414 | 415 | if args.encoding != 'latin-1' and not PY3: 416 | if args.delim is not None: 417 | args.delim = args.delim.decode(args.encoding) 418 | if args.query is not None: 419 | args.query = args.query.decode(args.encoding) 420 | 421 | is_interactive_mode = args.query is None 422 | if is_interactive_mode: 423 | if args.color: 424 | show_error('generic', '"--color" option is not compatible with interactive mode. Output and Input files preview would be colorized anyway', is_interactive=False) 425 | sys.exit(1) 426 | start_preview_mode_csv(args) 427 | else: 428 | if args.delim is None: 429 | show_error('generic', 'Separator must be provided with "--delim" option in non-interactive mode', is_interactive=False) 430 | sys.exit(1) 431 | if not run_with_python_csv(args, is_interactive=False): 432 | sys.exit(1) 433 | 434 | 435 | sqlite_tool_description = ''' 436 | Run RBQL queries against sqlite databases 437 | Although sqlite database can serve as an input data source, the query engine which will be used is RBQL (not sqlite). 438 | Result set will be written to a csv file. This is also true for UPDATE queries because in RBQL UPDATE is just a special case of SELECT. 439 | 440 | rbql sqlite supports two modes: non-interactive (with "--query" option) and interactive (without "--query" option) 441 | 442 | Interactive mode shows source table preview which makes query editing much easier. 443 | $ rbql sqlite path/to/database.sqlite 444 | 445 | Non-interactive mode supports reading input tables from stdin and writing output to stdout. Usage example: 446 | $ rbql sqlite path/to/database.sqlite --input Employee --query "select top 20 a1, random.random(), a.salary // 1000 order by a.emp_id" 447 | 448 | ''' 449 | 450 | 451 | def sqlite_main(): 452 | parser = argparse.ArgumentParser(prog='rbql sqlite', formatter_class=argparse.RawDescriptionHelpFormatter, description=sqlite_tool_description) 453 | parser.add_argument('database', metavar='PATH', help='PATH to sqlite db') 454 | parser.add_argument('--input', metavar='NAME', help='NAME of the table in sqlite database') 455 | parser.add_argument('--query', help='query string in rbql. Run in interactive mode if empty') 456 | parser.add_argument('--out-format', help='output format', default='csv', choices=['csv', 'tsv']) 457 | parser.add_argument('--output', metavar='FILE', help='write output table to FILE instead of stdout') 458 | parser.add_argument('--color', action='store_true', help='colorize columns in output in non-interactive mode. Do NOT use if redirecting output to a file') 459 | parser.add_argument('--version', action='store_true', help='print RBQL version and exit') 460 | parser.add_argument('--init-source-file', metavar='FILE', help=argparse.SUPPRESS) # Path to init source file to use instead of ~/.rbql_init_source.py 461 | parser.add_argument('--debug-mode', action='store_true', help=argparse.SUPPRESS) # Run in debug mode 462 | args = parser.parse_args() 463 | 464 | if args.version: 465 | print(_version.__version__) 466 | return 467 | 468 | if not os.path.isfile(args.database): 469 | show_error('generic', 'The database does not exist: {}'.format(args.database), is_interactive=False) 470 | sys.exit(1) 471 | 472 | is_interactive_mode = args.query is None 473 | 474 | import sqlite3 475 | if not args.input: 476 | db_connection = sqlite3.connect(args.database) 477 | table_names = read_table_names(db_connection) 478 | db_connection.close() 479 | if len(table_names) == 1: 480 | args.input = table_names[0] 481 | # TODO Consider showing a warning here 482 | elif not is_interactive_mode: 483 | show_error('generic', 'Please provide input table name with --input parameter: source database has more than one table', is_interactive=False) 484 | sys.exit(1) 485 | 486 | if args.output is not None and args.color: 487 | show_error('generic', '"--output" is not compatible with "--color" option', is_interactive=False) 488 | sys.exit(1) 489 | 490 | args.encoding = 'utf-8' 491 | args.output_delim, args.output_policy = (',', 'quoted_rfc') if args.out_format == 'csv' else rbql_csv.interpret_named_csv_format(args.out_format) 492 | 493 | if is_interactive_mode: 494 | if args.color: 495 | show_error('generic', '"--color" option is not compatible with interactive mode. Output and Input files preview would be colorized anyway', is_interactive=False) 496 | sys.exit(1) 497 | start_preview_mode_sqlite(args) 498 | else: 499 | if not run_with_python_sqlite(args, is_interactive=False): 500 | sys.exit(1) 501 | 502 | 503 | def main(): 504 | if len(sys.argv) > 1: 505 | if sys.argv[1] == 'sqlite': 506 | del sys.argv[1] 507 | sqlite_main() 508 | elif sys.argv[1] == 'csv': 509 | del sys.argv[1] 510 | csv_main() 511 | else: 512 | # TODO Consider showing "uknown mode" error if the first argument doesn't start with '--' 513 | csv_main() 514 | else: 515 | csv_main() 516 | 517 | 518 | if __name__ == '__main__': 519 | main() 520 | -------------------------------------------------------------------------------- /rbql/rbql_csv.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | from __future__ import print_function 4 | 5 | import sys 6 | import os 7 | import codecs 8 | import io 9 | from errno import EPIPE 10 | 11 | from . import rbql_engine 12 | from . import csv_utils 13 | 14 | 15 | PY3 = sys.version_info[0] == 3 16 | polymorphic_xrange = range if PY3 else xrange 17 | 18 | default_csv_encoding = 'utf-8' 19 | ansi_reset_color_code = '\u001b[0m' 20 | 21 | debug_mode = False 22 | 23 | try: 24 | broken_pipe_exception = BrokenPipeError 25 | except NameError: # Python 2 26 | broken_pipe_exception = IOError 27 | 28 | 29 | def is_ascii(s): 30 | return all(ord(c) < 128 for c in s) 31 | 32 | 33 | def read_user_init_code(rbql_init_source_path): 34 | with open(rbql_init_source_path) as src: 35 | return src.read() 36 | 37 | 38 | def normalize_delim(delim): 39 | if delim == 'TAB': 40 | return '\t' 41 | if delim == r'\t': 42 | return '\t' 43 | return delim 44 | 45 | 46 | def interpret_named_csv_format(format_name): 47 | format_name = format_name.lower() 48 | if format_name == 'monocolumn': 49 | return ('', 'monocolumn') 50 | if format_name == 'csv': 51 | return (',', 'quoted') 52 | if format_name == 'tsv': 53 | return ('\t', 'simple') 54 | raise RuntimeError('Unknown format name: "{}"'.format(format_name)) 55 | 56 | 57 | 58 | def encode_input_stream(stream, encoding): 59 | if encoding is None: 60 | return stream 61 | if PY3: 62 | # Reference: https://stackoverflow.com/a/16549381/2898283 63 | # typical stream (e.g. sys.stdin) in Python 3 is actually a io.TextIOWrapper but with some unknown encoding 64 | try: 65 | return io.TextIOWrapper(stream.buffer, encoding=encoding) 66 | except AttributeError: 67 | # BytesIO doesn't have "buffer" 68 | return io.TextIOWrapper(stream, encoding=encoding) 69 | else: 70 | # Reference: https://stackoverflow.com/a/27425797/2898283 71 | # Python 2 streams don't have stream.buffer and therefore we can't use io.TextIOWrapper. Instead we use codecs 72 | return codecs.getreader(encoding)(stream) 73 | 74 | 75 | def encode_output_stream(stream, encoding): 76 | if encoding is None: 77 | return stream 78 | if PY3: 79 | try: 80 | return io.TextIOWrapper(stream.buffer, encoding=encoding) 81 | except AttributeError: 82 | # BytesIO doesn't have "buffer" 83 | return io.TextIOWrapper(stream, encoding=encoding) 84 | else: 85 | return codecs.getwriter(encoding)(stream) 86 | 87 | 88 | def remove_utf8_bom(line, assumed_source_encoding): 89 | if assumed_source_encoding == 'latin-1' and len(line) >= 3 and line[:3] == '\xef\xbb\xbf': 90 | return line[3:] 91 | # TODO consider replacing "utf-8" with "utf-8-sig" to automatically remove BOM, see https://stackoverflow.com/a/44573867/2898283 92 | if assumed_source_encoding == 'utf-8' and len(line) >= 1 and line[0] == u'\ufeff': 93 | return line[1:] 94 | return line 95 | 96 | 97 | def try_read_index(index_path): 98 | lines = [] 99 | try: 100 | with open(index_path) as f: 101 | lines = f.readlines() 102 | except Exception: 103 | return [] 104 | result = list() 105 | for line in lines: 106 | line = line.rstrip('\r\n') 107 | record = line.split('\t') 108 | result.append(record) 109 | return result 110 | 111 | 112 | def get_index_record(index_path, key): 113 | records = try_read_index(index_path) 114 | for record in records: 115 | if len(record) and record[0] == key: 116 | return record 117 | return None 118 | 119 | 120 | def find_table_path(main_table_dir, table_id): 121 | # If table_id is a relative path it could be relative either to the current directory or to the main table dir. 122 | candidate_path = os.path.expanduser(table_id) 123 | if os.path.exists(candidate_path): 124 | return candidate_path 125 | if main_table_dir and not os.path.isabs(candidate_path): 126 | candidate_path = os.path.join(main_table_dir, candidate_path) 127 | if os.path.exists(candidate_path): 128 | return candidate_path 129 | user_home_dir = os.path.expanduser('~') 130 | table_names_settings_path = os.path.join(user_home_dir, '.rbql_table_names') 131 | name_record = get_index_record(table_names_settings_path, table_id) 132 | if name_record is not None and len(name_record) > 1 and os.path.exists(name_record[1]): 133 | return name_record[1] 134 | return None 135 | 136 | 137 | def make_inconsistent_num_fields_warning(table_name, inconsistent_records_info): 138 | assert len(inconsistent_records_info) > 1 139 | inconsistent_records_info = inconsistent_records_info.items() 140 | inconsistent_records_info = sorted(inconsistent_records_info, key=lambda v: v[1]) 141 | num_fields_1, record_num_1 = inconsistent_records_info[0] 142 | num_fields_2, record_num_2 = inconsistent_records_info[1] 143 | warn_msg = 'Number of fields in "{}" table is not consistent: '.format(table_name) 144 | warn_msg += 'e.g. record {} -> {} fields, record {} -> {} fields'.format(record_num_1, num_fields_1, record_num_2, num_fields_2) 145 | return warn_msg 146 | 147 | 148 | def init_ansi_terminal_colors(): 149 | result = [ansi_reset_color_code] 150 | foreground_codes = list(range(31, 37 + 1)) 151 | background_codes = list(range(41, 47 + 1)) 152 | for fc in foreground_codes: 153 | result.append('\u001b[{}m'.format(fc)) 154 | for fc in foreground_codes: 155 | for bc in background_codes: 156 | if fc % 10 == bc % 10: 157 | continue 158 | if fc % 10 in [2, 6] and bc % 10 in [2, 6]: # Skipping green - cyan pair cause they might have low contrast 159 | continue 160 | result.append('\u001b[{};{}m'.format(fc, bc)) 161 | return result 162 | 163 | 164 | 165 | class CSVWriter(rbql_engine.RBQLOutputWriter): 166 | def __init__(self, stream, close_stream_on_finish, encoding, delim, policy, line_separator='\n', colorize_output=False): 167 | assert encoding in ['utf-8', 'latin-1', None] 168 | self.stream = encode_output_stream(stream, encoding) 169 | self.line_separator = line_separator 170 | self.delim = delim 171 | self.sub_array_delim = '|' if delim != '|' else ';' 172 | self.broken_pipe = False 173 | self.close_stream_on_finish = close_stream_on_finish 174 | self.polymorphic_preprocess = None 175 | self.polymorphic_join = self.join_by_delim 176 | self.check_separators_after_join = False 177 | self.colors = None 178 | if policy == 'simple' or policy == 'whitespace': 179 | if colorize_output: 180 | self.polymorphic_preprocess = self.check_separators_in_fields_before_join 181 | else: 182 | self.check_separators_after_join = True 183 | elif policy == 'quoted': 184 | self.polymorphic_preprocess = self.quote_fields 185 | elif policy == 'quoted_rfc': 186 | self.polymorphic_preprocess = self.quote_fields_rfc 187 | elif policy == 'monocolumn': 188 | colorize_output = False 189 | self.polymorphic_preprocess = self.ensure_single_field 190 | self.polymorphic_join = self.monocolumn_join 191 | else: 192 | raise RuntimeError('unknown output csv policy') 193 | 194 | if colorize_output: 195 | self.colors = init_ansi_terminal_colors() 196 | 197 | self.none_in_output = False 198 | self.delim_in_simple_output = False 199 | self.header_len = None 200 | 201 | 202 | def set_header(self, header): 203 | if header is not None: 204 | self.header_len = len(header) 205 | self.write(header) 206 | 207 | 208 | def monocolumn_join(self, fields): 209 | return fields[0] 210 | 211 | 212 | def check_separators_in_fields_before_join(self, fields): 213 | if ''.join(fields).find(self.delim) != -1: 214 | self.delim_in_simple_output = True 215 | 216 | 217 | def check_separator_in_fields_after_join(self, output_line, num_fields_expected): 218 | num_fields_calculated = output_line.count(self.delim) + 1 219 | if num_fields_calculated != num_fields_expected: 220 | self.delim_in_simple_output = True 221 | 222 | 223 | def join_by_delim(self, fields): 224 | return self.delim.join(fields) 225 | 226 | 227 | def write(self, fields): 228 | if self.header_len is not None and len(fields) != self.header_len: 229 | raise rbql_engine.RbqlIOHandlingError('Inconsistent number of columns in output header and the current record: {} != {}'.format(self.header_len, len(fields))) 230 | self.normalize_fields(fields) 231 | 232 | if self.polymorphic_preprocess is not None: 233 | self.polymorphic_preprocess(fields) 234 | 235 | if self.colors is not None: 236 | self.colorize_fields(fields) 237 | 238 | out_line = self.polymorphic_join(fields) 239 | 240 | if self.check_separators_after_join: 241 | self.check_separator_in_fields_after_join(out_line, len(fields)) 242 | 243 | try: 244 | self.stream.write(out_line) 245 | if self.colors is not None: 246 | self.stream.write(ansi_reset_color_code) 247 | self.stream.write(self.line_separator) 248 | return True 249 | except broken_pipe_exception as exc: 250 | if broken_pipe_exception == IOError: 251 | if exc.errno != EPIPE: 252 | raise 253 | self.broken_pipe = True 254 | return False 255 | 256 | 257 | def colorize_fields(self, fields): 258 | for i in polymorphic_xrange(len(fields)): 259 | fields[i] = self.colors[i % len(self.colors)] + fields[i] 260 | 261 | 262 | def quote_fields(self, fields): 263 | for i in polymorphic_xrange(len(fields)): 264 | fields[i] = csv_utils.quote_field(fields[i], self.delim) 265 | 266 | 267 | def quote_fields_rfc(self, fields): 268 | for i in polymorphic_xrange(len(fields)): 269 | fields[i] = csv_utils.rfc_quote_field(fields[i], self.delim) 270 | 271 | 272 | def ensure_single_field(self, fields): 273 | if len(fields) > 1: 274 | raise rbql_engine.RbqlIOHandlingError('Unable to use "Monocolumn" output format: some records have more than one field') 275 | 276 | 277 | def normalize_fields(self, fields): 278 | for i in polymorphic_xrange(len(fields)): 279 | if PY3 and isinstance(fields[i], str): 280 | continue 281 | elif not PY3 and isinstance(fields[i], basestring): 282 | continue 283 | elif fields[i] is None: 284 | fields[i] = '' 285 | self.none_in_output = True 286 | elif isinstance(fields[i], list): 287 | self.normalize_fields(fields[i]) 288 | fields[i] = self.sub_array_delim.join(fields[i]) 289 | else: 290 | fields[i] = str(fields[i]) 291 | 292 | 293 | def _write_all(self, table): 294 | for record in table: 295 | self.write(record[:]) 296 | self.finish() 297 | 298 | 299 | def finish(self): 300 | if self.broken_pipe: 301 | return 302 | if self.close_stream_on_finish: 303 | self.stream.close() 304 | else: 305 | try: 306 | self.stream.flush() # This flush still can throw if all flushes before were sucessfull! And the exceptions would be printed anyway, even if it was explicitly catched just couple of lines after. 307 | # Basically this fails if output is small and this is the first flush after the pipe was broken e.g. second flush if piped to head -n 1 308 | # Here head -n 1 finished after the first flush, and the final explict flush here just killing it 309 | except broken_pipe_exception as exc: 310 | if broken_pipe_exception == IOError: 311 | if exc.errno != EPIPE: 312 | raise 313 | # In order to avoid BrokenPipeError from being printed as a warning to stderr, we need to perform this magic below. See: 314 | # Explanation 1: https://stackoverflow.com/a/35761190/2898283 315 | # Explanation 2: https://bugs.python.org/issue11380 316 | try: 317 | sys.stdout.close() 318 | except Exception: 319 | pass 320 | 321 | 322 | def get_warnings(self): 323 | result = list() 324 | if self.none_in_output: 325 | result.append('None values in output were replaced by empty strings') 326 | if self.delim_in_simple_output: 327 | result.append('Some output fields contain separator') 328 | return result 329 | 330 | 331 | class CSVRecordIterator(rbql_engine.RBQLInputIterator): 332 | def __init__(self, stream, encoding, delim, policy, has_header=False, comment_prefix=None, table_name='input', variable_prefix='a', chunk_size=1024, line_mode=False): 333 | assert encoding in ['utf-8', 'latin-1', None] 334 | self.encoding = encoding 335 | self.stream = encode_input_stream(stream, encoding) 336 | self.delim = delim 337 | self.policy = policy 338 | self.table_name = table_name 339 | self.variable_prefix = variable_prefix 340 | self.comment_prefix = comment_prefix if (comment_prefix is not None and len(comment_prefix)) else None 341 | 342 | self.buffer = '' 343 | self.detected_line_separator = '\n' 344 | self.exhausted = False 345 | self.NR = 0 # Record number 346 | self.NL = 0 # Line number (NL != NR when the CSV file has comments or multiline fields) 347 | self.chunk_size = chunk_size 348 | self.fields_info = dict() 349 | 350 | self.utf8_bom_removed = False 351 | self.first_defective_line = None 352 | self.polymorphic_get_row = self.get_row_rfc if policy == 'quoted_rfc' else self.get_row_simple 353 | self.has_header = has_header 354 | self.first_record_should_be_emitted = False 355 | 356 | if not line_mode: 357 | self.first_record = None 358 | self.first_record = self.get_record() 359 | self.first_record_should_be_emitted = not has_header 360 | 361 | 362 | def handle_query_modifier(self, modifier): 363 | # For `... WITH (header) ...` syntax 364 | if modifier in ['header', 'headers']: 365 | self.has_header = True 366 | self.first_record_should_be_emitted = False 367 | if modifier in ['noheader', 'noheaders']: 368 | self.has_header = False 369 | self.first_record_should_be_emitted = True 370 | 371 | 372 | def get_variables_map(self, query_text): 373 | variable_map = dict() 374 | rbql_engine.parse_basic_variables(query_text, self.variable_prefix, variable_map) 375 | rbql_engine.parse_array_variables(query_text, self.variable_prefix, variable_map) 376 | if self.has_header and self.first_record is not None: 377 | rbql_engine.parse_attribute_variables(query_text, self.variable_prefix, self.first_record, 'CSV header line', variable_map) 378 | rbql_engine.parse_dictionary_variables(query_text, self.variable_prefix, self.first_record, variable_map) 379 | return variable_map 380 | 381 | def get_header(self): 382 | return self.first_record if self.has_header else None 383 | 384 | def _get_row_from_buffer(self): 385 | str_before, separator, str_after = csv_utils.extract_line_from_data(self.buffer) 386 | if separator is None: 387 | return None 388 | if separator == '\r' and str_after == '': 389 | one_more = self.stream.read(1) 390 | if one_more == '\n': 391 | separator = '\r\n' 392 | else: 393 | str_after = one_more 394 | self.detected_line_separator = separator 395 | self.buffer = str_after 396 | return str_before 397 | 398 | 399 | def _read_until_found(self): 400 | if self.exhausted: 401 | return 402 | chunks = [] 403 | while True: 404 | chunk = self.stream.read(self.chunk_size) 405 | if not chunk: 406 | self.exhausted = True 407 | break 408 | chunks.append(chunk) 409 | if csv_utils.newline_rgx.search(chunk) is not None: 410 | break 411 | self.buffer += ''.join(chunks) 412 | 413 | 414 | def get_row_simple(self): 415 | try: 416 | row = self._get_row_from_buffer() 417 | if row is None: 418 | self._read_until_found() 419 | row = self._get_row_from_buffer() 420 | if row is None: 421 | assert self.exhausted 422 | if not len(self.buffer): 423 | return None 424 | row = self.buffer 425 | self.buffer = '' 426 | self.NL += 1 427 | if self.NL == 1: 428 | clean_line = remove_utf8_bom(row, self.encoding) 429 | if clean_line != row: 430 | row = clean_line 431 | self.utf8_bom_removed = True 432 | return row 433 | except UnicodeDecodeError: 434 | raise rbql_engine.RbqlIOHandlingError('Unable to decode input table as UTF-8. Use binary (latin-1) encoding instead') 435 | 436 | 437 | def get_row_rfc(self): 438 | first_row = self.get_row_simple() 439 | if first_row is None: 440 | return None 441 | if self.comment_prefix is not None and first_row.startswith(self.comment_prefix): 442 | return first_row 443 | if first_row.count('"') % 2 == 0: 444 | return first_row 445 | rows_buffer = [first_row] 446 | while True: 447 | row = self.get_row_simple() 448 | if row is None: 449 | return '\n'.join(rows_buffer) 450 | rows_buffer.append(row) 451 | if row.count('"') % 2 == 1: 452 | return '\n'.join(rows_buffer) 453 | 454 | 455 | def get_record(self): 456 | if self.first_record_should_be_emitted: 457 | self.first_record_should_be_emitted = False 458 | return self.first_record 459 | while True: 460 | line = self.polymorphic_get_row() 461 | if line is None: 462 | return None 463 | if self.comment_prefix is None or not line.startswith(self.comment_prefix): 464 | break 465 | self.NR += 1 466 | record, warning = csv_utils.smart_split(line, self.delim, self.policy, preserve_quotes_and_whitespaces=False) 467 | if warning: 468 | if self.first_defective_line is None: 469 | self.first_defective_line = self.NL 470 | if self.policy == 'quoted_rfc': 471 | raise rbql_engine.RbqlIOHandlingError('Inconsistent double quote escaping in {} table at record {}, line {}'.format(self.table_name, self.NR, self.NL)) 472 | num_fields = len(record) 473 | if num_fields not in self.fields_info: 474 | self.fields_info[num_fields] = self.NR 475 | return record 476 | 477 | 478 | def _get_all_rows(self): 479 | result = [] 480 | while True: 481 | row = self.polymorphic_get_row() 482 | if row is None: 483 | break 484 | result.append(row) 485 | return result 486 | 487 | 488 | def get_all_records(self, num_rows=None): 489 | result = [] 490 | while True: 491 | record = self.get_record() 492 | if record is None: 493 | break 494 | result.append(record) 495 | if num_rows is not None and len(result) >= num_rows: 496 | break 497 | return result 498 | 499 | 500 | def get_warnings(self): 501 | result = list() 502 | if self.utf8_bom_removed: 503 | result.append('UTF-8 Byte Order Mark (BOM) was found and skipped in {} table'.format(self.table_name)) 504 | if self.first_defective_line is not None: 505 | result.append('Inconsistent double quote escaping in {} table. E.g. at line {}'.format(self.table_name, self.first_defective_line)) 506 | if len(self.fields_info) > 1: 507 | result.append(make_inconsistent_num_fields_warning(self.table_name, self.fields_info)) 508 | return result 509 | 510 | 511 | class FileSystemCSVRegistry(rbql_engine.RBQLTableRegistry): 512 | def __init__(self, input_file_dir, delim, policy, encoding, has_header, comment_prefix): 513 | self.input_file_dir = input_file_dir 514 | self.delim = delim 515 | self.policy = policy 516 | self.encoding = encoding 517 | self.record_iterator = None 518 | self.input_stream = None 519 | self.has_header = has_header 520 | self.comment_prefix = comment_prefix 521 | self.table_path = None 522 | 523 | def get_iterator_by_table_id(self, table_id, single_char_alias): 524 | self.table_path = find_table_path(self.input_file_dir, table_id) 525 | if self.table_path is None: 526 | raise rbql_engine.RbqlIOHandlingError('Unable to find join table "{}"'.format(table_id)) 527 | self.input_stream = open(self.table_path, 'rb') 528 | self.record_iterator = CSVRecordIterator(self.input_stream, self.encoding, self.delim, self.policy, self.has_header, comment_prefix=self.comment_prefix, table_name=table_id, variable_prefix=single_char_alias) 529 | return self.record_iterator 530 | 531 | def finish(self): 532 | if self.input_stream is not None: 533 | self.input_stream.close() 534 | 535 | def get_warnings(self): 536 | result = [] 537 | if self.record_iterator is not None and self.has_header: 538 | result.append('The first record in JOIN file {} was also treated as header (and skipped)'.format(os.path.basename(self.table_path))) # UT JSON CSV 539 | return result 540 | 541 | 542 | def query_csv(query_text, input_path, input_delim, input_policy, output_path, output_delim, output_policy, csv_encoding, output_warnings, with_headers, comment_prefix=None, user_init_code='', colorize_output=False): 543 | output_stream, close_output_on_finish = (None, False) 544 | input_stream, close_input_on_finish = (None, False) 545 | join_tables_registry = None 546 | try: 547 | output_stream, close_output_on_finish = (sys.stdout, False) if output_path is None else (open(output_path, 'wb'), True) 548 | input_stream, close_input_on_finish = (sys.stdin, False) if input_path is None else (open(input_path, 'rb'), True) 549 | 550 | if input_delim == '"' and input_policy == 'quoted': 551 | raise rbql_engine.RbqlIOHandlingError('Double quote delimiter is incompatible with "quoted" policy') 552 | if input_delim != ' ' and input_policy == 'whitespace': 553 | raise rbql_engine.RbqlIOHandlingError('Only whitespace " " delim is supported with "whitespace" policy') 554 | 555 | if not is_ascii(query_text) and csv_encoding == 'latin-1': 556 | raise rbql_engine.RbqlIOHandlingError('To use non-ascii characters in query enable UTF-8 encoding instead of latin-1/binary') 557 | 558 | if (not is_ascii(input_delim) or not is_ascii(output_delim)) and csv_encoding == 'latin-1': 559 | raise rbql_engine.RbqlIOHandlingError('To use non-ascii separators enable UTF-8 encoding instead of latin-1/binary') 560 | 561 | default_init_source_path = os.path.join(os.path.expanduser('~'), '.rbql_init_source.py') 562 | if user_init_code == '' and os.path.exists(default_init_source_path): 563 | user_init_code = read_user_init_code(default_init_source_path) 564 | 565 | input_file_dir = None if not input_path else os.path.dirname(input_path) 566 | join_tables_registry = FileSystemCSVRegistry(input_file_dir, input_delim, input_policy, csv_encoding, with_headers, comment_prefix) 567 | input_iterator = CSVRecordIterator(input_stream, csv_encoding, input_delim, input_policy, with_headers, comment_prefix=comment_prefix) 568 | output_writer = CSVWriter(output_stream, close_output_on_finish, csv_encoding, output_delim, output_policy, colorize_output=colorize_output) 569 | if debug_mode: 570 | rbql_engine.set_debug_mode() 571 | rbql_engine.query(query_text, input_iterator, output_writer, output_warnings, join_tables_registry, user_init_code) 572 | finally: 573 | if close_input_on_finish: 574 | input_stream.close() 575 | if close_output_on_finish: 576 | output_stream.close() 577 | if join_tables_registry: 578 | join_tables_registry.finish() 579 | output_warnings += join_tables_registry.get_warnings() 580 | 581 | 582 | def set_debug_mode(): 583 | global debug_mode 584 | debug_mode = True 585 | --------------------------------------------------------------------------------