├── test_data ├── 1.json ├── 2.json ├── a.json ├── b.json ├── empty.json ├── null.json ├── s1.json ├── s2.json ├── s3.json ├── m1.json ├── m2.json ├── m1-m2-diff.json ├── tests.json └── json-diff-tests.json ├── debian ├── source │ └── format ├── README.Debian ├── README.source ├── rules ├── control └── copyright ├── .pep8 ├── src ├── json-objs-to-table ├── python-to-json-ast ├── json-table-to-objs ├── json-format ├── xml-to-json ├── env-to-json ├── binary-to-json ├── yaml-to-json ├── csv-to-json ├── json-to-yaml ├── pjito ├── toml-to-json ├── generate-test-results ├── json-to-csv ├── dsv-to-json ├── json-to-env ├── json-make-schema ├── json-to-dsv ├── json-to-plot ├── diff-to-json ├── Makefile ├── generate-json-diff-results ├── json-to-logfmt.go ├── json-run ├── json-to-binary ├── html-to-json ├── logfmt-to-json.go ├── json-to-xml ├── json-diff.go └── json-sql ├── .gitignore ├── go.mod ├── run-json-diff-tests ├── go.sum ├── run-bash-linter ├── requirements.txt ├── run-linter ├── run-all-tests ├── run-formatter ├── verify-ppa-install.sh ├── VERSIONING.md ├── CLAUDE.md ├── check-ppa-build.sh ├── Makefile ├── RELEASE.md ├── README.md ├── release.sh ├── LICENSE └── .pylintrc /test_data/1.json: -------------------------------------------------------------------------------- 1 | 1 2 | -------------------------------------------------------------------------------- /test_data/2.json: -------------------------------------------------------------------------------- 1 | 2 2 | -------------------------------------------------------------------------------- /test_data/a.json: -------------------------------------------------------------------------------- 1 | "a" 2 | -------------------------------------------------------------------------------- /test_data/b.json: -------------------------------------------------------------------------------- 1 | "b" 2 | -------------------------------------------------------------------------------- /test_data/empty.json: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /test_data/null.json: -------------------------------------------------------------------------------- 1 | null 2 | -------------------------------------------------------------------------------- /test_data/s1.json: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /test_data/s2.json: -------------------------------------------------------------------------------- 1 | [1] 2 | -------------------------------------------------------------------------------- /test_data/s3.json: -------------------------------------------------------------------------------- 1 | [2] 2 | -------------------------------------------------------------------------------- /test_data/m1.json: -------------------------------------------------------------------------------- 1 | {"a": 1} 2 | -------------------------------------------------------------------------------- /debian/source/format: -------------------------------------------------------------------------------- 1 | 3.0 (quilt) 2 | -------------------------------------------------------------------------------- /test_data/m2.json: -------------------------------------------------------------------------------- 1 | {"a": 2, "b": 1} 2 | -------------------------------------------------------------------------------- /.pep8: -------------------------------------------------------------------------------- 1 | [pycodestyle] 2 | max_line_length = 120 3 | -------------------------------------------------------------------------------- /src/json-objs-to-table: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | jq '[(.[0] | keys)] + map(to_entries | map(.value))' 3 | -------------------------------------------------------------------------------- /test_data/m1-m2-diff.json: -------------------------------------------------------------------------------- 1 | [{"leftValue":1,"path":["a"],"rightValue":2},{"path":["b"],"rightValue":1}] 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | json-diff 2 | *.swp 3 | json-toolkit* 4 | target/ 5 | logs/ 6 | *.log 7 | debian/changelog 8 | CHANGELOG.md 9 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tyleradams/json-toolkit 2 | 3 | go 1.18 4 | 5 | require github.com/go-logfmt/logfmt v0.6.0 6 | -------------------------------------------------------------------------------- /run-json-diff-tests: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | cat test_data/json-diff-tests.json | ./src/generate-json-diff-results | jq -e '. == []' > /dev/null 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= 2 | github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= 3 | -------------------------------------------------------------------------------- /run-bash-linter: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | shellcheck -e SC2002 \ 4 | json-format \ 5 | json-to-plot \ 6 | run-all-tests \ 7 | run-formatter \ 8 | run-json-diff-tests \ 9 | run-linter 10 | -------------------------------------------------------------------------------- /src/python-to-json-ast: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import ast 4 | import json 5 | import sys 6 | 7 | import ast2json 8 | 9 | print(json.dumps(ast2json.ast2json(ast.parse(sys.stdin.read())))) 10 | -------------------------------------------------------------------------------- /debian/README.Debian: -------------------------------------------------------------------------------- 1 | json-toolkit for Debian 2 | ---------------------- 3 | 4 | 5 | 6 | -- Tyler Adams Mon, 10 Apr 2023 20:26:11 -0400 7 | -------------------------------------------------------------------------------- /debian/README.source: -------------------------------------------------------------------------------- 1 | json-toolkit for Debian 2 | ---------------------- 3 | 4 | Source can be found at https://github.com/tyleradams/json-toolkit 5 | 6 | 7 | -- Tyler Adams Mon, 10 Apr 2023 20:26:11 -0400 8 | 9 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | ast2json==0.2.1 2 | click==8.0.0 3 | cx_Oracle==8.1.0 4 | genson==1.2.2 5 | psycopg2-binary==2.8.6 6 | PyMySQL==0.10.1 7 | python-dotenv==0.21.0 8 | PyYAML==5.4 9 | toml==0.10.2 10 | unidiff==0.6.0 11 | xmltodict==0.12.0 12 | -------------------------------------------------------------------------------- /src/json-table-to-objs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import sys 5 | 6 | def f(data): 7 | keys = data[0] 8 | values = data[1:] 9 | return [{keys[i]:v[i] for i in range(len(v))} for v in values] 10 | 11 | print(json.dumps(f(json.load(sys.stdin)))) 12 | 13 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | 3 | %: 4 | dh $@ 5 | DESTDIR=debian/json-toolkit 6 | export DESTDIR 7 | prefix=/usr 8 | export prefix 9 | override_dh_strip: 10 | echo don\'t strip 11 | override_dh_auto_build: 12 | echo don\'t build yet 13 | override_dh_auto_install: 14 | make install 15 | -------------------------------------------------------------------------------- /run-linter: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | pylint \ 4 | csv-to-json \ 5 | dsv-to-json \ 6 | generate-json-diff-results \ 7 | generate-test-results \ 8 | json-sql \ 9 | json-to-csv \ 10 | json-to-dsv \ 11 | json-to-logfmt \ 12 | json-to-xml \ 13 | json-to-yaml \ 14 | logfmt-to-json \ 15 | python-to-json-ast \ 16 | xml-to-json \ 17 | yaml-to-json \ 18 | 2>&1 19 | 20 | go vet 21 | 22 | ./run-bash-linter 23 | -------------------------------------------------------------------------------- /run-all-tests: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | # As the rest of the test program uses json diff, we cannot test json diff using json diff. 6 | ./run-json-diff-tests 7 | 8 | TEST_FILE=test_data/tests.json 9 | TMP_FILE=$(mktemp) 10 | cat "${TEST_FILE}" | ./src/generate-test-results > "${TMP_FILE}" 11 | ./target/json-diff "${TMP_FILE}" "${TEST_FILE}" | jq -e '. == []' > /dev/null 12 | rm "${TMP_FILE}" 13 | -------------------------------------------------------------------------------- /run-formatter: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | go fmt json-diff.go & 4 | autopep8 -i csv-to-json & 5 | autopep8 -i dsv-to-json & 6 | autopep8 -i generate-json-diff-results & 7 | autopep8 -i generate-test-results & 8 | autopep8 -i json-make-schema & 9 | autopep8 -i json-sql & 10 | autopep8 -i json-to-csv & 11 | autopep8 -i json-to-dsv & 12 | autopep8 -i json-to-logfmt & 13 | autopep8 -i json-to-xml & 14 | autopep8 -i json-to-yaml & 15 | autopep8 -i logfmt-to-json & 16 | autopep8 -i python-to-json-ast & 17 | autopep8 -i xml-to-json & 18 | autopep8 -i yaml-to-json & 19 | wait 20 | -------------------------------------------------------------------------------- /src/json-format: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eou pipefail 4 | 5 | if [[ "$#" -gt "1" ]]; then 6 | echo "Usage: $0 [FILE]" 7 | exit 1 8 | fi 9 | 10 | if [[ "$#" == "0" ]]; then 11 | jq -S 12 | exit 0 13 | fi 14 | 15 | FILE="$1" 16 | if [[ "$FILE" == "-" ]]; then 17 | jq -S 18 | elif [[ ! -f "$FILE" ]]; then 19 | echo "$FILE does not exist" 20 | exit 1 21 | elif ! jq '.' "$FILE" > /dev/null; then 22 | echo "json is invalid, aborting" 23 | exit 1 24 | elif [[ -z "$(jq '.' "$FILE")" ]]; then 25 | echo "jq output returned nothing, aborting" 26 | exit 1 27 | else 28 | TMP="$(mktemp)" 29 | jq -S '.' "$FILE" > "$TMP" 30 | mv "$TMP" "$FILE" 31 | fi 32 | -------------------------------------------------------------------------------- /src/xml-to-json: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import sys 5 | import click 6 | import xmltodict 7 | 8 | CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} 9 | 10 | @click.command(context_settings=CONTEXT_SETTINGS) 11 | @click.version_option("1.1.0", prog_name="xml-to-json") 12 | def main(): 13 | """ 14 | Convert XML data from stdin to JSON. 15 | 16 | Reads XML data from standard input and outputs JSON representation 17 | of the XML structure. 18 | 19 | \b 20 | Examples: 21 | echo 'value' | xml-to-json 22 | xml-to-json < data.xml > data.json 23 | cat config.xml | xml-to-json | jq '.' 24 | """ 25 | try: 26 | xml_string = sys.stdin.read() 27 | data = xmltodict.parse(xml_string) 28 | print(json.dumps(data)) 29 | sys.exit(0) 30 | except Exception as e: 31 | click.echo(f"Error: {e}", err=True) 32 | sys.exit(1) 33 | 34 | if __name__ == "__main__": 35 | main() 36 | -------------------------------------------------------------------------------- /src/env-to-json: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import sys 5 | import click 6 | import dotenv 7 | 8 | CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} 9 | 10 | @click.command(context_settings=CONTEXT_SETTINGS) 11 | @click.version_option("1.1.0", prog_name="env-to-json") 12 | def main(): 13 | """ 14 | Convert environment variable file from stdin to JSON. 15 | 16 | Reads .env format data from standard input and outputs a JSON object 17 | with key-value pairs. 18 | 19 | \b 20 | Examples: 21 | echo "FOO=bar" | env-to-json 22 | env-to-json < .env > config.json 23 | cat variables.env | env-to-json | jq '.' 24 | """ 25 | try: 26 | d = dotenv.dotenv_values(stream=sys.stdin) 27 | print(json.dumps(d)) 28 | sys.exit(0) 29 | except BrokenPipeError: 30 | sys.stderr.close() 31 | sys.exit(0) 32 | except Exception as e: 33 | click.echo(f"Error: {e}", err=True) 34 | sys.exit(1) 35 | 36 | if __name__ == "__main__": 37 | main() 38 | -------------------------------------------------------------------------------- /src/binary-to-json: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import sys 5 | import click 6 | 7 | CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} 8 | 9 | @click.command(context_settings=CONTEXT_SETTINGS) 10 | @click.version_option("1.1.0", prog_name="binary-to-json") 11 | def main(): 12 | """ 13 | Convert binary data from stdin to JSON array format. 14 | 15 | Reads binary data from standard input and outputs a JSON array where 16 | each element is a byte value (0-255). 17 | 18 | \b 19 | Examples: 20 | echo "hello" | binary-to-json 21 | binary-to-json < file.bin > file.json 22 | cat image.png | binary-to-json | jq '.' 23 | """ 24 | try: 25 | data = list(sys.stdin.buffer.read()) 26 | print(json.dumps(data)) 27 | sys.exit(0) 28 | except BrokenPipeError: 29 | sys.stderr.close() 30 | sys.exit(0) 31 | except Exception as e: 32 | click.echo(f"Error: {e}", err=True) 33 | sys.exit(1) 34 | 35 | if __name__ == "__main__": 36 | main() 37 | -------------------------------------------------------------------------------- /src/yaml-to-json: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import sys 5 | import click 6 | import yaml 7 | 8 | CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} 9 | 10 | @click.command(context_settings=CONTEXT_SETTINGS) 11 | @click.version_option("1.1.0", prog_name="yaml-to-json") 12 | def main(): 13 | """ 14 | Convert YAML data from stdin to JSON. 15 | 16 | Reads YAML data from standard input and outputs JSON. 17 | 18 | \b 19 | Examples: 20 | echo "a: 1" | yaml-to-json 21 | yaml-to-json < config.yaml > config.json 22 | cat data.yaml | yaml-to-json | jq '.' 23 | """ 24 | try: 25 | data = yaml.load(sys.stdin, Loader=yaml.SafeLoader) 26 | print(json.dumps(data)) 27 | sys.exit(0) 28 | except yaml.YAMLError as e: 29 | click.echo(f"Error: Invalid YAML: {e}", err=True) 30 | sys.exit(1) 31 | except BrokenPipeError: 32 | sys.stderr.close() 33 | sys.exit(0) 34 | except Exception as e: 35 | click.echo(f"Error: {e}", err=True) 36 | sys.exit(1) 37 | 38 | 39 | if __name__ == "__main__": 40 | main() 41 | -------------------------------------------------------------------------------- /src/csv-to-json: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import csv 4 | import json 5 | import sys 6 | import click 7 | 8 | CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} 9 | 10 | @click.command(context_settings=CONTEXT_SETTINGS) 11 | @click.version_option("1.1.0", prog_name="csv-to-json") 12 | def main(): 13 | """ 14 | Convert CSV data from stdin to JSON array format. 15 | 16 | Reads CSV data from standard input and outputs a JSON array of arrays. 17 | Each row in the CSV becomes an array in the output. 18 | 19 | \b 20 | Examples: 21 | echo "a,b,c" | csv-to-json 22 | csv-to-json < data.csv > data.json 23 | cat file.csv | csv-to-json | jq '.' 24 | """ 25 | try: 26 | csv_reader = csv.reader(sys.stdin) 27 | print(json.dumps(list(csv_reader))) 28 | sys.exit(0) 29 | except BrokenPipeError: 30 | # Handle pipe being closed (e.g., when piped to head) 31 | sys.stderr.close() 32 | sys.exit(0) 33 | except Exception as e: 34 | click.echo(f"Error: {e}", err=True) 35 | sys.exit(1) 36 | 37 | 38 | if __name__ == "__main__": 39 | main() 40 | -------------------------------------------------------------------------------- /src/json-to-yaml: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import sys 5 | import click 6 | import yaml 7 | 8 | CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} 9 | 10 | @click.command(context_settings=CONTEXT_SETTINGS) 11 | @click.version_option("1.1.0", prog_name="json-to-yaml") 12 | def main(): 13 | """ 14 | Convert JSON data from stdin to YAML. 15 | 16 | Reads JSON data from standard input and outputs YAML. 17 | 18 | \b 19 | Examples: 20 | echo '{"a": 1}' | json-to-yaml 21 | json-to-yaml < config.json > config.yaml 22 | cat data.json | json-to-yaml 23 | """ 24 | try: 25 | data = json.load(sys.stdin) 26 | yaml_s = yaml.dump(data, default_flow_style=False) 27 | print(yaml_s, end='') 28 | sys.exit(0) 29 | except json.JSONDecodeError as e: 30 | click.echo(f"Error: Invalid JSON: {e}", err=True) 31 | sys.exit(1) 32 | except BrokenPipeError: 33 | sys.stderr.close() 34 | sys.exit(0) 35 | except Exception as e: 36 | click.echo(f"Error: {e}", err=True) 37 | sys.exit(1) 38 | 39 | 40 | if __name__ == "__main__": 41 | main() 42 | -------------------------------------------------------------------------------- /src/pjito: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import click 5 | 6 | CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} 7 | 8 | PROGRAM = """\ 9 | #!/usr/bin/env python3 10 | 11 | import json 12 | import sys 13 | 14 | def t(data): 15 | return data 16 | 17 | print(json.dumps(t(json.load(sys.stdin)))) 18 | """ 19 | 20 | @click.command(context_settings=CONTEXT_SETTINGS) 21 | @click.version_option("1.1.0", prog_name="pjito") 22 | def main(): 23 | """ 24 | Generate a Python JSON "Input-Transform-Output" (PJITO) template. 25 | 26 | Outputs a Python script template that reads JSON from stdin, transforms 27 | it with a function, and outputs JSON. You can edit the transform function 28 | to process JSON data. 29 | 30 | \b 31 | Examples: 32 | pjito > fix-data && chmod +x fix-data 33 | pjito > transform.py 34 | """ 35 | try: 36 | print(PROGRAM, end="") 37 | sys.exit(0) 38 | except BrokenPipeError: 39 | sys.stderr.close() 40 | sys.exit(0) 41 | except Exception as e: 42 | click.echo(f"Error: {e}", err=True) 43 | sys.exit(1) 44 | 45 | if __name__ == "__main__": 46 | main() 47 | -------------------------------------------------------------------------------- /src/toml-to-json: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import json 5 | import click 6 | import toml 7 | 8 | CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} 9 | 10 | @click.command(context_settings=CONTEXT_SETTINGS) 11 | @click.version_option("1.1.0", prog_name="toml-to-json") 12 | def main(): 13 | """ 14 | Convert TOML data from stdin to JSON. 15 | 16 | Reads TOML data from standard input and outputs JSON. 17 | 18 | \b 19 | Examples: 20 | echo "key = 'value'" | toml-to-json 21 | toml-to-json < config.toml > config.json 22 | cat data.toml | toml-to-json | jq '.' 23 | """ 24 | try: 25 | toml_data = toml.loads(sys.stdin.read()) 26 | json.dump(toml_data, sys.stdout, indent=4) 27 | print() # Add newline at end 28 | sys.exit(0) 29 | except toml.TomlDecodeError as e: 30 | click.echo(f"Error: Invalid TOML: {e}", err=True) 31 | sys.exit(1) 32 | except BrokenPipeError: 33 | sys.stderr.close() 34 | sys.exit(0) 35 | except Exception as e: 36 | click.echo(f"Error: {e}", err=True) 37 | sys.exit(1) 38 | 39 | if __name__ == "__main__": 40 | main() 41 | -------------------------------------------------------------------------------- /src/generate-test-results: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import subprocess 5 | import sys 6 | 7 | 8 | def flatten(ll): 9 | a = [] 10 | for l in ll: 11 | a.extend(l) 12 | return a 13 | 14 | 15 | def run_command(command, stdin): 16 | p = subprocess.Popen(command, stdin=subprocess.PIPE, 17 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) 18 | stdout, stderr = p.communicate(stdin.encode("ASCII")) 19 | # Combine stderr and stdout (stderr first, as it typically contains error messages) 20 | output = stderr.decode("ASCII") + stdout.decode("ASCII") 21 | return output, p.returncode 22 | 23 | 24 | def run_test(test): 25 | actual_output, actual_return_code = run_command( 26 | test["command"], test["input"]) 27 | return [{ 28 | "command": test["command"], 29 | "input": test["input"], 30 | "output": actual_output, 31 | "returnCode": actual_return_code, 32 | }] 33 | 34 | 35 | def main(): 36 | tests = json.load(sys.stdin) 37 | results = flatten(map(run_test, tests)) 38 | print(json.dumps(results)) 39 | sys.exit(0) 40 | 41 | 42 | if __name__ == "__main__": 43 | main() 44 | -------------------------------------------------------------------------------- /src/json-to-csv: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import csv 4 | import json 5 | import sys 6 | import click 7 | 8 | CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} 9 | 10 | @click.command(context_settings=CONTEXT_SETTINGS) 11 | @click.version_option("1.1.0", prog_name="json-to-csv") 12 | def main(): 13 | """ 14 | Convert JSON array to CSV format. 15 | 16 | Reads a JSON array of arrays from stdin and outputs CSV data. 17 | Each inner array becomes a row in the CSV output. 18 | 19 | \b 20 | Examples: 21 | echo '[["a","b"],["c","d"]]' | json-to-csv 22 | json-to-csv < data.json > data.csv 23 | cat table.json | json-to-csv 24 | """ 25 | try: 26 | rows = json.load(sys.stdin) 27 | csv_writer = csv.writer(sys.stdout, dialect='unix') 28 | csv_writer.writerows(rows) 29 | sys.exit(0) 30 | except json.JSONDecodeError as e: 31 | click.echo(f"Error: Invalid JSON: {e}", err=True) 32 | sys.exit(1) 33 | except BrokenPipeError: 34 | sys.stderr.close() 35 | sys.exit(0) 36 | except Exception as e: 37 | click.echo(f"Error: {e}", err=True) 38 | sys.exit(1) 39 | 40 | if __name__ == "__main__": 41 | main() 42 | -------------------------------------------------------------------------------- /src/dsv-to-json: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import sys 5 | import click 6 | 7 | CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} 8 | 9 | @click.command(context_settings=CONTEXT_SETTINGS) 10 | @click.version_option("1.1.0", prog_name="dsv-to-json") 11 | @click.argument('delimiter') 12 | def main(delimiter): 13 | """ 14 | Convert delimiter-separated values from stdin to JSON. 15 | 16 | Reads DSV data from standard input and outputs JSON array of arrays. 17 | 18 | \b 19 | DELIMITER: The delimiter character to use (use -z for null byte) 20 | 21 | \b 22 | Examples: 23 | echo "a:b:c" | dsv-to-json ":" 24 | dsv-to-json ";" < data.dsv > data.json 25 | cat file.tsv | dsv-to-json $'\\t' | jq '.' 26 | """ 27 | try: 28 | if delimiter == "-z": 29 | delimiter = chr(0) 30 | lines = sys.stdin.readlines() 31 | cells = [l.strip("\n").split(delimiter) for l in lines] 32 | print(json.dumps(cells)) 33 | sys.exit(0) 34 | except BrokenPipeError: 35 | sys.stderr.close() 36 | sys.exit(0) 37 | except Exception as e: 38 | click.echo(f"Error: {e}", err=True) 39 | sys.exit(1) 40 | 41 | 42 | if __name__ == "__main__": 43 | main() 44 | -------------------------------------------------------------------------------- /src/json-to-env: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import sys 5 | import click 6 | 7 | CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} 8 | 9 | def escape_value(v): 10 | return v.replace("\n","\\n") 11 | 12 | @click.command(context_settings=CONTEXT_SETTINGS) 13 | @click.version_option("1.1.0", prog_name="json-to-env") 14 | def main(): 15 | """ 16 | Convert JSON object to environment variable format. 17 | 18 | Reads a JSON object from stdin and outputs .env format data with 19 | key-value pairs. Newlines in values are escaped. 20 | 21 | \b 22 | Examples: 23 | echo '{"FOO":"bar"}' | json-to-env 24 | json-to-env < config.json > .env 25 | cat data.json | json-to-env 26 | """ 27 | try: 28 | d = json.load(sys.stdin) 29 | for k, v in d.items(): 30 | v2 = escape_value(v) 31 | print(f"{k}=\"{v2}\"") 32 | sys.exit(0) 33 | except json.JSONDecodeError as e: 34 | click.echo(f"Error: Invalid JSON: {e}", err=True) 35 | sys.exit(1) 36 | except BrokenPipeError: 37 | sys.stderr.close() 38 | sys.exit(0) 39 | except Exception as e: 40 | click.echo(f"Error: {e}", err=True) 41 | sys.exit(1) 42 | 43 | if __name__ == "__main__": 44 | main() 45 | -------------------------------------------------------------------------------- /src/json-make-schema: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import sys 5 | import click 6 | import genson 7 | 8 | CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} 9 | 10 | @click.command(context_settings=CONTEXT_SETTINGS) 11 | @click.version_option("1.1.0", prog_name="json-make-schema") 12 | def main(): 13 | """ 14 | Generate JSON Schema from JSON data. 15 | 16 | Reads JSON data from standard input and generates a JSON Schema 17 | that describes the structure of the input data. 18 | 19 | \b 20 | Examples: 21 | echo '{"name": "John", "age": 30}' | json-make-schema 22 | json-make-schema < example.json > schema.json 23 | cat data.json | json-make-schema | jq '.' 24 | """ 25 | try: 26 | data = json.load(sys.stdin) 27 | builder = genson.SchemaBuilder() 28 | builder.add_object(data) 29 | schema = builder.to_schema() 30 | print(json.dumps(schema)) 31 | sys.exit(0) 32 | except json.JSONDecodeError as e: 33 | click.echo(f"Error: Invalid JSON input: {e}", err=True) 34 | sys.exit(1) 35 | except BrokenPipeError: 36 | sys.stderr.close() 37 | sys.exit(0) 38 | except Exception as e: 39 | click.echo(f"Error: {e}", err=True) 40 | sys.exit(1) 41 | 42 | 43 | if __name__ == "__main__": 44 | main() 45 | -------------------------------------------------------------------------------- /src/json-to-dsv: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import sys 5 | import click 6 | 7 | CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} 8 | 9 | @click.command(context_settings=CONTEXT_SETTINGS) 10 | @click.version_option("1.1.0", prog_name="json-to-dsv") 11 | @click.argument('delimiter') 12 | def main(delimiter): 13 | """ 14 | Convert JSON array to delimiter-separated values. 15 | 16 | Reads a JSON array of arrays from stdin and outputs DSV data using 17 | the specified delimiter. Each inner array becomes a row. 18 | 19 | \b 20 | DELIMITER: The delimiter character to use 21 | 22 | \b 23 | Examples: 24 | echo '[["a","b"],["c","d"]]' | json-to-dsv ":" 25 | json-to-dsv ";" < data.json > data.dsv 26 | cat table.json | json-to-dsv $'\\t' > data.tsv 27 | """ 28 | try: 29 | rows = json.load(sys.stdin) 30 | for row in rows: 31 | print(delimiter.join([str(e) for e in row])) 32 | sys.exit(0) 33 | except json.JSONDecodeError as e: 34 | click.echo(f"Error: Invalid JSON: {e}", err=True) 35 | sys.exit(1) 36 | except BrokenPipeError: 37 | sys.stderr.close() 38 | sys.exit(0) 39 | except Exception as e: 40 | click.echo(f"Error: {e}", err=True) 41 | sys.exit(1) 42 | 43 | if __name__ == "__main__": 44 | main() 45 | -------------------------------------------------------------------------------- /src/json-to-plot: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | DATA="$(cat | jq '.')" 4 | TITLE="$(echo "$DATA" | jq '.Title')" 5 | XLABEL="$(echo "$DATA" | jq '.XLabel')" 6 | YLABEL="$(echo "$DATA" | jq '.YLabel')" 7 | OUTFILE="$1" 8 | OUTFILE_EXTENSION="$(echo "$OUTFILE" | cut -d '.' -f 2-)" 9 | 10 | if [ -z "$DATA" ]; then 11 | echo "stdin must be valid json" 12 | exit 1 13 | fi 14 | 15 | if [ -z "$TITLE" ]; then 16 | echo ".Title must be set" 17 | exit 1 18 | fi 19 | 20 | if [ -z "$XLABEL" ]; then 21 | echo ".XLabel must be set" 22 | exit 1 23 | fi 24 | 25 | if [ -z "$YLABEL" ]; then 26 | echo ".YLabel must be set" 27 | exit 1 28 | fi 29 | 30 | if [ -z "$OUTFILE" ]; then 31 | echo "OUTFILE must be specified as argv[1]" 32 | exit 1 33 | fi 34 | 35 | if [ "$OUTFILE_EXTENSION" != "png" ]; then 36 | echo "OUTFILE must be a png" 37 | exit 1 38 | fi 39 | 40 | echo "$DATA" | jq '.data' | json-to-dsv '#' | gnuplot \ 41 | -e "set terminal png enhanced font 'verdana,8' size 1920,1080" \ 42 | -e "set title '$TITLE'" \ 43 | -e "set xlabel '$XLABEL'" \ 44 | -e "set ylabel '$YLABEL'" \ 45 | -e "set output '$OUTFILE'" \ 46 | -e "set datafile separator '#'" \ 47 | -e "set nokey" \ 48 | -e "set border linewidth 1.5" \ 49 | -e "set style line 1 lc rgb '#0060ad' pt 7 ps .2 lt 1 lw 2 # --- blue" \ 50 | -e "plot '< cat -' using 2:xticlabels(1) with lines ls 1" 51 | -------------------------------------------------------------------------------- /src/diff-to-json: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import copy 4 | import json 5 | import sys 6 | import click 7 | import unidiff 8 | 9 | CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} 10 | 11 | def hunk_to_d(h): 12 | d = copy.deepcopy(h.__dict__) 13 | d["source"] = h.source 14 | d["target"] = h.target 15 | return d 16 | 17 | def patch_to_d(p): 18 | d = copy.deepcopy(p.__dict__) 19 | d["hunks"] = [hunk_to_d(h) for h in p] 20 | return d 21 | 22 | @click.command(context_settings=CONTEXT_SETTINGS) 23 | @click.version_option("1.1.0", prog_name="diff-to-json") 24 | def main(): 25 | """ 26 | Convert unified diff format from stdin to JSON. 27 | 28 | Reads unified diff data (patch format) from standard input and outputs 29 | a structured JSON representation of the patches and hunks. 30 | 31 | \b 32 | Examples: 33 | git diff | diff-to-json 34 | diff -u old.txt new.txt | diff-to-json 35 | diff-to-json < changes.patch | jq '.' 36 | """ 37 | try: 38 | patch_set = unidiff.PatchSet(sys.stdin.read()) 39 | o = [patch_to_d(p) for p in patch_set] 40 | print(json.dumps(o)) 41 | sys.exit(0) 42 | except BrokenPipeError: 43 | sys.stderr.close() 44 | sys.exit(0) 45 | except Exception as e: 46 | click.echo(f"Error: {e}", err=True) 47 | sys.exit(1) 48 | 49 | if __name__ == "__main__": 50 | main() 51 | -------------------------------------------------------------------------------- /src/Makefile: -------------------------------------------------------------------------------- 1 | SHELL = /bin/sh 2 | 3 | # Tools and flags (user-overridable) 4 | GO ?= go 5 | INSTALL ?= install 6 | INSTALL_PROGRAM ?= $(INSTALL) 7 | MKDIR_P ?= mkdir -p 8 | RM ?= rm -f 9 | 10 | # Installation directories (should be passed from parent Makefile) 11 | prefix ?= /usr/local 12 | exec_prefix ?= $(prefix) 13 | bindir ?= $(exec_prefix)/bin 14 | DESTDIR ?= 15 | 16 | # Programs to install 17 | PROGRAMS = \ 18 | binary-to-json \ 19 | csv-to-json \ 20 | diff-to-json \ 21 | dsv-to-json \ 22 | env-to-json \ 23 | json-format \ 24 | json-make-schema \ 25 | json-objs-to-table \ 26 | json-run \ 27 | json-sql \ 28 | json-table-to-objs \ 29 | json-to-binary \ 30 | json-to-csv \ 31 | json-to-dsv \ 32 | json-to-env \ 33 | json-to-plot \ 34 | json-to-xml \ 35 | json-to-yaml \ 36 | pjito \ 37 | python-to-json-ast \ 38 | xml-to-json \ 39 | yaml-to-json 40 | 41 | # Go programs (built separately in parent Makefile) 42 | GO_PROGRAMS = json-diff 43 | 44 | all: $(GO_PROGRAMS) 45 | 46 | json-diff: json-diff.go 47 | $(GO) fmt json-diff.go 48 | $(GO) build json-diff.go 49 | 50 | 51 | .PHONY: install 52 | install: 53 | $(MKDIR_P) $(DESTDIR)$(bindir) 54 | @for prog in $(PROGRAMS); do \ 55 | echo " INSTALL $$prog -> $(DESTDIR)$(bindir)/$$prog"; \ 56 | $(INSTALL_PROGRAM) $$prog $(DESTDIR)$(bindir)/; \ 57 | done 58 | 59 | .PHONY: uninstall 60 | uninstall: 61 | @for prog in $(PROGRAMS) $(GO_PROGRAMS); do \ 62 | echo " RM $(DESTDIR)$(bindir)/$$prog"; \ 63 | $(RM) $(DESTDIR)$(bindir)/$$prog; \ 64 | done 65 | 66 | .PHONY: all install uninstall 67 | -------------------------------------------------------------------------------- /src/generate-json-diff-results: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import subprocess 5 | import sys 6 | 7 | 8 | def flatten(ll): 9 | a = [] 10 | for l in ll: 11 | a.extend(l) 12 | return a 13 | 14 | 15 | def run_command(command, stdin): 16 | p = subprocess.Popen(command, stdin=subprocess.PIPE, 17 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) 18 | stdout, stderr = p.communicate(stdin.encode("ASCII")) 19 | # Combine stderr and stdout (stderr first, as it typically contains error messages) 20 | output = stderr.decode("ASCII") + stdout.decode("ASCII") 21 | return output, p.returncode 22 | 23 | 24 | def run_test(test): 25 | actual_output, actual_return_code = run_command( 26 | test["command"], test["input"]) 27 | 28 | errors = [] 29 | if test["expectedOutput"] != actual_output: 30 | errors.append({ 31 | "actualOutput": actual_output, 32 | "command": test["command"], 33 | "expectedOutput": test["expectedOutput"], 34 | }) 35 | if test["expectedReturnCode"] != actual_return_code: 36 | errors.append({ 37 | "actualReturnCode": actual_return_code, 38 | "command": test["command"], 39 | "expectedReturnCode": test["expectedReturnCode"], 40 | }) 41 | 42 | return errors 43 | 44 | 45 | def main(): 46 | tests = json.load(sys.stdin) 47 | results = flatten(map(run_test, tests)) 48 | print(json.dumps(results)) 49 | 50 | 51 | if __name__ == "__main__": 52 | main() 53 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: json-toolkit 2 | Section: utils 3 | Priority: optional 4 | Maintainer: Tyler Adams 5 | Build-Depends: debhelper-compat (= 12), 6 | dh-sequence-python3, 7 | golang-go, 8 | golang-github-go-logfmt-logfmt-dev, 9 | python3-all 10 | Standards-Version: 4.6.0 11 | Homepage: https://github.com/tyleradams/json-toolkit 12 | Vcs-Browser: https://github.com/tyleradams/json-toolkit 13 | Vcs-Git: https://github.com/tyleradams/json-toolkit.git 14 | Rules-Requires-Root: no 15 | 16 | Package: json-toolkit 17 | Architecture: any 18 | Depends: ${shlibs:Depends}, 19 | ${misc:Depends}, 20 | ${python3:Depends}, 21 | golang-go, 22 | jq (>= 1.5), 23 | python3, 24 | python3-click, 25 | python3-dotenv, 26 | python3-psycopg2, 27 | python3-pymysql, 28 | python3-toml, 29 | python3-unidiff, 30 | python3-xmltodict, 31 | python3-yaml 32 | Description: Work with json data and convert non-json data to/from json 33 | JSON Toolkit is a comprehensive suite of command-line utilities for working 34 | with JSON and other data formats. It provides bidirectional conversion between 35 | JSON and CSV, XML, YAML, TOML, HTML, logfmt, and environment variables, as well 36 | as tools for JSON manipulation, schema generation, and database integration. 37 | . 38 | Features include: 39 | * Format conversion between JSON and multiple formats 40 | * JSON diff, formatting, and schema generation 41 | * Database integration (PostgreSQL, MySQL, SQLite3) 42 | * Unix-friendly command-line tools that work with pipes 43 | -------------------------------------------------------------------------------- /src/json-to-logfmt.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "fmt" 7 | "os" 8 | 9 | "github.com/go-logfmt/logfmt" 10 | ) 11 | 12 | const version = "2.0.6" 13 | 14 | func main() { 15 | helpFlag := flag.Bool("help", false, "show help") 16 | hFlag := flag.Bool("h", false, "show help") 17 | flag.Parse() 18 | 19 | if *helpFlag || *hFlag { 20 | showHelp() 21 | os.Exit(0) 22 | } 23 | 24 | // Read JSON array from stdin 25 | var data []map[string]interface{} 26 | decoder := json.NewDecoder(os.Stdin) 27 | if err := decoder.Decode(&data); err != nil { 28 | fmt.Fprintf(os.Stderr, "Error: Invalid JSON: %v\n", err) 29 | os.Exit(1) 30 | } 31 | 32 | // Convert each object to logfmt 33 | encoder := logfmt.NewEncoder(os.Stdout) 34 | for _, record := range data { 35 | for key, value := range record { 36 | if err := encoder.EncodeKeyval(key, value); err != nil { 37 | fmt.Fprintf(os.Stderr, "Error encoding logfmt: %v\n", err) 38 | os.Exit(1) 39 | } 40 | } 41 | if err := encoder.EndRecord(); err != nil { 42 | fmt.Fprintf(os.Stderr, "Error writing logfmt: %v\n", err) 43 | os.Exit(1) 44 | } 45 | } 46 | } 47 | 48 | func showHelp() { 49 | fmt.Printf(`json-to-logfmt - convert JSON array to logfmt format (version %s) 50 | 51 | Usage: 52 | json-to-logfmt [flags] 53 | 54 | Overview: 55 | json-to-logfmt reads a JSON array of objects from stdin and outputs 56 | logfmt-formatted lines. Each object in the array becomes one logfmt line. 57 | 58 | Flags: 59 | -h, -help show help 60 | 61 | Examples: 62 | echo '[{"key":"value"}]' | json-to-logfmt 63 | json-to-logfmt < data.json 64 | cat logs.json | json-to-logfmt 65 | `, version) 66 | } 67 | -------------------------------------------------------------------------------- /src/json-run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import subprocess 5 | import sys 6 | import click 7 | 8 | CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} 9 | 10 | PROPS = [ 11 | "args", 12 | "returncode", 13 | "stdout", 14 | "stderr" 15 | ] 16 | DECODES = [ 17 | "stdout", 18 | "stderr", 19 | ] 20 | 21 | def run_command(c): 22 | o = subprocess.run(c, capture_output=True) 23 | retu = {k: getattr(o, k) for k in PROPS} 24 | for k in DECODES: 25 | retu[k] = retu[k].decode("UTF-8") 26 | return retu 27 | 28 | @click.command(context_settings=CONTEXT_SETTINGS) 29 | @click.version_option("1.1.0", prog_name="json-run") 30 | def main(): 31 | """ 32 | Execute commands from JSON array and return results as JSON. 33 | 34 | Reads a JSON array of commands from stdin, executes each command, 35 | and outputs a JSON array with the results including args, returncode, 36 | stdout, and stderr for each command. 37 | 38 | \b 39 | Examples: 40 | echo '[["echo", "hello"], ["ls", "-la"]]' | json-run 41 | json-run < commands.json > results.json 42 | cat tasks.json | json-run | jq '.' 43 | """ 44 | try: 45 | commands = json.load(sys.stdin) 46 | results = [run_command(c) for c in commands] 47 | print(json.dumps(results)) 48 | sys.exit(0) 49 | except json.JSONDecodeError as e: 50 | click.echo(f"Error: Invalid JSON: {e}", err=True) 51 | sys.exit(1) 52 | except BrokenPipeError: 53 | sys.stderr.close() 54 | sys.exit(0) 55 | except Exception as e: 56 | click.echo(f"Error: {e}", err=True) 57 | sys.exit(1) 58 | 59 | if __name__ == "__main__": 60 | main() 61 | -------------------------------------------------------------------------------- /src/json-to-binary: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import sys 5 | import click 6 | 7 | CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} 8 | 9 | def validate_binary(data): 10 | data_with_index = list(enumerate(data)) 11 | errors = [] 12 | errors.extend([{"message": "Value is not an int", "index": x[0], "value": x[1]} 13 | for x in data_with_index if type(x[1]) != int]) 14 | errors.extend([{"message": "Value is not within [0,255]", "index": x[0], "value": x[1]} 15 | for x in data_with_index if x[1] < 0 or x[1] > 255]) 16 | if len(errors) != 0: 17 | raise Exception("Invalid data:{}".format(json.dumps(errors))) 18 | 19 | @click.command(context_settings=CONTEXT_SETTINGS) 20 | @click.version_option("1.1.0", prog_name="json-to-binary") 21 | def main(): 22 | """ 23 | Convert JSON array of bytes to binary data. 24 | 25 | Reads a JSON array from stdin where each element is a byte value (0-255) 26 | and outputs the corresponding binary data to stdout. 27 | 28 | \b 29 | Examples: 30 | echo '[72, 101, 108, 108, 111]' | json-to-binary 31 | json-to-binary < data.json > output.bin 32 | cat bytes.json | json-to-binary > file.bin 33 | """ 34 | try: 35 | data = json.load(sys.stdin) 36 | validate_binary(data) 37 | 38 | a = bytearray() 39 | for byte in data: 40 | a.append(byte) 41 | 42 | sys.stdout.buffer.write(a) 43 | sys.exit(0) 44 | except json.JSONDecodeError as e: 45 | click.echo(f"Error: Invalid JSON: {e}", err=True) 46 | sys.exit(1) 47 | except BrokenPipeError: 48 | sys.stderr.close() 49 | sys.exit(0) 50 | except Exception as e: 51 | click.echo(f"Error: {e}", err=True) 52 | sys.exit(1) 53 | 54 | if __name__ == "__main__": 55 | main() 56 | -------------------------------------------------------------------------------- /src/html-to-json: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import json 5 | import click 6 | from html.parser import HTMLParser 7 | 8 | CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} 9 | 10 | class JSONEncoderHTMLParser(HTMLParser): 11 | def __init__(self): 12 | HTMLParser.__init__(self) 13 | self.document = [] 14 | self.stack = [self.document] 15 | 16 | def handle_starttag(self, tag, attrs): 17 | new_tag = {'tag': tag, 'attrs': dict(attrs), 'children': []} 18 | self.stack[-1].append(new_tag) 19 | self.stack.append(new_tag['children']) 20 | 21 | def handle_endtag(self, tag): 22 | if len(self.stack) > 1: 23 | self.stack.pop() 24 | 25 | def handle_data(self, data): 26 | trimmed_data = data.strip() 27 | if trimmed_data: 28 | self.stack[-1].append(trimmed_data) 29 | 30 | @click.command(context_settings=CONTEXT_SETTINGS) 31 | @click.version_option("1.1.0", prog_name="html-to-json") 32 | def main(): 33 | """ 34 | Convert HTML from stdin to JSON structure. 35 | 36 | Reads HTML data from standard input and outputs a JSON representation 37 | where each HTML tag is an object with 'tag', 'attrs', and 'children' keys. 38 | 39 | \b 40 | Examples: 41 | echo '

Hello

' | html-to-json 42 | html-to-json < page.html > page.json 43 | cat index.html | html-to-json | jq '.' 44 | """ 45 | try: 46 | parser = JSONEncoderHTMLParser() 47 | html = sys.stdin.read() 48 | parser.feed(html) 49 | print(json.dumps(parser.document, indent=4)) 50 | sys.exit(0) 51 | except BrokenPipeError: 52 | sys.stderr.close() 53 | sys.exit(0) 54 | except Exception as e: 55 | click.echo(f"Error: {e}", err=True) 56 | sys.exit(1) 57 | 58 | if __name__ == "__main__": 59 | main() 60 | -------------------------------------------------------------------------------- /src/logfmt-to-json.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "os" 9 | "strings" 10 | 11 | "github.com/go-logfmt/logfmt" 12 | ) 13 | 14 | const version = "2.0.6" 15 | 16 | func main() { 17 | helpFlag := flag.Bool("help", false, "show help") 18 | hFlag := flag.Bool("h", false, "show help") 19 | flag.Parse() 20 | 21 | if *helpFlag || *hFlag { 22 | showHelp() 23 | os.Exit(0) 24 | } 25 | 26 | // Read logfmt lines from stdin and convert to JSON array 27 | scanner := bufio.NewScanner(os.Stdin) 28 | var result []map[string]interface{} 29 | 30 | for scanner.Scan() { 31 | line := strings.TrimSpace(scanner.Text()) 32 | if line == "" { 33 | continue 34 | } 35 | 36 | // Parse the logfmt line 37 | decoder := logfmt.NewDecoder(strings.NewReader(line)) 38 | record := make(map[string]interface{}) 39 | 40 | for decoder.ScanRecord() { 41 | for decoder.ScanKeyval() { 42 | key := string(decoder.Key()) 43 | value := string(decoder.Value()) 44 | record[key] = value 45 | } 46 | } 47 | 48 | if err := decoder.Err(); err != nil { 49 | fmt.Fprintf(os.Stderr, "Error parsing logfmt: %v\n", err) 50 | os.Exit(1) 51 | } 52 | 53 | result = append(result, record) 54 | } 55 | 56 | if err := scanner.Err(); err != nil { 57 | fmt.Fprintf(os.Stderr, "Error reading input: %v\n", err) 58 | os.Exit(1) 59 | } 60 | 61 | // Output as JSON 62 | encoder := json.NewEncoder(os.Stdout) 63 | encoder.SetIndent("", " ") 64 | if err := encoder.Encode(result); err != nil { 65 | fmt.Fprintf(os.Stderr, "Error encoding JSON: %v\n", err) 66 | os.Exit(1) 67 | } 68 | } 69 | 70 | func showHelp() { 71 | fmt.Printf(`logfmt-to-json - convert logfmt format to JSON (version %s) 72 | 73 | Usage: 74 | logfmt-to-json [flags] 75 | 76 | Overview: 77 | logfmt-to-json reads logfmt-formatted lines from standard input and outputs 78 | a JSON array. Each logfmt line is parsed into a JSON object. 79 | 80 | Flags: 81 | -h, -help show help 82 | 83 | Examples: 84 | echo "key=value foo=bar" | logfmt-to-json 85 | logfmt-to-json < logs.txt 86 | cat access.log | logfmt-to-json | jq '.' 87 | `, version) 88 | } 89 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: json-toolkit 3 | Upstream-Contact: Tyler Adams 4 | Source: https://github.com/tyleradams/json-toolkit 5 | 6 | Files: * 7 | Copyright: 2018-2023 Tyler Adams 8 | License: GPL-2+ 9 | 10 | This package is free software; you can redistribute it and/or modify 11 | it under the terms of the GNU General Public License as published by 12 | the Free Software Foundation; either version 2 of the License, or 13 | (at your option) any later version. 14 | . 15 | This package is distributed in the hope that it will be useful, 16 | but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | GNU General Public License for more details. 19 | . 20 | You should have received a copy of the GNU General Public License 21 | along with this program. If not, see 22 | . 23 | On Debian systems, the complete text of the GNU General 24 | Public License version 2 can be found in "/usr/share/common-licenses/GPL-2". 25 | 26 | Files: debian/* 27 | Copyright: 2023 Tyler Adams 28 | License: GPL-2+ 29 | This package is free software; you can redistribute it and/or modify 30 | it under the terms of the GNU General Public License as published by 31 | the Free Software Foundation; either version 2 of the License, or 32 | (at your option) any later version. 33 | . 34 | This package is distributed in the hope that it will be useful, 35 | but WITHOUT ANY WARRANTY; without even the implied warranty of 36 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 37 | GNU General Public License for more details. 38 | . 39 | You should have received a copy of the GNU General Public License 40 | along with this program. If not, see 41 | . 42 | On Debian systems, the complete text of the GNU General 43 | Public License version 2 can be found in "/usr/share/common-licenses/GPL-2". 44 | 45 | # Please also look if there are files or directories which have a 46 | # different copyright/license attached and list them here. 47 | # Please avoid picking licenses with terms that are more restrictive than the 48 | # packaged work, as it may make Debian's contributions unacceptable upstream. 49 | # 50 | # If you need, there are some extra license texts available in two places: 51 | # /usr/share/debhelper/dh_make/licenses/ 52 | # /usr/share/common-licenses/ 53 | -------------------------------------------------------------------------------- /src/json-to-xml: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import sys 5 | import click 6 | import xmltodict 7 | 8 | CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} 9 | 10 | JSON_FROM_PYTHON_NAMES = { 11 | dict: "object", 12 | list: "array", 13 | int: "Number", 14 | float: "Number", 15 | bool: "Boolean", 16 | None: "null" 17 | } 18 | 19 | class InvalidXMLSerializableData(Exception): 20 | pass 21 | 22 | def validate_data(data): 23 | if type(data) == dict and len(data.keys()) == 1: 24 | return 25 | 26 | # Prefacing \n makes multierror lines easier to read 27 | message = "\n Only a json object with 1 key can be serialized to xml" 28 | if type(data) != dict: 29 | type_name = JSON_FROM_PYTHON_NAMES[type(data)] 30 | if type_name[0] in ["a", "e", "i", "o", "u"]: 31 | message += "\n The inputted json value is not an object, it is an {}".format( 32 | type_name) 33 | else: 34 | message += "\n The inputted json value is not an object, it is a {}".format( 35 | type_name) 36 | elif type(data) == dict and len(data.keys()) != 1: 37 | message += "\n Input object does not have 1 key, it has {} keys".format( 38 | len(data.keys())) 39 | else: 40 | raise Exception( 41 | "The code cannot handle this input, to receive support, please file a bug specifying the input") 42 | raise InvalidXMLSerializableData(message) 43 | 44 | @click.command(context_settings=CONTEXT_SETTINGS) 45 | @click.version_option("1.1.0", prog_name="json-to-xml") 46 | def main(): 47 | """ 48 | Convert JSON to XML format. 49 | 50 | Reads a JSON object from stdin and outputs XML. The input JSON must 51 | be an object with exactly one key, which becomes the root XML element. 52 | 53 | \b 54 | Examples: 55 | echo '{"root":{"item":"value"}}' | json-to-xml 56 | json-to-xml < data.json > data.xml 57 | cat config.json | json-to-xml 58 | """ 59 | try: 60 | data = json.load(sys.stdin) 61 | validate_data(data) 62 | print(xmltodict.unparse(data, pretty=True)) 63 | sys.exit(0) 64 | except json.JSONDecodeError as e: 65 | click.echo(f"Error: Invalid JSON: {e}", err=True) 66 | sys.exit(1) 67 | except InvalidXMLSerializableData as e: 68 | click.echo(f"Error: {e}", err=True) 69 | sys.exit(1) 70 | except BrokenPipeError: 71 | sys.stderr.close() 72 | sys.exit(0) 73 | except Exception as e: 74 | click.echo(f"Error: {e}", err=True) 75 | sys.exit(1) 76 | 77 | if __name__ == "__main__": 78 | main() 79 | -------------------------------------------------------------------------------- /verify-ppa-install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | VERSION="${1:-2.0.5}" 5 | UBUNTU_VERSION="${2:-24.04}" 6 | PPA_NAME="${PPA_NAME:-ppa:code-faster/ppa}" 7 | 8 | echo "Testing PPA installation of json-toolkit $VERSION from $PPA_NAME" 9 | echo "Ubuntu version: $UBUNTU_VERSION" 10 | 11 | dockerfile=$(mktemp) 12 | cat > "$dockerfile" < /dev/null; then 66 | BUILD_STATUS=$(curl -sL "$PPA_URL" | grep -A 20 "json-toolkit" | head -30) 67 | 68 | if echo "$BUILD_STATUS" | grep -q "$VERSION"; then 69 | echo "" 70 | echo "Found version $VERSION in PPA:" 71 | echo "$BUILD_STATUS" | grep -E "(Building|Failed|Published|Pending)" | head -5 72 | 73 | if echo "$BUILD_STATUS" | grep -qi "building"; then 74 | echo "" 75 | echo "⏳ Package is currently BUILDING on Launchpad" 76 | echo " Wait a few more minutes and try again" 77 | elif echo "$BUILD_STATUS" | grep -qi "failed"; then 78 | echo "" 79 | echo "❌ Package BUILD FAILED on Launchpad" 80 | echo " Check build logs at: $PPA_URL" 81 | elif echo "$BUILD_STATUS" | grep -qi "pending"; then 82 | echo "" 83 | echo "⏳ Package is PENDING in build queue" 84 | echo " Wait a few more minutes and try again" 85 | fi 86 | else 87 | echo "" 88 | echo "Version $VERSION not found in PPA yet" 89 | echo "Latest version available: $(docker run --rm ubuntu:22.04 bash -c 'apt-get update -qq 2>/dev/null && apt-get install -y -qq software-properties-common 2>/dev/null && add-apt-repository -y '$PPA_NAME' 2>/dev/null && apt-get update -qq 2>/dev/null && apt-cache policy json-toolkit 2>/dev/null | grep Candidate' 2>/dev/null || echo 'unknown')" 90 | fi 91 | 92 | echo "" 93 | echo "Check full status at: $PPA_URL" 94 | else 95 | echo "curl not available - cannot check Launchpad status" 96 | echo "Check manually at: $PPA_URL" 97 | fi 98 | 99 | echo "" 100 | echo "Dockerfile saved at: $dockerfile" 101 | exit 1 102 | fi 103 | -------------------------------------------------------------------------------- /test_data/tests.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "command": [ 4 | "./src/json-to-csv" 5 | ], 6 | "input": "[[\"single cell\"]]\n", 7 | "output": "\"single cell\"\n", 8 | "returnCode": 0 9 | }, 10 | { 11 | "command": [ 12 | "./src/json-format", 13 | "-" 14 | ], 15 | "input": "{\"second\": 2, \"first\": 1}", 16 | "output": "{\n \"first\": 1,\n \"second\": 2\n}\n", 17 | "returnCode": 0 18 | }, 19 | { 20 | "command": [ 21 | "./src/json-format" 22 | ], 23 | "input": "[]", 24 | "output": "[]\n", 25 | "returnCode": 0 26 | }, 27 | { 28 | "command": [ 29 | "./src/json-format", 30 | "NONEXISTANT_FILE" 31 | ], 32 | "input": "[]", 33 | "output": "NONEXISTANT_FILE does not exist\n", 34 | "returnCode": 1 35 | }, 36 | { 37 | "command": [ 38 | "./src/json-to-csv" 39 | ], 40 | "input": "[[\"multiple\", \"cells\", \"single\", \"row\"]]\n", 41 | "output": "\"multiple\",\"cells\",\"single\",\"row\"\n", 42 | "returnCode": 0 43 | }, 44 | { 45 | "command": [ 46 | "./src/json-to-csv" 47 | ], 48 | "input": "[[\"multiple\", \"cells\"], [\"multiple \", \"rows\"]]\n", 49 | "output": "\"multiple\",\"cells\"\n\"multiple \",\"rows\"\n", 50 | "returnCode": 0 51 | }, 52 | { 53 | "command": [ 54 | "./src/json-to-dsv", 55 | "," 56 | ], 57 | "input": "[[\"single cell\"]]\n", 58 | "output": "single cell\n", 59 | "returnCode": 0 60 | }, 61 | { 62 | "command": [ 63 | "./src/json-to-dsv", 64 | "," 65 | ], 66 | "input": "[[\"multiple\", \"cells\", \"single\", \"row\"]]\n", 67 | "output": "multiple,cells,single,row\n", 68 | "returnCode": 0 69 | }, 70 | { 71 | "command": [ 72 | "./src/json-to-dsv", 73 | "," 74 | ], 75 | "input": "[[\"multiple\", \"cells\"], [\"multiple \", \"rows\"]]\n", 76 | "output": "multiple,cells\nmultiple ,rows\n", 77 | "returnCode": 0 78 | }, 79 | { 80 | "command": [ 81 | "./src/json-to-xml" 82 | ], 83 | "input": "{\"a\": 1}", 84 | "output": "\n1\n", 85 | "returnCode": 0 86 | }, 87 | { 88 | "command": [ 89 | "./src/json-to-yaml" 90 | ], 91 | "input": "{\"a\": 1, \"b\": 2}\n", 92 | "output": "a: 1\nb: 2\n", 93 | "returnCode": 0 94 | }, 95 | { 96 | "command": [ 97 | "./src/csv-to-json" 98 | ], 99 | "input": "Single cell\n", 100 | "output": "[[\"Single cell\"]]\n", 101 | "returnCode": 0 102 | }, 103 | { 104 | "command": [ 105 | "./src/csv-to-json" 106 | ], 107 | "input": "Multiple,cells,but,one,row\n", 108 | "output": "[[\"Multiple\", \"cells\", \"but\", \"one\", \"row\"]]\n", 109 | "returnCode": 0 110 | }, 111 | { 112 | "command": [ 113 | "./src/csv-to-json" 114 | ], 115 | "input": "Multiple,cells\nand\nmultiple,rows", 116 | "output": "[[\"Multiple\", \"cells\"], [\"and\"], [\"multiple\", \"rows\"]]\n", 117 | "returnCode": 0 118 | }, 119 | { 120 | "command": [ 121 | "./src/dsv-to-json", 122 | ":" 123 | ], 124 | "input": "Single cell\n", 125 | "output": "[[\"Single cell\"]]\n", 126 | "returnCode": 0 127 | }, 128 | { 129 | "command": [ 130 | "./src/dsv-to-json", 131 | ":" 132 | ], 133 | "input": "Multiple:cells:but:one:row\n", 134 | "output": "[[\"Multiple\", \"cells\", \"but\", \"one\", \"row\"]]\n", 135 | "returnCode": 0 136 | }, 137 | { 138 | "command": [ 139 | "./src/dsv-to-json", 140 | ";" 141 | ], 142 | "input": "Multiple cells\nand\nmultiple;rows", 143 | "output": "[[\"Multiple cells\"], [\"and\"], [\"multiple\", \"rows\"]]\n", 144 | "returnCode": 0 145 | }, 146 | { 147 | "command": [ 148 | "./src/xml-to-json" 149 | ], 150 | "input": "b", 151 | "output": "{\"a\": \"b\"}\n", 152 | "returnCode": 0 153 | }, 154 | { 155 | "command": [ 156 | "./src/yaml-to-json" 157 | ], 158 | "input": "a: b", 159 | "output": "{\"a\": \"b\"}\n", 160 | "returnCode": 0 161 | } 162 | ] 163 | -------------------------------------------------------------------------------- /VERSIONING.md: -------------------------------------------------------------------------------- 1 | # Versioning Strategy 2 | 3 | json-toolkit uses a **tag-driven versioning approach**, following best practices from sophisticated projects like the Linux kernel and Go modules. 4 | 5 | ## Philosophy 6 | 7 | - **Git tags are the sole source of truth** for releases 8 | - **No special "Release" commits** - commits are for code changes only 9 | - **Version numbers live in git tags**, not in committed files (except CHANGELOG.md for documentation) 10 | 11 | ## How It Works 12 | 13 | ### Version Discovery 14 | 15 | The Makefile automatically derives the version from git tags: 16 | 17 | ```makefile 18 | VERSION ?= $(shell git describe --tags --always --dirty | sed 's/^v//' || echo "dev") 19 | ``` 20 | 21 | **Examples:** 22 | - On tag `v2.0.0`: `VERSION=2.0.0` 23 | - After tag: `VERSION=2.0.0-1-gabcd123` (1 commit past v2.0.0) 24 | - With uncommitted changes: `VERSION=2.0.0-dirty` 25 | - No tags yet: `VERSION=dev` 26 | 27 | ### Debian Changelog 28 | 29 | `debian/changelog` is **NOT checked into git**. It's generated at package-build time from git history using either: 30 | 1. `gbp dch` (git-buildpackage) - preferred 31 | 2. Fallback: Simple generation from git user/date 32 | 33 | This keeps the repo clean and avoids merge conflicts. 34 | 35 | ## Release Process 36 | 37 | ### 1. Normal Development 38 | Make commits as usual. No version numbers needed: 39 | ```bash 40 | git commit -m "Add new feature" 41 | git commit -m "Fix bug in csv-to-json" 42 | ``` 43 | 44 | ### 2. Prepare for Release 45 | Update CHANGELOG.md (optional, for human documentation): 46 | ```bash 47 | # Edit CHANGELOG.md: Change ## [2.0.0] - TBD to actual date 48 | ``` 49 | 50 | ### 3. Tag the Release 51 | ```bash 52 | # Commit CHANGELOG if modified 53 | git commit -am "Update CHANGELOG for v2.0.0" 54 | 55 | # Create annotated, signed tag 56 | git tag -s -a v2.0.0 -m "Release 2.0.0" 57 | 58 | # Push 59 | git push origin master v2.0.0 60 | ``` 61 | 62 | ### 4. Automated Release (using release.sh) 63 | ```bash 64 | ./release.sh 2.0.0 65 | ``` 66 | 67 | This automatically: 68 | - Updates CHANGELOG.md date (TBD → actual date) 69 | - Runs all tests (local + Docker) 70 | - Commits CHANGELOG 71 | - Creates signed tag `v2.0.0` 72 | - Generates `debian/changelog` from git 73 | - Builds & publishes to PPA 74 | 75 | ## Comparison to Other Approaches 76 | 77 | ### "Normie" Approach (Common but not ideal) 78 | ```bash 79 | # Update version in Makefile 80 | sed -i 's/VERSION = 1.0/VERSION = 2.0/' Makefile 81 | git commit -m "Release version 2.0.0" # ← Special release commit 82 | git tag v2.0.0 83 | ``` 84 | 85 | **Problems:** 86 | - Empty "Release" commit wastes history 87 | - Version in file can drift from tags 88 | - Merge conflicts on version file 89 | 90 | ### Sophisticated Approach (Linux, Go, json-toolkit) 91 | ```bash 92 | # No version files to update! 93 | git commit -m "Fix bug" # ← Normal commit 94 | git tag -s v2.0.0 # ← Tag marks the release 95 | ``` 96 | 97 | **Benefits:** 98 | - Tags are canonical 99 | - No version drift 100 | - Clean history 101 | - Version auto-computed from VCS 102 | 103 | ## For Contributors 104 | 105 | ### To check current version: 106 | ```bash 107 | make help # Shows VERSION=$(VERSION) 108 | git describe --tags 109 | ``` 110 | 111 | ### To test without tags: 112 | ```bash 113 | # Will show VERSION=dev 114 | make clean && make 115 | ``` 116 | 117 | ### After cloning: 118 | ```bash 119 | # Fetch tags 120 | git fetch --tags 121 | 122 | # Now version works 123 | make 124 | ``` 125 | 126 | ## Technical Details 127 | 128 | ### git describe format: 129 | ``` 130 | v2.0.0-3-gabcd123-dirty 131 | │ │ │ └── uncommitted changes 132 | │ │ └────────── short commit hash 133 | │ └──────────── commits since tag 134 | └─────────────────── nearest tag 135 | ``` 136 | 137 | ### Debian packaging: 138 | The `make package` target: 139 | 1. Runs `gbp dch --new-version=$(VERSION)` to generate `debian/changelog` 140 | 2. Falls back to simple generation if `gbp` not available 141 | 3. Packages with the derived version 142 | 143 | ### Why not check in debian/changelog? 144 | - Avoids merge conflicts (changelog always at top of file) 145 | - Git history is the source of truth 146 | - Automatically includes all commits 147 | - Cleaner repo 148 | 149 | ## References 150 | 151 | - Linux kernel: Uses tags + version in Makefile (small bump commit then tag) 152 | - Go modules: Tags only, version injected at build time 153 | - git-buildpackage: Debian packaging from git, generates changelogs 154 | - Semantic Versioning: https://semver.org/ 155 | -------------------------------------------------------------------------------- /CLAUDE.md: -------------------------------------------------------------------------------- 1 | # CLAUDE.md 2 | 3 | This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. 4 | 5 | ## Project Overview 6 | 7 | JSON Toolkit is a comprehensive suite of command-line utilities for working with JSON and other data formats. The toolkit provides bidirectional converters between JSON and various formats (CSV, XML, YAML, TOML, HTML, logfmt, etc.), as well as utilities for JSON manipulation, schema generation, and database integration. 8 | 9 | ## Architecture 10 | 11 | The project consists of three types of utilities: 12 | 13 | 1. **Python3 scripts**: Most converters and utilities (e.g., `csv-to-json`, `json-sql`, `json-make-schema`) 14 | 2. **Go programs**: Performance-critical utilities (`json-diff.go`) 15 | 3. **Bash scripts**: Simple wrappers and test utilities 16 | 17 | All utilities follow a Unix philosophy: 18 | - Read from stdin, write to stdout 19 | - Exit code 0 for success, non-zero for failure 20 | - JSON as the primary interchange format 21 | - Composable with pipes and other Unix tools (especially `jq`) 22 | 23 | ## Build System 24 | 25 | The project uses a two-level Makefile structure: 26 | - Root `Makefile`: Orchestrates builds, tests, and packaging 27 | - `src/Makefile`: Handles installation of individual utilities 28 | 29 | ### Common Commands 30 | 31 | ```bash 32 | # Build everything (compiles Go programs and installs dependencies) 33 | make 34 | 35 | # Run all tests 36 | make test 37 | 38 | # Run individual test suites 39 | ./run-json-diff-tests # Tests for json-diff 40 | ./run-all-tests # Complete test suite 41 | 42 | # Format code 43 | make fmt 44 | 45 | # Lint code 46 | make lint 47 | ./run-bash-linter # Bash-specific linting 48 | 49 | # Install locally (to /usr/local/bin by default) 50 | make install 51 | sudo make install # Usually requires sudo 52 | 53 | # Clean build artifacts 54 | make clean 55 | 56 | # Build Go programs only 57 | make json-diff 58 | ``` 59 | 60 | ## Key Utilities 61 | 62 | ### Data Conversion 63 | - **To JSON**: `csv-to-json`, `xml-to-json`, `yaml-to-json`, `toml-to-json`, `html-to-json`, `logfmt-to-json`, `env-to-json`, `diff-to-json`, `binary-to-json`, `dsv-to-json` 64 | - **From JSON**: `json-to-csv`, `json-to-xml`, `json-to-yaml`, `json-to-env`, `json-to-logfmt`, `json-to-binary`, `json-to-dsv` 65 | 66 | ### JSON Manipulation 67 | - `json-format`: Format/prettify JSON 68 | - `json-diff`: Compare two JSON files and output structured differences (Go implementation) 69 | - `json-make-schema`: Generate JSON schema from JSON data 70 | - `json-table-to-objs`: Convert JSON table format to objects 71 | - `json-objs-to-table`: Convert JSON objects to table format 72 | 73 | ### Database Integration 74 | - `json-sql`: Read/write SQL databases (PostgreSQL, MySQL, SQLite3, Oracle ADB) 75 | - Supports both read (dump entire DB) and query (execute JSON array of queries) 76 | - See help: `./src/json-sql` (no args) 77 | 78 | ### Other Utilities 79 | - `python-to-json-ast`: Parse Python code to JSON AST 80 | - `pjito`: "Python JSON in, text out" - template renderer 81 | - `json-run`: Execute code based on JSON input 82 | 83 | ## Testing Strategy 84 | 85 | The test system uses `json-diff` to validate outputs: 86 | 87 | 1. `test_data/tests.json` contains test cases with expected results 88 | 2. `generate-test-results` runs tests and produces actual results 89 | 3. `json-diff` compares expected vs actual 90 | 4. `jq` checks if diff is empty (returns exit code 0 if no differences) 91 | 92 | Test files in `test_data/`: 93 | - `json-diff-tests.json`: Tests specifically for json-diff 94 | - `tests.json`: General test suite 95 | - Various `.json` files: Test fixtures (e.g., `a.json`, `b.json`, `m1.json`, `m2.json`) 96 | 97 | ## Dependencies 98 | 99 | ### System Dependencies 100 | - Bash 101 | - Go (golang-go) 102 | - Python3 103 | - jq (>= 1.5) 104 | - libpq-dev (for PostgreSQL support) 105 | 106 | ### Python Dependencies 107 | See `requirements.txt`. Install with: 108 | ```bash 109 | python3 -m pip install -r requirements.txt 110 | ``` 111 | 112 | Key Python packages: 113 | - Database drivers: `psycopg2-binary`, `PyMySQL`, `cx_Oracle` 114 | - Format parsers: `PyYAML`, `xmltodict`, `unidiff`, `logfmt` 115 | - Other: `genson` (schema generation), `ast2json` (Python AST parsing) 116 | 117 | ## Packaging and Distribution 118 | 119 | The project is distributed as a Debian package via PPA: 120 | ```bash 121 | sudo add-apt-repository ppa:code-faster/ppa 122 | sudo apt update 123 | sudo apt install json-toolkit 124 | ``` 125 | 126 | Build Debian package: 127 | ```bash 128 | make package version=1.2.3 129 | ``` 130 | 131 | Publish to PPA (requires DEBSIGN_KEY): 132 | ```bash 133 | make publish version=1.2.3 DEBSIGN_KEY=YOUR_KEY 134 | ``` 135 | 136 | ## Code Style 137 | 138 | - Python: Follow PEP8 (see `.pep8` and `.pylintrc` configs) 139 | - Go: Use `go fmt` (automatically applied during build) 140 | - Bash: Follow ShellCheck recommendations 141 | 142 | ## Development Notes 143 | 144 | - Go programs are built to `target/` directory during packaging 145 | - Most utilities are executable scripts in `src/` that can be run directly during development 146 | - The `json-diff` utility is special: it's both a compiled Go binary and used extensively in testing, so changes require rebuild + test 147 | - Python scripts have shebang `#!/usr/bin/env python3` for portability 148 | -------------------------------------------------------------------------------- /test_data/json-diff-tests.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "command": [ 4 | "./target/json-diff" 5 | ], 6 | "expectedOutput": "error: requires exactly 2 file arguments\njson-diff - compare two JSON files and report differences\n\nUsage:\n json-diff [flags] FILE1 FILE2\n\nOverview:\n json-diff reports differences between two JSON files as JSON.\n\n For each difference, json-diff reports the path, the value in FILE1\n (leftValue) if present, and the value in FILE2 (rightValue) if present.\n\nPath Notation:\n The path is an array of numbers and strings. Each number refers to an\n array index and each string refers to an object key.\n\n Example: [0, \"a\", 2] refers to \"foo\" in:\n [{\"a\": [null, null, \"foo\"]}]\n\nExit Codes:\n 0 - Files are identical\n 1 - Files differ\n 2 - Error occurred\n\nFlags:\n -h, -help show help\n\nExamples:\n json-diff old.json new.json\n json-diff config1.json config2.json | jq '.'\n", 7 | "expectedReturnCode": 2, 8 | "input": "" 9 | }, 10 | { 11 | "command": [ 12 | "./target/json-diff", 13 | "a.json" 14 | ], 15 | "expectedOutput": "error: requires exactly 2 file arguments\njson-diff - compare two JSON files and report differences\n\nUsage:\n json-diff [flags] FILE1 FILE2\n\nOverview:\n json-diff reports differences between two JSON files as JSON.\n\n For each difference, json-diff reports the path, the value in FILE1\n (leftValue) if present, and the value in FILE2 (rightValue) if present.\n\nPath Notation:\n The path is an array of numbers and strings. Each number refers to an\n array index and each string refers to an object key.\n\n Example: [0, \"a\", 2] refers to \"foo\" in:\n [{\"a\": [null, null, \"foo\"]}]\n\nExit Codes:\n 0 - Files are identical\n 1 - Files differ\n 2 - Error occurred\n\nFlags:\n -h, -help show help\n\nExamples:\n json-diff old.json new.json\n json-diff config1.json config2.json | jq '.'\n", 16 | "expectedReturnCode": 2, 17 | "input": "" 18 | }, 19 | { 20 | "command": [ 21 | "./target/json-diff", 22 | "a.json", 23 | "a.json", 24 | "a.json" 25 | ], 26 | "expectedOutput": "error: requires exactly 2 file arguments\njson-diff - compare two JSON files and report differences\n\nUsage:\n json-diff [flags] FILE1 FILE2\n\nOverview:\n json-diff reports differences between two JSON files as JSON.\n\n For each difference, json-diff reports the path, the value in FILE1\n (leftValue) if present, and the value in FILE2 (rightValue) if present.\n\nPath Notation:\n The path is an array of numbers and strings. Each number refers to an\n array index and each string refers to an object key.\n\n Example: [0, \"a\", 2] refers to \"foo\" in:\n [{\"a\": [null, null, \"foo\"]}]\n\nExit Codes:\n 0 - Files are identical\n 1 - Files differ\n 2 - Error occurred\n\nFlags:\n -h, -help show help\n\nExamples:\n json-diff old.json new.json\n json-diff config1.json config2.json | jq '.'\n", 27 | "expectedReturnCode": 2, 28 | "input": "" 29 | }, 30 | { 31 | "command": [ 32 | "./target/json-diff", 33 | "non_existant_file", 34 | "non_existant_file" 35 | ], 36 | "expectedOutput": "error reading non_existant_file: open non_existant_file: no such file or directory\n", 37 | "expectedReturnCode": 2, 38 | "input": "" 39 | }, 40 | { 41 | "command": [ 42 | "./target/json-diff", 43 | "test_data/null.json", 44 | "test_data/null.json" 45 | ], 46 | "expectedOutput": "[]\n", 47 | "expectedReturnCode": 0, 48 | "input": "" 49 | }, 50 | { 51 | "command": [ 52 | "./target/json-diff", 53 | "test_data/s1.json", 54 | "test_data/s1.json" 55 | ], 56 | "expectedOutput": "[]\n", 57 | "expectedReturnCode": 0, 58 | "input": "" 59 | }, 60 | { 61 | "command": [ 62 | "./target/json-diff", 63 | "test_data/s2.json", 64 | "test_data/s2.json" 65 | ], 66 | "expectedOutput": "[]\n", 67 | "expectedReturnCode": 0, 68 | "input": "" 69 | }, 70 | { 71 | "command": [ 72 | "./target/json-diff", 73 | "test_data/s2.json", 74 | "test_data/s3.json" 75 | ], 76 | "expectedOutput": "[{\"leftValue\":1,\"path\":[0],\"rightValue\":2}]\n", 77 | "expectedReturnCode": 1, 78 | "input": "" 79 | }, 80 | { 81 | "command": [ 82 | "./target/json-diff", 83 | "test_data/m1.json", 84 | "test_data/m1.json" 85 | ], 86 | "expectedOutput": "[]\n", 87 | "expectedReturnCode": 0, 88 | "input": "" 89 | }, 90 | { 91 | "command": [ 92 | "./target/json-diff", 93 | "test_data/m2.json", 94 | "test_data/m2.json" 95 | ], 96 | "expectedOutput": "[]\n", 97 | "expectedReturnCode": 0, 98 | "input": "" 99 | }, 100 | { 101 | "command": [ 102 | "./target/json-diff", 103 | "test_data/m1.json", 104 | "test_data/m2.json" 105 | ], 106 | "expectedOutput": "[{\"leftValue\":1,\"path\":[\"a\"],\"rightValue\":2},{\"path\":[\"b\"],\"rightValue\":1}]\n", 107 | "expectedReturnCode": 1, 108 | "input": "" 109 | }, 110 | { 111 | "command": [ 112 | "./target/json-diff", 113 | "test_data/a.json", 114 | "test_data/b.json" 115 | ], 116 | "expectedOutput": "[{\"leftValue\":\"a\",\"path\":[],\"rightValue\":\"b\"}]\n", 117 | "expectedReturnCode": 1, 118 | "input": "" 119 | } 120 | ] 121 | -------------------------------------------------------------------------------- /check-ppa-build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | # Check Launchpad PPA build status with detailed diagnostics 5 | 6 | PACKAGE="${1:-json-toolkit}" 7 | VERSION="${2:-}" 8 | PPA_NAME="${PPA_NAME:-ppa:code-faster/ppa}" 9 | 10 | # Extract PPA owner and name 11 | PPA_OWNER=$(echo "$PPA_NAME" | sed 's/ppa:\([^/]*\).*/\1/') 12 | PPA_REPO=$(echo "$PPA_NAME" | sed 's/ppa:[^/]*\/\(.*\)/\1/') 13 | 14 | PPA_URL="https://launchpad.net/~${PPA_OWNER}/+archive/ubuntu/${PPA_REPO}" 15 | BUILDERS_URL="https://launchpad.net/builders/" 16 | 17 | echo "===== PPA Build Diagnostics =====" 18 | echo "Package: $PACKAGE" 19 | echo "Version: ${VERSION:-latest}" 20 | echo "PPA: $PPA_NAME" 21 | echo "" 22 | 23 | # Check if curl is available 24 | if ! command -v curl &> /dev/null; then 25 | echo "ERROR: curl is required but not installed" 26 | exit 1 27 | fi 28 | 29 | echo "===== 1. Builder Queue Status =====" 30 | echo "Fetching: $BUILDERS_URL" 31 | echo "" 32 | 33 | BUILDERS_PAGE=$(curl -sL "$BUILDERS_URL") 34 | 35 | # Extract queue information 36 | echo "Current Build Farm Status:" 37 | echo "$BUILDERS_PAGE" | grep -A 5 "available build machine" | head -10 || echo "Could not parse builder status" 38 | echo "" 39 | 40 | # Check for specific architecture queues 41 | echo "Architecture Queue Status:" 42 | for ARCH in amd64 arm64 armhf i386 ppc64el s390x; do 43 | QUEUE_INFO=$(echo "$BUILDERS_PAGE" | grep -i "$ARCH" | head -1 || true) 44 | if [ -n "$QUEUE_INFO" ]; then 45 | echo " $ARCH: $QUEUE_INFO" 46 | fi 47 | done 48 | echo "" 49 | 50 | echo "===== 2. PPA Package Status =====" 51 | echo "Fetching: $PPA_URL/+packages" 52 | echo "" 53 | 54 | PPA_PACKAGES=$(curl -sL "$PPA_URL/+packages") 55 | 56 | # Find our package 57 | if echo "$PPA_PACKAGES" | grep -q "$PACKAGE"; then 58 | echo "Package '$PACKAGE' found in PPA" 59 | echo "" 60 | 61 | if [ -n "$VERSION" ]; then 62 | echo "Checking for version $VERSION:" 63 | VERSION_INFO=$(echo "$PPA_PACKAGES" | grep -A 20 "$PACKAGE" | grep -A 15 "$VERSION" | head -20 || echo "Version not found") 64 | echo "$VERSION_INFO" 65 | else 66 | echo "All versions:" 67 | echo "$PPA_PACKAGES" | grep -A 10 "$PACKAGE" | head -15 68 | fi 69 | else 70 | echo "Package '$PACKAGE' NOT found in PPA" 71 | fi 72 | echo "" 73 | 74 | echo "===== 3. Ubuntu Series (Distributions) =====" 75 | echo "Checking which Ubuntu releases have builds..." 76 | echo "" 77 | 78 | DISTS_URL="https://ppa.launchpadcontent.net/${PPA_OWNER}/${PPA_REPO}/ubuntu/dists/" 79 | DISTS_PAGE=$(curl -sL "$DISTS_URL" 2>/dev/null || echo "") 80 | 81 | if [ -n "$DISTS_PAGE" ]; then 82 | echo "Available Ubuntu series in this PPA:" 83 | echo "$DISTS_PAGE" | grep -oP 'href="\K[^/]+(?=/)' | grep -v 'Parent' | sort || echo "Could not parse distributions" 84 | else 85 | echo "Could not fetch distribution list" 86 | fi 87 | echo "" 88 | 89 | echo "===== 4. Build Records =====" 90 | echo "Fetching build records for $PACKAGE..." 91 | echo "" 92 | 93 | BUILDS_URL="$PPA_URL/+builds?build_text=$PACKAGE&build_state=all" 94 | BUILDS_PAGE=$(curl -sL "$BUILDS_URL") 95 | 96 | if echo "$BUILDS_PAGE" | grep -q "$PACKAGE"; then 97 | echo "Recent builds for $PACKAGE:" 98 | echo "" 99 | 100 | # Extract build information (this is approximate due to HTML structure) 101 | echo "$BUILDS_PAGE" | grep -A 5 "$PACKAGE" | grep -E "(amd64|arm64|armhf|i386|Pending|Building|Successfully built|Failed|noble|jammy|focal)" | head -30 102 | else 103 | echo "No build records found for $PACKAGE" 104 | fi 105 | echo "" 106 | 107 | echo "===== 5. Recommendations =====" 108 | echo "" 109 | 110 | # Check if version-specific build exists 111 | if [ -n "$VERSION" ]; then 112 | if echo "$PPA_PACKAGES" | grep -q "$VERSION"; then 113 | echo "✓ Version $VERSION found in PPA" 114 | 115 | # Check if it's still pending 116 | if echo "$PPA_PACKAGES" | grep -A 10 "$VERSION" | grep -qi "pending"; then 117 | echo "⏳ Build is PENDING" 118 | echo "" 119 | echo " This means:" 120 | echo " - Source was accepted by Launchpad" 121 | echo " - Build job created but waiting for available builder" 122 | echo " - Typical wait: 0-30 minutes, up to 2-3 hours during heavy load" 123 | echo "" 124 | echo " Check builder queue above for current backlog" 125 | echo " Visit $BUILDERS_URL for live status" 126 | elif echo "$PPA_PACKAGES" | grep -A 10 "$VERSION" | grep -qi "building"; then 127 | echo "🔨 Build is BUILDING" 128 | echo " Click the build link to see live logs" 129 | elif echo "$PPA_PACKAGES" | grep -A 10 "$VERSION" | grep -qi "published"; then 130 | echo "✅ Build is PUBLISHED" 131 | echo " Package should be available via apt" 132 | elif echo "$PPA_PACKAGES" | grep -A 10 "$VERSION" | grep -qi "fail"; then 133 | echo "❌ Build FAILED" 134 | echo " Check build logs for errors" 135 | fi 136 | else 137 | echo "⚠ Version $VERSION NOT found in PPA" 138 | echo " Either upload failed or version mismatch" 139 | fi 140 | fi 141 | echo "" 142 | 143 | echo "Direct links:" 144 | echo " PPA packages: $PPA_URL/+packages" 145 | echo " Build records: $BUILDS_URL" 146 | echo " Builder queue: $BUILDERS_URL" 147 | echo "" 148 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL = /bin/sh 2 | .DEFAULT_GOAL := all 3 | 4 | # Project metadata 5 | PACKAGE ?= json-toolkit 6 | VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null | sed 's/^v//' || echo "dev") 7 | 8 | # Tools and flags (user-overridable) 9 | GO ?= go 10 | PYTHON3 ?= python3 11 | INSTALL ?= install 12 | INSTALL_PROGRAM ?= $(INSTALL) 13 | INSTALL_DATA ?= $(INSTALL) -m 644 14 | MKDIR_P ?= mkdir -p 15 | RM ?= rm -f 16 | 17 | # Installation directories (GNU-style) 18 | prefix ?= /usr/local 19 | exec_prefix ?= $(prefix) 20 | bindir ?= $(exec_prefix)/bin 21 | datarootdir ?= $(prefix)/share 22 | docdir ?= $(datarootdir)/doc/$(PACKAGE) 23 | DESTDIR ?= 24 | 25 | # Build directories 26 | BUILDDIR := target 27 | 28 | # Quiet by default; make V=1 for verbose 29 | ifneq ($(V),1) 30 | Q := @ 31 | else 32 | Q := 33 | endif 34 | 35 | # Go build targets 36 | GO_SOURCES := src/json-diff.go src/logfmt-to-json.go src/json-to-logfmt.go 37 | GO_BINARIES := $(BUILDDIR)/json-diff $(BUILDDIR)/logfmt-to-json $(BUILDDIR)/json-to-logfmt 38 | 39 | all: $(GO_BINARIES) 40 | 41 | $(BUILDDIR)/json-diff: src/json-diff.go 42 | @echo " GO FMT $<" 43 | $(Q)$(GO) fmt $< 44 | @echo " GO BUILD $@" 45 | $(Q)$(MKDIR_P) $(BUILDDIR) 46 | $(Q)$(GO) build -o $@ $< 47 | 48 | $(BUILDDIR)/logfmt-to-json: src/logfmt-to-json.go 49 | @echo " GO FMT $<" 50 | $(Q)$(GO) fmt $< 51 | @echo " GO BUILD $@" 52 | $(Q)$(MKDIR_P) $(BUILDDIR) 53 | $(Q)$(GO) build -o $@ $< 54 | 55 | $(BUILDDIR)/json-to-logfmt: src/json-to-logfmt.go 56 | @echo " GO FMT $<" 57 | $(Q)$(GO) fmt $< 58 | @echo " GO BUILD $@" 59 | $(Q)$(MKDIR_P) $(BUILDDIR) 60 | $(Q)$(GO) build -o $@ $< 61 | 62 | # Dependencies (optional - only if user wants) 63 | .PHONY: dependencies 64 | dependencies: python-dependencies 65 | 66 | .PHONY: python-dependencies 67 | python-dependencies: requirements.txt 68 | @echo " PIP installing Python dependencies" 69 | $(Q)$(PYTHON3) -m pip install -r requirements.txt 70 | 71 | .PHONY: clean 72 | clean: 73 | @echo " CLEAN" 74 | $(Q)$(RM) -r $(BUILDDIR) 75 | 76 | .PHONY: mostlyclean 77 | mostlyclean: clean 78 | 79 | .PHONY: distclean 80 | distclean: clean 81 | 82 | .PHONY: lint 83 | lint: 84 | @echo " LINT" 85 | $(Q)./run-linter 86 | 87 | .PHONY: fmt format 88 | fmt format: 89 | @echo " FMT" 90 | $(Q)./run-formatter 91 | 92 | .PHONY: check test 93 | check test: all 94 | @echo " TEST" 95 | $(Q)./run-all-tests 96 | 97 | .PHONY: install 98 | install: all 99 | @echo " INSTALL" 100 | $(Q)$(MKDIR_P) $(DESTDIR)$(bindir) 101 | $(Q)$(MKDIR_P) $(DESTDIR)$(docdir) 102 | @echo "Installing Go binaries to $(DESTDIR)$(bindir)" 103 | $(Q)$(INSTALL_PROGRAM) $(BUILDDIR)/json-diff $(DESTDIR)$(bindir)/ 104 | $(Q)$(INSTALL_PROGRAM) $(BUILDDIR)/logfmt-to-json $(DESTDIR)$(bindir)/ 105 | $(Q)$(INSTALL_PROGRAM) $(BUILDDIR)/json-to-logfmt $(DESTDIR)$(bindir)/ 106 | @echo "Installing Python/Bash utilities from src/" 107 | $(Q)$(MAKE) -C src prefix=$(prefix) DESTDIR=$(DESTDIR) install 108 | @echo "Installing documentation" 109 | $(Q)$(INSTALL_DATA) README.md $(DESTDIR)$(docdir)/ 110 | $(Q)$(INSTALL_DATA) LICENSE $(DESTDIR)$(docdir)/ 111 | 112 | .PHONY: install-strip 113 | install-strip: install 114 | @echo " STRIP $(DESTDIR)$(bindir)/json-diff" 115 | $(Q)strip $(DESTDIR)$(bindir)/json-diff || true 116 | $(Q)strip $(DESTDIR)$(bindir)/logfmt-to-json || true 117 | $(Q)strip $(DESTDIR)$(bindir)/json-to-logfmt || true 118 | 119 | .PHONY: uninstall 120 | uninstall: 121 | @echo " UNINSTALL" 122 | $(Q)$(RM) $(DESTDIR)$(bindir)/json-diff 123 | $(Q)$(RM) $(DESTDIR)$(bindir)/logfmt-to-json 124 | $(Q)$(RM) $(DESTDIR)$(bindir)/json-to-logfmt 125 | $(Q)$(MAKE) -C src prefix=$(prefix) DESTDIR=$(DESTDIR) uninstall 126 | $(Q)$(RM) $(DESTDIR)$(docdir)/README.md 127 | $(Q)$(RM) $(DESTDIR)$(docdir)/LICENSE 128 | $(Q)-rmdir $(DESTDIR)$(docdir) 2>/dev/null || true 129 | 130 | .PHONY: dist 131 | dist: clean 132 | @echo " DIST $(PACKAGE)-$(VERSION).tar.gz" 133 | $(Q)git archive --format=tar --prefix=$(PACKAGE)-$(VERSION)/ HEAD | gzip -9 > $(PACKAGE)-$(VERSION).tar.gz 134 | 135 | # Debian packaging targets 136 | .PHONY: package 137 | package: 138 | @echo " PACKAGE Building Debian source package for $(VERSION)" 139 | $(Q)$(MKDIR_P) $(BUILDDIR)/bin 140 | $(Q)$(MAKE) -C src 141 | @echo " GBP Generating debian/changelog from git" 142 | $(Q)gbp dch --new-version=$(VERSION) --debian-tag='v%(version)s' --spawn-editor=never --git-author --distribution=noble 2>/dev/null || \ 143 | (echo "$(PACKAGE) ($(VERSION)) noble; urgency=medium" > debian/changelog && \ 144 | echo "" >> debian/changelog && \ 145 | echo " * Release $(VERSION)" >> debian/changelog && \ 146 | echo "" >> debian/changelog && \ 147 | echo " -- $(shell git config user.name) <$(shell git config user.email)> $(shell date -R)" >> debian/changelog) 148 | $(Q)tar czf $(BUILDDIR)/$(PACKAGE)_$(VERSION).orig.tar.gz src --transform "s#src#$(PACKAGE)-$(VERSION)#" 149 | $(Q)cd $(BUILDDIR) && tar xf $(PACKAGE)_$(VERSION).orig.tar.gz 150 | $(Q)cp -r debian $(BUILDDIR)/$(PACKAGE)-$(VERSION)/debian 151 | $(Q)cd $(BUILDDIR)/$(PACKAGE)-$(VERSION) && debuild -S -sa -d -nc -k"$(DEBSIGN_KEY)" 152 | 153 | .PHONY: publish 154 | publish: 155 | debsign -k $(DEBSIGN_KEY) ./$(BUILDDIR)/$(PACKAGE)_$(VERSION)_source.changes 156 | dput code-faster ./$(BUILDDIR)/$(PACKAGE)_$(VERSION)_source.changes 157 | 158 | .PHONY: help 159 | help: 160 | @echo "JSON Toolkit - Makefile targets" 161 | @echo "" 162 | @echo "Build targets:" 163 | @echo " all (default) Build all Go binaries" 164 | @echo " clean Remove build artifacts" 165 | @echo " fmt Format code (Go and Python)" 166 | @echo " lint Run linters" 167 | @echo "" 168 | @echo "Test targets:" 169 | @echo " test, check Run all tests" 170 | @echo "" 171 | @echo "Install targets:" 172 | @echo " install Install to $(prefix) (default: /usr/local)" 173 | @echo " install-strip Install and strip binaries" 174 | @echo " uninstall Remove installed files" 175 | @echo "" 176 | @echo "Packaging targets:" 177 | @echo " dist Create source tarball" 178 | @echo " package Build Debian package" 179 | @echo " publish Publish Debian package to PPA" 180 | @echo "" 181 | @echo "Variables:" 182 | @echo " prefix=$(prefix) Installation prefix" 183 | @echo " DESTDIR=$(DESTDIR) Staging directory for packaging" 184 | @echo " V=1 Enable verbose output" 185 | @echo " VERSION=$(VERSION) Package version" 186 | @echo "" 187 | @echo "Examples:" 188 | @echo " make" 189 | @echo " make test" 190 | @echo " make prefix=/usr install" 191 | @echo " make DESTDIR=/tmp/staging install" 192 | @echo " make VERSION=1.2.0 package" 193 | 194 | .PHONY: all 195 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | # Release Process 2 | 3 | This document describes the release process for JSON Toolkit using the automated `release.sh` script. 4 | 5 | ## Quick Start 6 | 7 | For a full release with all validation and publishing: 8 | 9 | ```bash 10 | export DEBSIGN_KEY=YOUR_GPG_KEY_ID 11 | ./release.sh 1.2.3 12 | ``` 13 | 14 | **Note**: All output is automatically logged to `logs/release-YYYYMMDD-HHMMSS.log` with timestamps to prevent collisions. No need to manually pipe to `tee`. 15 | 16 | ## Prerequisites 17 | 18 | 1. **Docker** - Required for clean environment testing 19 | 2. **Git** - For version control and tagging 20 | 3. **debuild, debsign, dput** - For Debian packaging (install `devscripts`) 21 | 4. **GPG Key** - For signing packages (set `DEBSIGN_KEY` environment variable) 22 | 23 | ## Release Pipeline Phases 24 | 25 | The release script executes six phases: 26 | 27 | ### Phase 1: Local Build and Test 28 | - Cleans previous builds 29 | - Compiles all Go programs 30 | - Runs full test suite locally 31 | - **Purpose**: Quick sanity check before expensive Docker operations 32 | 33 | ### Phase 2: Docker Package Build 34 | - Builds Ubuntu 22.04 Docker image with all dependencies 35 | - Compiles project in clean environment 36 | - Extracts built artifacts 37 | - **Purpose**: Ensure build works in clean environment 38 | 39 | ### Phase 3: Docker Install and Test 40 | - Creates fresh Ubuntu 22.04 container 41 | - Installs json-toolkit from source 42 | - Runs complete test suite 43 | - Performs smoke tests on key utilities 44 | - **Purpose**: Verify installation and functionality in production-like environment 45 | 46 | ### Phase 4: Git Publishing 47 | - Commits any outstanding changes (with confirmation) 48 | - Creates version tag (e.g., `v1.2.3`) 49 | - Pushes commits and tags to origin 50 | - **Purpose**: Version control and GitHub release 51 | 52 | ### Phase 5: PPA Publishing 53 | - Builds Debian source package 54 | - Signs package with GPG key 55 | - Uploads to PPA via dput 56 | - **Purpose**: Make package available via apt 57 | 58 | ### Phase 6: PPA Verification 59 | - Waits for PPA processing (2 minutes) 60 | - Creates Docker container that installs from PPA 61 | - Verifies installation and runs smoke tests 62 | - **Purpose**: Ensure published package works correctly 63 | 64 | ## Usage Examples 65 | 66 | ### Full Release 67 | ```bash 68 | # Set your GPG key for signing 69 | export DEBSIGN_KEY=ABCD1234 70 | 71 | # Run complete release pipeline 72 | ./release.sh 1.2.3 73 | ``` 74 | 75 | ### Testing Before Release 76 | 77 | #### Local testing only (fastest) 78 | ```bash 79 | ./release.sh --local-only 1.2.3 80 | ``` 81 | 82 | #### Docker testing only (no publishing) 83 | ```bash 84 | ./release.sh --docker-only 1.2.3 85 | ``` 86 | 87 | #### Everything except PPA 88 | ```bash 89 | ./release.sh --skip-ppa 1.2.3 90 | ``` 91 | 92 | ### Skip Specific Phases 93 | ```bash 94 | # Skip Docker build (if already validated) 95 | ./release.sh --skip-phase 2 1.2.3 96 | 97 | # Skip multiple phases 98 | ./release.sh --skip-phase 2 --skip-phase 3 1.2.3 99 | ``` 100 | 101 | ## Environment Variables 102 | 103 | - **DEBSIGN_KEY** (required for PPA): Your GPG key ID for signing packages 104 | - **PPA_NAME** (optional): Override default PPA (default: `ppa:code-faster/ppa`) 105 | - **DEBFULLNAME** (optional): Full name for debian/changelog 106 | - **DEBEMAIL** (optional): Email for debian/changelog 107 | 108 | Example: 109 | ```bash 110 | export DEBSIGN_KEY=0x1234ABCD 111 | export DEBFULLNAME="Tyler Adams" 112 | export DEBEMAIL="tyler@example.com" 113 | export PPA_NAME="ppa:myteam/json-toolkit" 114 | ./release.sh 1.2.3 115 | ``` 116 | 117 | ## Troubleshooting 118 | 119 | ### PPA Verification Fails 120 | 121 | If Phase 6 fails, the package may not be processed yet. PPA processing can take 5-30 minutes. Try: 122 | 123 | 1. Wait 10-15 minutes 124 | 2. Manually verify: `docker build -f -t json-toolkit-verify .` 125 | 3. Check PPA status: https://launchpad.net/~code-faster/+archive/ubuntu/ppa 126 | 127 | ### Docker Errors 128 | 129 | If Docker phases fail: 130 | 131 | 1. Ensure Docker daemon is running: `sudo systemctl start docker` 132 | 2. Check Docker permissions: `sudo usermod -aG docker $USER` (then logout/login) 133 | 3. Clean Docker images: `docker system prune -a` 134 | 135 | ### Git Push Fails 136 | 137 | If you don't have push access or tag already exists: 138 | 139 | 1. Check remote: `git remote -v` 140 | 2. Delete local tag: `git tag -d v1.2.3` 141 | 3. Delete remote tag: `git push origin :refs/tags/v1.2.3` 142 | 143 | ### GPG Signing Fails 144 | 145 | If package signing fails: 146 | 147 | 1. List keys: `gpg --list-secret-keys --keyid-format LONG` 148 | 2. Export key: `export DEBSIGN_KEY=YOUR_KEY_ID` 149 | 3. Test signing: `debsign -k $DEBSIGN_KEY .changes` 150 | 151 | ## Manual Release Steps 152 | 153 | If the automated script fails, you can run steps manually: 154 | 155 | ```bash 156 | # 1. Local build and test 157 | make clean && make && make test 158 | 159 | # 2. Update version 160 | vim Makefile # Update VERSION 161 | dch -v 1.2.3 -D focal "Release version 1.2.3" 162 | 163 | # 3. Git operations 164 | git add -A 165 | git commit -m "Release version 1.2.3" 166 | git tag -a v1.2.3 -m "Release version 1.2.3" 167 | git push origin master 168 | git push origin v1.2.3 169 | 170 | # 4. Build and publish package 171 | make package 172 | make publish DEBSIGN_KEY=YOUR_KEY_ID 173 | 174 | # 5. Wait and verify 175 | # Wait 15-30 minutes, then test: 176 | docker run -it ubuntu:22.04 177 | apt update 178 | apt install software-properties-common 179 | add-apt-repository ppa:code-faster/ppa 180 | apt update 181 | apt install json-toolkit 182 | json-diff --help 183 | ``` 184 | 185 | ## Post-Release Checklist 186 | 187 | After successful release: 188 | 189 | - [ ] Verify GitHub tag created: https://github.com/tyleradams/json-toolkit/tags 190 | - [ ] Check PPA build status: https://launchpad.net/~code-faster/+archive/ubuntu/ppa 191 | - [ ] Test installation on fresh Ubuntu VM/container 192 | - [ ] Update any documentation with new version number 193 | - [ ] Announce release (if applicable) 194 | 195 | ## Versioning 196 | 197 | Follow [Semantic Versioning](https://semver.org/): 198 | 199 | - **MAJOR.MINOR.PATCH** (e.g., 1.2.3) 200 | - **MAJOR**: Breaking changes 201 | - **MINOR**: New features, backwards compatible 202 | - **PATCH**: Bug fixes, backwards compatible 203 | 204 | ## Release Frequency 205 | 206 | - **Patch releases**: As needed for bug fixes 207 | - **Minor releases**: Monthly or when new features are ready 208 | - **Major releases**: When breaking changes are necessary 209 | 210 | ## Rollback 211 | 212 | If a release has issues: 213 | 214 | 1. **Git**: Create hotfix from previous tag 215 | 2. **PPA**: Upload new version (Launchpad doesn't allow deletions) 216 | 3. **Users**: Can pin to previous version: `apt install json-toolkit=1.2.2` 217 | -------------------------------------------------------------------------------- /src/json-diff.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "encoding/json" 4 | import "errors" 5 | import "flag" 6 | import "fmt" 7 | import "io/ioutil" 8 | import "os" 9 | import "path/filepath" 10 | import "reflect" 11 | 12 | func check(e error) { 13 | if e != nil { 14 | panic(e) 15 | } 16 | } 17 | 18 | func min(a int, b int) int { 19 | if a < b { 20 | return a 21 | } 22 | 23 | return b 24 | } 25 | 26 | type compare func(path []interface{}, v1 interface{}, v2 interface{}) []map[string]interface{} 27 | 28 | func compareSimple(path []interface{}, v1 interface{}, v2 interface{}) []map[string]interface{} { 29 | if v1 == v2 { 30 | return []map[string]interface{}{} 31 | } 32 | 33 | return []map[string]interface{}{ 34 | { 35 | "path": path, 36 | "leftValue": v1, 37 | "rightValue": v2, 38 | }, 39 | } 40 | } 41 | 42 | func compareSlice(path []interface{}, v1 interface{}, v2 interface{}) []map[string]interface{} { 43 | slice1 := v1.([]interface{}) 44 | slice2 := v2.([]interface{}) 45 | 46 | m := []map[string]interface{}{} 47 | 48 | for i := 0; i < min(len(slice1), len(slice2)); i++ { 49 | i1 := slice1[i] 50 | i2 := slice2[i] 51 | new_path := make([]interface{}, len(path)) 52 | copy(new_path, path) 53 | m = append(m, compareObject(append(new_path, i), i1, i2)...) 54 | } 55 | 56 | if len(slice1) > len(slice2) { 57 | for i := len(slice2); i < len(slice1); i++ { 58 | new_path := make([]interface{}, len(path)) 59 | copy(new_path, path) 60 | m = append(m, map[string]interface{}{ 61 | "path": append(new_path, i), 62 | "leftValue": slice1[i], 63 | }) 64 | } 65 | } 66 | 67 | if len(slice2) > len(slice1) { 68 | for i := len(slice1); i < len(slice2); i++ { 69 | new_path := make([]interface{}, len(path)) 70 | copy(new_path, path) 71 | m = append(m, map[string]interface{}{ 72 | "path": append(new_path, i), 73 | "rightValue": slice2[i], 74 | }) 75 | } 76 | } 77 | return m 78 | } 79 | 80 | func compareMap(path []interface{}, v1 interface{}, v2 interface{}) []map[string]interface{} { 81 | 82 | map1 := v1.(map[string]interface{}) 83 | map2 := v2.(map[string]interface{}) 84 | 85 | diff := []map[string]interface{}{} 86 | 87 | for key := range map1 { 88 | _, keyInMap2 := map2[key] 89 | if keyInMap2 { 90 | new_path := make([]interface{}, len(path)) 91 | copy(new_path, path) 92 | diff = append(diff, compareObject(append(new_path, key), map1[key], map2[key])...) 93 | } else { 94 | new_path := make([]interface{}, len(path)) 95 | copy(new_path, path) 96 | diff = append(diff, map[string]interface{}{ 97 | "path": append(new_path, key), 98 | "leftValue": map1[key], 99 | }) 100 | } 101 | } 102 | 103 | for key := range map2 { 104 | _, keyInMap1 := map1[key] 105 | if !keyInMap1 { 106 | new_path := make([]interface{}, len(path)) 107 | copy(new_path, path) 108 | diff = append(diff, map[string]interface{}{ 109 | "path": append(new_path, key), 110 | "rightValue": map2[key], 111 | }) 112 | } 113 | } 114 | 115 | return diff 116 | } 117 | 118 | func compareObject(path []interface{}, object1 interface{}, object2 interface{}) []map[string]interface{} { 119 | // nil does not have a reflection type kind, so we need to check for this case first 120 | if object1 == nil || object2 == nil { 121 | return compareSimple(path, object1, object2) 122 | } 123 | 124 | // This cannot be defined outside because it makes an initialization loop 125 | var compares = map[reflect.Kind]compare{ 126 | reflect.Float64: compareSimple, 127 | reflect.Bool: compareSimple, 128 | reflect.String: compareSimple, 129 | reflect.Slice: compareSlice, 130 | reflect.Map: compareMap, 131 | } 132 | 133 | var type1 reflect.Kind = reflect.TypeOf(object1).Kind() 134 | var type2 reflect.Kind = reflect.TypeOf(object2).Kind() 135 | 136 | if type1 != type2 { 137 | return compareSimple(path, object1, object2) 138 | } else if val, ok := compares[type1]; type1 == type2 && ok { 139 | return val(path, object1, object2) 140 | } else { 141 | panic(errors.New(fmt.Sprintf("IncompleteIfTree:type1:%v:type2:%v", type1, type2))) 142 | } 143 | } 144 | 145 | func main() { 146 | app := filepath.Base(os.Args[0]) 147 | 148 | fs := flag.NewFlagSet(app, flag.ContinueOnError) 149 | fs.SetOutput(os.Stderr) 150 | 151 | fs.Usage = func() { 152 | fmt.Fprintf(os.Stderr, "%s - compare two JSON files and report differences\n\n", app) 153 | fmt.Fprintf(os.Stderr, "Usage:\n %s [flags] FILE1 FILE2\n\n", app) 154 | fmt.Fprintln(os.Stderr, "Overview:") 155 | fmt.Fprintln(os.Stderr, " json-diff reports differences between two JSON files as JSON.") 156 | fmt.Fprintln(os.Stderr, "") 157 | fmt.Fprintln(os.Stderr, " For each difference, json-diff reports the path, the value in FILE1") 158 | fmt.Fprintln(os.Stderr, " (leftValue) if present, and the value in FILE2 (rightValue) if present.") 159 | fmt.Fprintln(os.Stderr, "") 160 | fmt.Fprintln(os.Stderr, "Path Notation:") 161 | fmt.Fprintln(os.Stderr, " The path is an array of numbers and strings. Each number refers to an") 162 | fmt.Fprintln(os.Stderr, " array index and each string refers to an object key.") 163 | fmt.Fprintln(os.Stderr, "") 164 | fmt.Fprintln(os.Stderr, " Example: [0, \"a\", 2] refers to \"foo\" in:") 165 | fmt.Fprintln(os.Stderr, " [{\"a\": [null, null, \"foo\"]}]") 166 | fmt.Fprintln(os.Stderr, "") 167 | fmt.Fprintln(os.Stderr, "Exit Codes:") 168 | fmt.Fprintln(os.Stderr, " 0 - Files are identical") 169 | fmt.Fprintln(os.Stderr, " 1 - Files differ") 170 | fmt.Fprintln(os.Stderr, " 2 - Error occurred") 171 | fmt.Fprintln(os.Stderr, "") 172 | fmt.Fprintln(os.Stderr, "Flags:") 173 | fmt.Fprintln(os.Stderr, " -h, -help show help") 174 | fmt.Fprintln(os.Stderr, "") 175 | fmt.Fprintln(os.Stderr, "Examples:") 176 | fmt.Fprintf(os.Stderr, " %s old.json new.json\n", app) 177 | fmt.Fprintf(os.Stderr, " %s config1.json config2.json | jq '.'\n", app) 178 | } 179 | 180 | // Support GNU-style --help explicitly 181 | for _, a := range os.Args[1:] { 182 | if a == "--help" { 183 | fs.Usage() 184 | os.Exit(0) 185 | } 186 | } 187 | 188 | if err := fs.Parse(os.Args[1:]); err != nil { 189 | if errors.Is(err, flag.ErrHelp) { 190 | os.Exit(0) 191 | } 192 | os.Exit(2) 193 | } 194 | 195 | if fs.NArg() != 2 { 196 | fmt.Fprintln(os.Stderr, "error: requires exactly 2 file arguments") 197 | fs.Usage() 198 | os.Exit(2) 199 | } 200 | 201 | file1Path := fs.Arg(0) 202 | file2Path := fs.Arg(1) 203 | 204 | file1, err := ioutil.ReadFile(file1Path) 205 | if err != nil { 206 | fmt.Fprintf(os.Stderr, "error reading %s: %v\n", file1Path, err) 207 | os.Exit(2) 208 | } 209 | 210 | file2, err := ioutil.ReadFile(file2Path) 211 | if err != nil { 212 | fmt.Fprintf(os.Stderr, "error reading %s: %v\n", file2Path, err) 213 | os.Exit(2) 214 | } 215 | 216 | var object1 interface{} 217 | var object2 interface{} 218 | 219 | err = json.Unmarshal(file1, &object1) 220 | if err != nil { 221 | fmt.Fprintf(os.Stderr, "error parsing %s: %v\n", file1Path, err) 222 | os.Exit(2) 223 | } 224 | 225 | err = json.Unmarshal(file2, &object2) 226 | if err != nil { 227 | fmt.Fprintf(os.Stderr, "error parsing %s: %v\n", file2Path, err) 228 | os.Exit(2) 229 | } 230 | 231 | diff := compareObject([]interface{}{}, object1, object2) 232 | 233 | output, _ := json.Marshal(diff) 234 | fmt.Println(string(output)) 235 | 236 | if len(diff) == 0 { 237 | os.Exit(0) 238 | } 239 | 240 | os.Exit(1) 241 | } 242 | -------------------------------------------------------------------------------- /src/json-sql: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import datetime 4 | import decimal 5 | import json 6 | import os 7 | import platform 8 | import sqlite3 9 | import sys 10 | 11 | import click 12 | import cx_Oracle 13 | import pymysql 14 | import psycopg2 15 | 16 | CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} 17 | 18 | 19 | def jsonable(o): 20 | if type(o) in [str, int, float, bool, type(None)]: 21 | return o 22 | elif type(o) in [datetime.datetime]: 23 | return o.isoformat() 24 | elif type(o) in [list]: 25 | return [jsonable(e) for e in o] 26 | elif type(o) in [dict]: 27 | return {str(k): jsonable(o[k]) for k in o} 28 | elif type(o) in [decimal.Decimal]: 29 | return float(o) 30 | else: 31 | raise Exception( 32 | "Cannot make object jsonable:{}:{}".format(str(type(o)), str(o))) 33 | 34 | 35 | def make_table(data, columns): 36 | def make_dict(keys, values): 37 | if len(keys) != len(values): 38 | raise Exception("Cannot make_dict: {}".format( 39 | {"keys": keys, "values": values})) 40 | return {keys[i]: values[i] for i in range(len(keys))} 41 | 42 | return [make_dict(columns, row) for row in data] 43 | 44 | 45 | class DB: 46 | connection = None 47 | 48 | def execute_query(self, query_string): 49 | def cursor_has_results(cursor): 50 | return bool(cursor.description) 51 | 52 | cursor = self.connection.cursor() 53 | cursor.execute(query_string) 54 | self.connection.commit() 55 | 56 | if cursor_has_results(cursor): 57 | return [list(r) for r in cursor.fetchall()] 58 | return cursor.rowcount 59 | 60 | def read_table(self, table_name): 61 | columns = self._column_names(table_name) 62 | data = self.execute_query("select * from {};".format(table_name)) 63 | return make_table(data, columns) 64 | 65 | def read_tables(self): 66 | return {name: self.read_table(name) for name in self._table_names()} 67 | 68 | def read(self): 69 | return jsonable(self.read_tables()) 70 | 71 | @staticmethod 72 | def get_type(command): 73 | TYPES = [SQLITE3, PSQL, MYSQL, ORACLEADB] 74 | possible_t = [t for t in TYPES if t.__name__ == command.upper()] 75 | 76 | if possible_t == []: 77 | raise Exception("No type found for: {}".format(command)) 78 | elif len(possible_t) > 1: 79 | raise Exception("More than one type found for: {}".format(command)) 80 | 81 | return possible_t[0] 82 | 83 | def _column_names(self, table_name): 84 | return NotImplementedError 85 | 86 | def _table_names(self): 87 | return NotImplementedError 88 | 89 | 90 | class ORACLEADB(DB): 91 | def __init__(self, user, password, schema, tnsname, lib_dir=None): 92 | if os.environ.get('TNS_ADMIN', '') == "": 93 | raise Exception("Please set the TNS_ADMIN environment variable to the path of your Autonomous DB wallet.") 94 | if lib_dir is not None and platform.platform()[:6] == 'Darwin': 95 | cx_Oracle.init_oracle_client(lib_dir) 96 | self.connection = cx_Oracle.connect(user, password, tnsname) 97 | self.schema = schema 98 | 99 | def read_table(self, table_name): 100 | columns = self._column_names(table_name) 101 | data = self.execute_query("select * from {}".format(table_name)) 102 | return make_table(data, columns) 103 | 104 | def _column_names(self, table_name): 105 | names = self.execute_query( 106 | "select column_name from all_tab_columns where table_name = '{}'".format(table_name)) 107 | return [n[0] for n in names] 108 | 109 | def _table_names(self): 110 | return [t[0] for t in self.execute_query("select table_name from all_tables where num_rows > 1 and tablespace_name = '{}'".format(self.schema))] 111 | 112 | 113 | class SQLITE3(DB): 114 | def __init__(self, filename): 115 | self.connection = sqlite3.connect(filename) 116 | 117 | def _column_names(self, table_name): 118 | columns = self.execute_query( 119 | "pragma table_info({});".format(table_name)) 120 | return [column[1] for column in columns] 121 | 122 | def _table_names(self): 123 | master_table = self.read_table("sqlite_master") 124 | return [e["name"] for e in master_table if e["type"] == "table"] 125 | 126 | 127 | class PSQL(DB): 128 | def __init__(self, user_or_connection_string, password=None, host=None, port=None, database=None): 129 | if password is None: 130 | connection_string = user_or_connection_string 131 | self.connection = psycopg2.connect(connection_string) 132 | else: 133 | user = user_or_connection_string 134 | self.connection = psycopg2.connect( 135 | user=user, 136 | password=password, 137 | host=host, 138 | port=port, 139 | database=database) 140 | 141 | def _column_names(self, table_name): 142 | names = self.execute_query( 143 | "select column_name from information_schema.columns where table_name = '{}';".format(table_name)) 144 | return [n[0] for n in names] 145 | 146 | def _table_names(self): 147 | return [t[0] for t in self.execute_query("select table_name from information_schema.tables where table_schema = 'public';")] 148 | 149 | 150 | class MYSQL(DB): 151 | def __init__(self, user, password, host, database): 152 | self.database = database 153 | self.connection = pymysql.connect( 154 | host, 155 | user, 156 | password, 157 | database) 158 | 159 | def _column_names(self, table_name): 160 | return self.execute_query("select column_name from information_schema.columns where table_name = '{}';".format(table_name))[0] 161 | 162 | def _table_names(self): 163 | return [t[0] for t in self.execute_query("select table_name from information_schema.tables where table_schema = '{}';".format(self.database))] 164 | 165 | 166 | @click.group(context_settings=CONTEXT_SETTINGS) 167 | @click.version_option("1.1.0", prog_name="json-sql") 168 | def cli(): 169 | """ 170 | Read from or write to SQL databases as JSON. 171 | 172 | Supports PostgreSQL, MySQL, SQLite3, and Oracle Autonomous Database. 173 | """ 174 | pass 175 | 176 | 177 | @cli.command() 178 | @click.argument('db_type', type=click.Choice(['psql', 'mysql', 'sqlite3', 'oracleadb'], case_sensitive=False)) 179 | @click.argument('credentials', nargs=-1, required=True) 180 | def read(db_type, credentials): 181 | """ 182 | Read entire database and output as JSON. 183 | 184 | \b 185 | DB_TYPE: psql, mysql, sqlite3, or oracleadb 186 | 187 | \b 188 | CREDENTIALS (varies by DB_TYPE): 189 | psql: user password host port database 190 | OR connection-url 191 | mysql: user password host port database 192 | sqlite3: filename 193 | oracleadb: user password schemaname tnsname [path_to_instant_client] 194 | 195 | \b 196 | Examples: 197 | json-sql read psql user password localhost 5432 mydb 198 | json-sql read psql postgres://user:pass@localhost:5432/db 199 | json-sql read mysql user password localhost 3306 mydb 200 | json-sql read sqlite3 database.db 201 | json-sql read oracleadb user pass SCHEMA tns_name 202 | """ 203 | try: 204 | db = DB.get_type(db_type)(*credentials) 205 | print(json.dumps(db.read())) 206 | except Exception as e: 207 | click.echo(f"Error: {e}", err=True) 208 | sys.exit(1) 209 | 210 | 211 | @cli.command() 212 | @click.argument('db_type', type=click.Choice(['psql', 'mysql', 'sqlite3', 'oracleadb'], case_sensitive=False)) 213 | @click.argument('credentials', nargs=-1, required=True) 214 | def query(db_type, credentials): 215 | """ 216 | Execute queries from stdin (JSON array) and output results as JSON. 217 | 218 | Reads a JSON array of query strings from stdin and executes them. 219 | For queries with results, outputs the result rows. For queries without 220 | results (INSERT, UPDATE, DELETE), outputs the number of rows affected. 221 | 222 | \b 223 | DB_TYPE: psql, mysql, sqlite3, or oracleadb 224 | 225 | \b 226 | CREDENTIALS: Same as read command (varies by DB_TYPE) 227 | 228 | \b 229 | Examples: 230 | echo '["SELECT * FROM users WHERE id = 1"]' | json-sql query psql user pass localhost 5432 db 231 | echo '["INSERT INTO logs VALUES (1, 2, 3)"]' | json-sql query sqlite3 db.sqlite 232 | cat queries.json | json-sql query mysql user pass localhost 3306 db 233 | """ 234 | try: 235 | db = DB.get_type(db_type)(*credentials) 236 | queries = json.load(sys.stdin) 237 | results = [db.execute_query(q) for q in queries] 238 | print(json.dumps(results, default=jsonable)) 239 | except json.JSONDecodeError as e: 240 | click.echo(f"Error: Invalid JSON input: {e}", err=True) 241 | sys.exit(1) 242 | except Exception as e: 243 | click.echo(f"Error: {e}", err=True) 244 | sys.exit(1) 245 | 246 | 247 | if __name__ == "__main__": 248 | cli() 249 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # JSON Toolkit 2 | 3 | Fast, composable command-line utilities for converting, querying, validating, and transforming JSON and related data formats. 4 | 5 | [![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](LICENSE) 6 | 7 | ## Overview 8 | 9 | JSON Toolkit is a comprehensive suite of command-line utilities designed to make it easy to work with JSON and other data formats. Whether you need to convert data, manipulate it, or extract information, this toolkit has you covered. 10 | 11 | ### Features 12 | 13 | - **Format conversion**: Bidirectional conversion between JSON and CSV, XML, YAML, TOML, HTML, logfmt, environment variables, and more 14 | - **JSON manipulation**: Format, diff, schema generation, and data transformation 15 | - **Database integration**: Read from and write to PostgreSQL, MySQL, SQLite3, and Oracle Autonomous Database 16 | - **Unix-friendly**: All tools follow Unix philosophy—read from stdin, write to stdout, composable with pipes 17 | - **Lightweight**: Simple utilities with minimal dependencies, designed for scripting and automation 18 | 19 | ## Installation 20 | 21 | ### Via APT (Debian/Ubuntu) 22 | 23 | ```bash 24 | sudo add-apt-repository ppa:code-faster/ppa 25 | sudo apt update 26 | sudo apt install json-toolkit 27 | ``` 28 | 29 | ### Build from Source 30 | 31 | #### Prerequisites 32 | 33 | - Bash 34 | - Go (golang-go) 35 | - Python 3 36 | - [jq](https://stedolan.github.io/jq/) (>= 1.5) 37 | 38 | #### Build Steps 39 | 40 | ```bash 41 | git clone https://github.com/tyleradams/json-toolkit.git 42 | cd json-toolkit 43 | make 44 | make test 45 | sudo make install 46 | ``` 47 | 48 | ## Quick Start 49 | 50 | ```bash 51 | # Convert CSV to JSON 52 | csv-to-json < data.csv > data.json 53 | 54 | # Pretty-print JSON 55 | json-format < data.json 56 | 57 | # Compare two JSON files 58 | json-diff file1.json file2.json 59 | 60 | # Query PostgreSQL and output as JSON 61 | json-sql read psql user password localhost 5432 mydb 62 | 63 | # Convert JSON to YAML 64 | json-to-yaml < config.json > config.yaml 65 | 66 | # Generate JSON schema from sample data 67 | json-make-schema < sample.json > schema.json 68 | ``` 69 | 70 | ## Usage 71 | 72 | All utilities follow the same pattern: 73 | - Read from stdin (or files where specified) 74 | - Write to stdout 75 | - Return exit code 0 on success, non-zero on failure 76 | - Provide `--help` or `-h` for detailed usage 77 | 78 | ### Format Conversion Tools 79 | 80 | #### To JSON 81 | 82 | - `csv-to-json` - Convert CSV to JSON array 83 | - `dsv-to-json` - Convert delimiter-separated values to JSON 84 | - `xml-to-json` - Convert XML to JSON 85 | - `yaml-to-json` - Convert YAML to JSON 86 | - `toml-to-json` - Convert TOML to JSON 87 | - `html-to-json` - Convert HTML to JSON 88 | - `logfmt-to-json` - Convert logfmt to JSON 89 | - `env-to-json` - Convert environment variables to JSON 90 | - `diff-to-json` - Convert unidiff patches to JSON 91 | - `binary-to-json` - Convert binary data to JSON 92 | - `python-to-json-ast` - Parse Python code to JSON AST 93 | 94 | #### From JSON 95 | 96 | - `json-to-csv` - Convert JSON to CSV 97 | - `json-to-dsv` - Convert JSON to delimiter-separated values 98 | - `json-to-xml` - Convert JSON to XML 99 | - `json-to-yaml` - Convert JSON to YAML 100 | - `json-to-env` - Convert JSON to environment variable format 101 | - `json-to-logfmt` - Convert JSON to logfmt 102 | - `json-to-binary` - Convert JSON to binary 103 | 104 | ### JSON Manipulation Tools 105 | 106 | - `json-format` - Pretty-print or minify JSON 107 | - `json-diff` - Compare two JSON files and output structured differences 108 | - `json-make-schema` - Generate JSON schema from input data 109 | - `json-table-to-objs` - Convert JSON table format to objects 110 | - `json-objs-to-table` - Convert JSON objects to table format 111 | 112 | ### Database Tools 113 | 114 | - `json-sql` - Read from or write to SQL databases (PostgreSQL, MySQL, SQLite3, Oracle ADB) 115 | 116 | ```bash 117 | # Read entire database as JSON 118 | json-sql read psql user password localhost 5432 database 119 | 120 | # Execute queries from JSON array 121 | echo '["SELECT * FROM users WHERE id = 1"]' | json-sql query psql user password localhost 5432 database 122 | 123 | # SQLite example 124 | json-sql read sqlite3 mydata.db 125 | 126 | # MySQL example 127 | json-sql read mysql user password localhost 3306 database 128 | ``` 129 | 130 | ### Other Tools 131 | 132 | - `json-run` - Execute code based on JSON input 133 | - `pjito` - "Python JSON in, text out" - template renderer 134 | - `json-to-plot` - Generate plots from JSON data 135 | 136 | ## Examples 137 | 138 | ### Convert and transform data 139 | 140 | ```bash 141 | # CSV to JSON, filter with jq, convert to YAML 142 | csv-to-json < users.csv | jq '.[] | select(.active == true)' | json-to-yaml > active-users.yaml 143 | 144 | # Read PostgreSQL table, extract specific fields 145 | json-sql read psql user pass localhost 5432 db | jq '.users[] | {id, email}' > user-emails.json 146 | 147 | # Compare two configuration files 148 | json-diff config-old.json config-new.json | json-format 149 | ``` 150 | 151 | ### Database operations 152 | 153 | ```bash 154 | # Dump entire database to JSON 155 | json-sql read psql postgres password localhost 5432 mydb > backup.json 156 | 157 | # Insert data from JSON 158 | echo '["INSERT INTO logs (message, level) VALUES ('"'"'test'"'"', '"'"'info'"'"')"]' | \ 159 | json-sql query sqlite3 app.db 160 | ``` 161 | 162 | ### Schema generation and validation 163 | 164 | ```bash 165 | # Generate schema from sample data 166 | json-make-schema < sample-data.json > schema.json 167 | 168 | # Use the schema for documentation or validation 169 | ``` 170 | 171 | ## Exit Codes 172 | 173 | All utilities follow standard exit code conventions: 174 | - `0` - Success 175 | - `1` - Runtime or I/O error (file not found, parse error, etc.) 176 | - `2` - Usage error (invalid arguments or options) 177 | 178 | Some utilities have specialized exit codes (see individual `--help` for details). 179 | 180 | ## Development 181 | 182 | ### Build 183 | 184 | ```bash 185 | make # Build all components 186 | make clean # Remove build artifacts 187 | ``` 188 | 189 | ### Test 190 | 191 | ```bash 192 | make test # Run all tests 193 | ./run-json-diff-tests # Test json-diff specifically 194 | ./run-all-tests # Complete test suite 195 | ``` 196 | 197 | ### Code Quality 198 | 199 | ```bash 200 | make fmt # Format code (Go and Python) 201 | make lint # Run linters 202 | ``` 203 | 204 | ### Install Locally 205 | 206 | ```bash 207 | make install # Install to /usr/local/bin (requires sudo) 208 | make prefix=/custom/path install # Install to custom location 209 | ``` 210 | 211 | ## Project Structure 212 | 213 | ``` 214 | json-toolkit/ 215 | ├── src/ # Source files for all utilities 216 | │ ├── *.go # Go programs (json-diff) 217 | │ ├── *-to-json # Python conversion utilities 218 | │ ├── json-to-* # Python output converters 219 | │ └── json-* # JSON manipulation tools 220 | ├── test_data/ # Test fixtures and expected results 221 | ├── debian/ # Debian packaging files 222 | ├── Makefile # Root build configuration 223 | └── README.md # This file 224 | ``` 225 | 226 | ## Dependencies 227 | 228 | ### Runtime Dependencies 229 | 230 | - Python 3 231 | - jq (>= 1.5) 232 | - Go (for json-diff) 233 | 234 | ### Python Libraries 235 | 236 | The following Python packages are required (automatically installed with the Debian package): 237 | 238 | - `psycopg2-binary` - PostgreSQL support 239 | - `PyMySQL` - MySQL support 240 | - `cx_Oracle` - Oracle Database support 241 | - `PyYAML` - YAML format support 242 | - `xmltodict` - XML format support 243 | - `unidiff` - Diff format support 244 | - `logfmt` - Logfmt format support 245 | - `genson` - JSON schema generation 246 | - `ast2json` - Python AST parsing 247 | 248 | ## Contributing 249 | 250 | Contributions are welcome! Here's how to get started: 251 | 252 | 1. Fork the repository 253 | 2. Create a feature branch (`git checkout -b feature/amazing-feature`) 254 | 3. Make your changes 255 | 4. Run tests (`make test`) 256 | 5. Commit your changes (`git commit -m 'Add amazing feature'`) 257 | 6. Push to your branch (`git push origin feature/amazing-feature`) 258 | 7. Open a Pull Request 259 | 260 | Please ensure: 261 | - All tests pass 262 | - Code follows existing style (run `make fmt` and `make lint`) 263 | - New utilities include help text and examples 264 | - Documentation is updated 265 | 266 | ## Versioning 267 | 268 | This project uses [Semantic Versioning](https://semver.org/). For available versions, see the [releases page](https://github.com/tyleradams/json-toolkit/releases). 269 | 270 | ## License 271 | 272 | This project is licensed under the GNU General Public License v2.0 - see the [LICENSE](LICENSE) file for details. 273 | 274 | ## Feedback and Support 275 | 276 | - **Issues**: Report bugs or request features on our [GitHub Issues](https://github.com/tyleradams/json-toolkit/issues) page 277 | - **Questions**: For usage questions, please open a discussion or issue 278 | 279 | ## Acknowledgements 280 | 281 | Special thanks to: 282 | - The jq project for inspiration on composable JSON tools 283 | - All contributors and users who have provided feedback 284 | - The open-source community for the excellent libraries this project builds upon 285 | 286 | --- 287 | 288 | *"the best opensource converter I've found across the Internet"* - dene14 289 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | # ===== LOGGING SETUP ===== 6 | TIMESTAMP=$(date +%Y%m%d-%H%M%S) 7 | LOG_DIR="logs" 8 | LOG_FILE="$LOG_DIR/release-${TIMESTAMP}.log" 9 | 10 | # Create logs directory 11 | mkdir -p "$LOG_DIR" 12 | 13 | # Redirect all output to log file AND console 14 | exec > >(tee -a "$LOG_FILE") 15 | exec 2>&1 16 | 17 | echo "Release log: $LOG_FILE" 18 | echo "Started: $(date)" 19 | echo "" 20 | 21 | # Colors for output 22 | RED='\033[0;31m' 23 | GREEN='\033[0;32m' 24 | YELLOW='\033[1;33m' 25 | BLUE='\033[0;34m' 26 | NC='\033[0m' # No Color 27 | 28 | # Configuration 29 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 30 | cd "$SCRIPT_DIR" 31 | 32 | PACKAGE_NAME="json-toolkit" 33 | DOCKER_IMAGE_BUILD="ubuntu:22.04" 34 | DOCKER_IMAGE_TEST="ubuntu:22.04" 35 | PPA_NAME="${PPA_NAME:-ppa:code-faster/ppa}" 36 | 37 | # Functions 38 | log_info() { 39 | echo -e "${BLUE}[INFO]${NC} $*" 40 | } 41 | 42 | log_success() { 43 | echo -e "${GREEN}[SUCCESS]${NC} $*" 44 | } 45 | 46 | log_warning() { 47 | echo -e "${YELLOW}[WARNING]${NC} $*" 48 | } 49 | 50 | log_error() { 51 | echo -e "${RED}[ERROR]${NC} $*" 52 | } 53 | 54 | log_section() { 55 | echo "" 56 | echo -e "${BLUE}========================================${NC}" 57 | echo -e "${BLUE}$*${NC}" 58 | echo -e "${BLUE}========================================${NC}" 59 | } 60 | 61 | check_prerequisites() { 62 | log_section "Checking Prerequisites" 63 | 64 | local missing=() 65 | 66 | if ! command -v docker &> /dev/null; then 67 | missing+=("docker") 68 | fi 69 | 70 | if ! command -v git &> /dev/null; then 71 | missing+=("git") 72 | fi 73 | 74 | if ! command -v debuild &> /dev/null; then 75 | log_warning "debuild not found (install devscripts)" 76 | fi 77 | 78 | if ! command -v debsign &> /dev/null; then 79 | log_warning "debsign not found (install devscripts)" 80 | fi 81 | 82 | if ! command -v dput &> /dev/null; then 83 | log_warning "dput not found" 84 | fi 85 | 86 | if [ ${#missing[@]} -gt 0 ]; then 87 | log_error "Missing required tools: ${missing[*]}" 88 | exit 1 89 | fi 90 | 91 | log_success "All prerequisites satisfied" 92 | } 93 | 94 | get_version_from_tag() { 95 | local version=$1 96 | echo "$version" 97 | } 98 | 99 | check_clean_working_tree() { 100 | log_info "Checking working tree status" 101 | 102 | if ! git diff-index --quiet HEAD --; then 103 | log_error "Working tree has uncommitted changes" 104 | log_error "Please commit or stash changes before releasing" 105 | git status --short 106 | exit 1 107 | fi 108 | 109 | log_success "Working tree is clean" 110 | } 111 | 112 | phase_1_local_build_test() { 113 | log_section "PHASE 1: Local Build and Test" 114 | 115 | log_info "Cleaning previous builds..." 116 | make clean 117 | 118 | log_info "Building all components..." 119 | make 120 | 121 | log_info "Running test suite..." 122 | make test 123 | 124 | log_success "Local build and test passed" 125 | } 126 | 127 | phase_2_docker_build_package() { 128 | log_section "PHASE 2: Build Debian Package in Docker" 129 | 130 | local dockerfile_build=$(mktemp) 131 | cat > "$dockerfile_build" <<'EOF' 132 | FROM ubuntu:22.04 133 | 134 | ENV DEBIAN_FRONTEND=noninteractive 135 | RUN apt-get update && apt-get install -y \ 136 | build-essential \ 137 | debhelper \ 138 | dh-python \ 139 | devscripts \ 140 | golang-go \ 141 | python3-all \ 142 | python3-pip \ 143 | python3-click \ 144 | python3-dotenv \ 145 | python3-psycopg2 \ 146 | python3-pymysql \ 147 | python3-yaml \ 148 | python3-toml \ 149 | python3-unidiff \ 150 | python3-xmltodict \ 151 | jq \ 152 | && rm -rf /var/lib/apt/lists/* 153 | 154 | WORKDIR /build 155 | COPY . . 156 | 157 | RUN make clean && make 158 | EOF 159 | 160 | log_info "Building Docker image for package building..." 161 | docker build -f "$dockerfile_build" -t "$PACKAGE_NAME-build:latest" . 162 | rm "$dockerfile_build" 163 | 164 | log_info "Extracting built package..." 165 | local container_id=$(docker create "$PACKAGE_NAME-build:latest") 166 | mkdir -p build/docker-output 167 | docker cp "$container_id:/build/target" build/docker-output/ || true 168 | docker rm "$container_id" 169 | 170 | log_success "Docker build completed" 171 | } 172 | 173 | phase_3_docker_install_test() { 174 | log_section "PHASE 3: Install and Test in Clean Docker Environment" 175 | 176 | local dockerfile_test=$(mktemp) 177 | cat > "$dockerfile_test" <<'EOF' 178 | FROM ubuntu:22.04 179 | 180 | ENV DEBIAN_FRONTEND=noninteractive 181 | RUN apt-get update && apt-get install -y \ 182 | golang-go \ 183 | jq \ 184 | python3 \ 185 | python3-pip \ 186 | python3-click \ 187 | python3-dotenv \ 188 | python3-psycopg2 \ 189 | python3-pymysql \ 190 | python3-yaml \ 191 | python3-toml \ 192 | python3-unidiff \ 193 | python3-xmltodict \ 194 | && rm -rf /var/lib/apt/lists/* 195 | 196 | WORKDIR /app 197 | COPY . . 198 | 199 | # Install from source 200 | RUN make clean && make && make install prefix=/usr 201 | 202 | # Run tests 203 | RUN make test 204 | 205 | # Verify installations 206 | RUN which json-diff 207 | RUN json-diff --help || true 208 | RUN which csv-to-json 209 | RUN csv-to-json --help || true 210 | 211 | # Basic smoke tests 212 | RUN echo '[]' | csv-to-json 213 | RUN echo '{"a":1}' | json-to-yaml 214 | RUN echo 'a: 1' | yaml-to-json 215 | EOF 216 | 217 | log_info "Building test Docker image..." 218 | docker build -f "$dockerfile_test" -t "$PACKAGE_NAME-test:latest" . 219 | rm "$dockerfile_test" 220 | 221 | log_info "Running tests in Docker..." 222 | docker run --rm "$PACKAGE_NAME-test:latest" make test 223 | 224 | log_success "Docker installation and tests passed" 225 | } 226 | 227 | phase_4_git_publish() { 228 | log_section "PHASE 4: Git Publishing" 229 | 230 | local version=$1 231 | local tag="v$version" 232 | 233 | # Check if tag already exists 234 | if git rev-parse "$tag" >/dev/null 2>&1; then 235 | log_warning "Tag $tag already exists locally" 236 | 237 | # Check if it exists on remote 238 | if git ls-remote --tags origin | grep -q "refs/tags/$tag"; then 239 | log_success "Tag $tag already pushed to origin (idempotent - skipping)" 240 | else 241 | log_info "Pushing existing tag to origin..." 242 | git push origin "$tag" 243 | log_success "Tag pushed to origin" 244 | fi 245 | else 246 | log_info "Creating annotated git tag: $tag" 247 | local key_id="${GPG_KEY:-${DEBSIGN_KEY:-}}" 248 | if [ -n "$key_id" ]; then 249 | git tag -s -u "$key_id" -a "$tag" -m "Release $version" 250 | log_info "Created signed tag with key $key_id" 251 | else 252 | git tag -a "$tag" -m "Release $version" 253 | log_warning "Created unsigned tag (set GPG_KEY or DEBSIGN_KEY to sign)" 254 | fi 255 | 256 | log_info "Pushing to origin..." 257 | git push origin "$(git branch --show-current)" 258 | git push origin "$tag" 259 | 260 | log_success "Git publishing completed" 261 | fi 262 | } 263 | 264 | phase_5_ppa_publish() { 265 | log_section "PHASE 5: PPA Publishing" 266 | 267 | if [ -z "${DEBSIGN_KEY:-}" ]; then 268 | log_error "DEBSIGN_KEY environment variable not set" 269 | log_info "Export your GPG key ID: export DEBSIGN_KEY=YOUR_KEY_ID" 270 | exit 1 271 | fi 272 | 273 | log_info "Building source package..." 274 | make package 275 | 276 | log_info "Signing package..." 277 | make publish DEBSIGN_KEY="$DEBSIGN_KEY" 278 | 279 | log_success "PPA publishing completed" 280 | log_warning "Note: It may take 5-30 minutes for the package to be available in the PPA" 281 | } 282 | 283 | phase_6_ppa_verify() { 284 | log_section "PHASE 6: PPA Installation Verification" 285 | 286 | local version=$1 287 | 288 | log_warning "Waiting 2 minutes for PPA to process package..." 289 | sleep 120 290 | 291 | local dockerfile_verify=$(mktemp) 292 | cat > "$dockerfile_verify" <.log 372 | EOF 373 | } 374 | 375 | main() { 376 | local skip_phases=() 377 | local version="" 378 | local local_only=false 379 | local docker_only=false 380 | local skip_ppa=false 381 | 382 | # Parse arguments 383 | while [[ $# -gt 0 ]]; do 384 | case $1 in 385 | -h|--help) 386 | usage 387 | exit 0 388 | ;; 389 | -s|--skip-phase) 390 | skip_phases+=("$2") 391 | shift 2 392 | ;; 393 | --skip-ppa) 394 | skip_ppa=true 395 | shift 396 | ;; 397 | --local-only) 398 | local_only=true 399 | shift 400 | ;; 401 | --docker-only) 402 | docker_only=true 403 | shift 404 | ;; 405 | *) 406 | if [ -z "$version" ]; then 407 | version=$1 408 | else 409 | log_error "Unknown option: $1" 410 | usage 411 | exit 1 412 | fi 413 | shift 414 | ;; 415 | esac 416 | done 417 | 418 | # Validate version 419 | if [ -z "$version" ]; then 420 | log_error "Version number required" 421 | usage 422 | exit 1 423 | fi 424 | 425 | if ! [[ $version =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then 426 | log_error "Version must be in format X.Y.Z (e.g., 1.2.0)" 427 | exit 1 428 | fi 429 | 430 | log_section "Starting Release Process for v$version" 431 | 432 | check_prerequisites 433 | 434 | # Ensure working tree is clean (no uncommitted changes) 435 | check_clean_working_tree 436 | 437 | # Run phases based on flags 438 | if [ "$local_only" = true ]; then 439 | phase_1_local_build_test 440 | log_success "Local-only release checks completed!" 441 | exit 0 442 | fi 443 | 444 | if [ "$docker_only" = true ]; then 445 | phase_2_docker_build_package 446 | phase_3_docker_install_test 447 | log_success "Docker-only release checks completed!" 448 | exit 0 449 | fi 450 | 451 | # Full release pipeline 452 | [[ ! " ${skip_phases[*]} " =~ " 1 " ]] && phase_1_local_build_test 453 | [[ ! " ${skip_phases[*]} " =~ " 2 " ]] && phase_2_docker_build_package 454 | [[ ! " ${skip_phases[*]} " =~ " 3 " ]] && phase_3_docker_install_test 455 | 456 | if [ "$skip_ppa" = false ]; then 457 | [[ ! " ${skip_phases[*]} " =~ " 4 " ]] && phase_4_git_publish "$version" 458 | [[ ! " ${skip_phases[*]} " =~ " 5 " ]] && phase_5_ppa_publish 459 | [[ ! " ${skip_phases[*]} " =~ " 6 " ]] && phase_6_ppa_verify "$version" 460 | else 461 | [[ ! " ${skip_phases[*]} " =~ " 4 " ]] && phase_4_git_publish "$version" 462 | log_warning "Skipping PPA publishing and verification" 463 | fi 464 | 465 | log_section "RELEASE COMPLETE! 🎉" 466 | log_success "Version $version has been successfully released" 467 | 468 | cat < 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # A comma-separated list of package or module names from where C extensions may 4 | # be loaded. Extensions are loading into the active Python interpreter and may 5 | # run arbitrary code. 6 | extension-pkg-whitelist= 7 | 8 | # Add files or directories to the blacklist. They should be base names, not 9 | # paths. 10 | ignore=CVS 11 | 12 | # Add files or directories matching the regex patterns to the blacklist. The 13 | # regex matches against base names, not paths. 14 | ignore-patterns= 15 | 16 | # Python code to execute, usually for sys.path manipulation such as 17 | # pygtk.require(). 18 | #init-hook= 19 | 20 | # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the 21 | # number of processors available to use. 22 | jobs=1 23 | 24 | # Control the amount of potential inferred values when inferring a single 25 | # object. This can help the performance when dealing with large functions or 26 | # complex, nested conditions. 27 | limit-inference-results=100 28 | 29 | # List of plugins (as comma separated values of python module names) to load, 30 | # usually to register additional checkers. 31 | load-plugins= 32 | 33 | # Pickle collected data for later comparisons. 34 | persistent=yes 35 | 36 | # Specify a configuration file. 37 | #rcfile= 38 | 39 | # When enabled, pylint would attempt to guess common misconfiguration and emit 40 | # user-friendly hints instead of false-positive error messages. 41 | suggestion-mode=yes 42 | 43 | # Allow loading of arbitrary C extensions. Extensions are imported into the 44 | # active Python interpreter and may run arbitrary code. 45 | unsafe-load-any-extension=no 46 | 47 | 48 | [MESSAGES CONTROL] 49 | 50 | # Only show warnings with the listed confidence levels. Leave empty to show 51 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. 52 | confidence= 53 | 54 | # Disable the message, report, category or checker with the given id(s). You 55 | # can either give multiple identifiers separated by comma (,) or put this 56 | # option multiple times (only on the command line, not in the configuration 57 | # file where it should appear only once). You can also use "--disable=all" to 58 | # disable everything first and then reenable specific checks. For example, if 59 | # you want to run only the similarities checker, you can use "--disable=all 60 | # --enable=similarities". If you want to run only the classes checker, but have 61 | # no Warning level messages displayed, use "--disable=all --enable=classes 62 | # --disable=W". 63 | disable=print-statement, 64 | parameter-unpacking, 65 | unpacking-in-except, 66 | old-raise-syntax, 67 | backtick, 68 | long-suffix, 69 | old-ne-operator, 70 | old-octal-literal, 71 | import-star-module-level, 72 | non-ascii-bytes-literal, 73 | raw-checker-failed, 74 | bad-inline-option, 75 | locally-disabled, 76 | file-ignored, 77 | suppressed-message, 78 | useless-suppression, 79 | deprecated-pragma, 80 | use-symbolic-message-instead, 81 | apply-builtin, 82 | basestring-builtin, 83 | buffer-builtin, 84 | cmp-builtin, 85 | coerce-builtin, 86 | execfile-builtin, 87 | file-builtin, 88 | long-builtin, 89 | raw_input-builtin, 90 | reduce-builtin, 91 | standarderror-builtin, 92 | unicode-builtin, 93 | xrange-builtin, 94 | coerce-method, 95 | delslice-method, 96 | getslice-method, 97 | setslice-method, 98 | no-absolute-import, 99 | old-division, 100 | dict-iter-method, 101 | dict-view-method, 102 | next-method-called, 103 | metaclass-assignment, 104 | indexing-exception, 105 | raising-string, 106 | reload-builtin, 107 | oct-method, 108 | hex-method, 109 | nonzero-method, 110 | cmp-method, 111 | input-builtin, 112 | round-builtin, 113 | intern-builtin, 114 | unichr-builtin, 115 | map-builtin-not-iterating, 116 | zip-builtin-not-iterating, 117 | range-builtin-not-iterating, 118 | filter-builtin-not-iterating, 119 | using-cmp-argument, 120 | eq-without-hash, 121 | div-method, 122 | idiv-method, 123 | rdiv-method, 124 | exception-message-attribute, 125 | invalid-str-codec, 126 | sys-max-int, 127 | bad-python3-import, 128 | deprecated-string-function, 129 | deprecated-str-translate-call, 130 | deprecated-itertools-function, 131 | deprecated-types-field, 132 | next-method-defined, 133 | dict-items-not-iterating, 134 | dict-keys-not-iterating, 135 | dict-values-not-iterating, 136 | deprecated-operator-function, 137 | deprecated-urllib-function, 138 | xreadlines-attribute, 139 | deprecated-sys-function, 140 | exception-escape, 141 | comprehension-escape, 142 | missing-module-docstring, 143 | missing-function-docstring, 144 | missing-class-docstring, 145 | unidiomatic-typecheck, 146 | invalid-name, 147 | line-too-long, 148 | no-else-return, 149 | no-else-raise, 150 | too-many-arguments, 151 | 152 | # Enable the message, report, category or checker with the given id(s). You can 153 | # either give multiple identifier separated by comma (,) or put this option 154 | # multiple time (only on the command line, not in the configuration file where 155 | # it should appear only once). See also the "--disable" option for examples. 156 | enable=c-extension-no-member 157 | 158 | 159 | [REPORTS] 160 | 161 | # Python expression which should return a score less than or equal to 10. You 162 | # have access to the variables 'error', 'warning', 'refactor', and 'convention' 163 | # which contain the number of messages in each category, as well as 'statement' 164 | # which is the total number of statements analyzed. This score is used by the 165 | # global evaluation report (RP0004). 166 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 167 | 168 | # Template used to display messages. This is a python new-style format string 169 | # used to format the message information. See doc for all details. 170 | #msg-template= 171 | 172 | # Set the output format. Available formats are text, parseable, colorized, json 173 | # and msvs (visual studio). You can also give a reporter class, e.g. 174 | # mypackage.mymodule.MyReporterClass. 175 | output-format=text 176 | 177 | # Tells whether to display a full report or only the messages. 178 | reports=no 179 | 180 | # Activate the evaluation score. 181 | score=yes 182 | 183 | 184 | [REFACTORING] 185 | 186 | # Maximum number of nested blocks for function / method body 187 | max-nested-blocks=5 188 | 189 | # Complete name of functions that never returns. When checking for 190 | # inconsistent-return-statements if a never returning function is called then 191 | # it will be considered as an explicit return statement and no message will be 192 | # printed. 193 | never-returning-functions=sys.exit 194 | 195 | 196 | [SPELLING] 197 | 198 | # Limits count of emitted suggestions for spelling mistakes. 199 | max-spelling-suggestions=4 200 | 201 | # Spelling dictionary name. Available dictionaries: none. To make it work, 202 | # install the python-enchant package. 203 | spelling-dict= 204 | 205 | # List of comma separated words that should not be checked. 206 | spelling-ignore-words= 207 | 208 | # A path to a file that contains the private dictionary; one word per line. 209 | spelling-private-dict-file= 210 | 211 | # Tells whether to store unknown words to the private dictionary (see the 212 | # --spelling-private-dict-file option) instead of raising a message. 213 | spelling-store-unknown-words=no 214 | 215 | 216 | [VARIABLES] 217 | 218 | # List of additional names supposed to be defined in builtins. Remember that 219 | # you should avoid defining new builtins when possible. 220 | additional-builtins= 221 | 222 | # Tells whether unused global variables should be treated as a violation. 223 | allow-global-unused-variables=yes 224 | 225 | # List of strings which can identify a callback function by name. A callback 226 | # name must start or end with one of those strings. 227 | callbacks=cb_, 228 | _cb 229 | 230 | # A regular expression matching the name of dummy variables (i.e. expected to 231 | # not be used). 232 | dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ 233 | 234 | # Argument names that match this expression will be ignored. Default to name 235 | # with leading underscore. 236 | ignored-argument-names=_.*|^ignored_|^unused_ 237 | 238 | # Tells whether we should check for unused import in __init__ files. 239 | init-import=no 240 | 241 | # List of qualified module names which can have objects that can redefine 242 | # builtins. 243 | redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io 244 | 245 | 246 | [TYPECHECK] 247 | 248 | # List of decorators that produce context managers, such as 249 | # contextlib.contextmanager. Add to this list to register other decorators that 250 | # produce valid context managers. 251 | contextmanager-decorators=contextlib.contextmanager 252 | 253 | # List of members which are set dynamically and missed by pylint inference 254 | # system, and so shouldn't trigger E1101 when accessed. Python regular 255 | # expressions are accepted. 256 | generated-members= 257 | 258 | # Tells whether missing members accessed in mixin class should be ignored. A 259 | # mixin class is detected if its name ends with "mixin" (case insensitive). 260 | ignore-mixin-members=yes 261 | 262 | # Tells whether to warn about missing members when the owner of the attribute 263 | # is inferred to be None. 264 | ignore-none=yes 265 | 266 | # This flag controls whether pylint should warn about no-member and similar 267 | # checks whenever an opaque object is returned when inferring. The inference 268 | # can return multiple potential results while evaluating a Python object, but 269 | # some branches might not be evaluated, which results in partial inference. In 270 | # that case, it might be useful to still emit no-member and other checks for 271 | # the rest of the inferred objects. 272 | ignore-on-opaque-inference=yes 273 | 274 | # List of class names for which member attributes should not be checked (useful 275 | # for classes with dynamically set attributes). This supports the use of 276 | # qualified names. 277 | ignored-classes=optparse.Values,thread._local,_thread._local 278 | 279 | # List of module names for which member attributes should not be checked 280 | # (useful for modules/projects where namespaces are manipulated during runtime 281 | # and thus existing member attributes cannot be deduced by static analysis). It 282 | # supports qualified module names, as well as Unix pattern matching. 283 | ignored-modules= 284 | 285 | # Show a hint with possible names when a member name was not found. The aspect 286 | # of finding the hint is based on edit distance. 287 | missing-member-hint=yes 288 | 289 | # The minimum edit distance a name should have in order to be considered a 290 | # similar match for a missing member name. 291 | missing-member-hint-distance=1 292 | 293 | # The total number of similar names that should be taken in consideration when 294 | # showing a hint for a missing member. 295 | missing-member-max-choices=1 296 | 297 | # List of decorators that change the signature of a decorated function. 298 | signature-mutators= 299 | 300 | 301 | [BASIC] 302 | 303 | # Naming style matching correct argument names. 304 | argument-naming-style=snake_case 305 | 306 | # Regular expression matching correct argument names. Overrides argument- 307 | # naming-style. 308 | #argument-rgx= 309 | 310 | # Naming style matching correct attribute names. 311 | attr-naming-style=snake_case 312 | 313 | # Regular expression matching correct attribute names. Overrides attr-naming- 314 | # style. 315 | #attr-rgx= 316 | 317 | # Bad variable names which should always be refused, separated by a comma. 318 | bad-names=foo, 319 | bar, 320 | baz, 321 | toto, 322 | tutu, 323 | tata 324 | 325 | # Naming style matching correct class attribute names. 326 | class-attribute-naming-style=any 327 | 328 | # Regular expression matching correct class attribute names. Overrides class- 329 | # attribute-naming-style. 330 | #class-attribute-rgx= 331 | 332 | # Naming style matching correct class names. 333 | class-naming-style=PascalCase 334 | 335 | # Regular expression matching correct class names. Overrides class-naming- 336 | # style. 337 | #class-rgx= 338 | 339 | # Naming style matching correct constant names. 340 | const-naming-style=UPPER_CASE 341 | 342 | # Regular expression matching correct constant names. Overrides const-naming- 343 | # style. 344 | #const-rgx= 345 | 346 | # Minimum line length for functions/classes that require docstrings, shorter 347 | # ones are exempt. 348 | docstring-min-length=-1 349 | 350 | # Naming style matching correct function names. 351 | function-naming-style=snake_case 352 | 353 | # Regular expression matching correct function names. Overrides function- 354 | # naming-style. 355 | #function-rgx= 356 | 357 | # Good variable names which should always be accepted, separated by a comma. 358 | good-names=i, 359 | j, 360 | k, 361 | ex, 362 | Run, 363 | _ 364 | 365 | # Include a hint for the correct naming format with invalid-name. 366 | include-naming-hint=no 367 | 368 | # Naming style matching correct inline iteration names. 369 | inlinevar-naming-style=any 370 | 371 | # Regular expression matching correct inline iteration names. Overrides 372 | # inlinevar-naming-style. 373 | #inlinevar-rgx= 374 | 375 | # Naming style matching correct method names. 376 | method-naming-style=snake_case 377 | 378 | # Regular expression matching correct method names. Overrides method-naming- 379 | # style. 380 | #method-rgx= 381 | 382 | # Naming style matching correct module names. 383 | module-naming-style=snake_case 384 | 385 | # Regular expression matching correct module names. Overrides module-naming- 386 | # style. 387 | #module-rgx= 388 | 389 | # Colon-delimited sets of names that determine each other's naming style when 390 | # the name regexes allow several styles. 391 | name-group= 392 | 393 | # Regular expression which should only match function or class names that do 394 | # not require a docstring. 395 | no-docstring-rgx=^_ 396 | 397 | # List of decorators that produce properties, such as abc.abstractproperty. Add 398 | # to this list to register other decorators that produce valid properties. 399 | # These decorators are taken in consideration only for invalid-name. 400 | property-classes=abc.abstractproperty 401 | 402 | # Naming style matching correct variable names. 403 | variable-naming-style=snake_case 404 | 405 | # Regular expression matching correct variable names. Overrides variable- 406 | # naming-style. 407 | #variable-rgx= 408 | 409 | 410 | [FORMAT] 411 | 412 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 413 | expected-line-ending-format= 414 | 415 | # Regexp for a line that is allowed to be longer than the limit. 416 | ignore-long-lines=^\s*(# )??$ 417 | 418 | # Number of spaces of indent required inside a hanging or continued line. 419 | indent-after-paren=4 420 | 421 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 422 | # tab). 423 | indent-string=' ' 424 | 425 | # Maximum number of characters on a single line. 426 | max-line-length=100 427 | 428 | # Maximum number of lines in a module. 429 | max-module-lines=1000 430 | 431 | # List of optional constructs for which whitespace checking is disabled. `dict- 432 | # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. 433 | # `trailing-comma` allows a space between comma and closing bracket: (a, ). 434 | # `empty-line` allows space-only lines. 435 | no-space-check=trailing-comma, 436 | dict-separator 437 | 438 | # Allow the body of a class to be on the same line as the declaration if body 439 | # contains single statement. 440 | single-line-class-stmt=no 441 | 442 | # Allow the body of an if to be on the same line as the test if there is no 443 | # else. 444 | single-line-if-stmt=no 445 | 446 | 447 | [LOGGING] 448 | 449 | # Format style used to check logging format string. `old` means using % 450 | # formatting, `new` is for `{}` formatting,and `fstr` is for f-strings. 451 | logging-format-style=old 452 | 453 | # Logging modules to check that the string format arguments are in logging 454 | # function parameter format. 455 | logging-modules=logging 456 | 457 | 458 | [STRING] 459 | 460 | # This flag controls whether the implicit-str-concat-in-sequence should 461 | # generate a warning on implicit string concatenation in sequences defined over 462 | # several lines. 463 | check-str-concat-over-line-jumps=no 464 | 465 | 466 | [MISCELLANEOUS] 467 | 468 | # List of note tags to take in consideration, separated by a comma. 469 | notes=FIXME, 470 | XXX, 471 | TODO 472 | 473 | 474 | [SIMILARITIES] 475 | 476 | # Ignore comments when computing similarities. 477 | ignore-comments=yes 478 | 479 | # Ignore docstrings when computing similarities. 480 | ignore-docstrings=yes 481 | 482 | # Ignore imports when computing similarities. 483 | ignore-imports=no 484 | 485 | # Minimum lines number of a similarity. 486 | min-similarity-lines=20 487 | 488 | 489 | [CLASSES] 490 | 491 | # List of method names used to declare (i.e. assign) instance attributes. 492 | defining-attr-methods=__init__, 493 | __new__, 494 | setUp, 495 | __post_init__ 496 | 497 | # List of member names, which should be excluded from the protected access 498 | # warning. 499 | exclude-protected=_asdict, 500 | _fields, 501 | _replace, 502 | _source, 503 | _make 504 | 505 | # List of valid names for the first argument in a class method. 506 | valid-classmethod-first-arg=cls 507 | 508 | # List of valid names for the first argument in a metaclass class method. 509 | valid-metaclass-classmethod-first-arg=cls 510 | 511 | 512 | [IMPORTS] 513 | 514 | # List of modules that can be imported at any level, not just the top level 515 | # one. 516 | allow-any-import-level= 517 | 518 | # Allow wildcard imports from modules that define __all__. 519 | allow-wildcard-with-all=no 520 | 521 | # Analyse import fallback blocks. This can be used to support both Python 2 and 522 | # 3 compatible code, which means that the block might have code that exists 523 | # only in one or another interpreter, leading to false positives when analysed. 524 | analyse-fallback-blocks=no 525 | 526 | # Deprecated modules which should not be used, separated by a comma. 527 | deprecated-modules=optparse,tkinter.tix 528 | 529 | # Create a graph of external dependencies in the given file (report RP0402 must 530 | # not be disabled). 531 | ext-import-graph= 532 | 533 | # Create a graph of every (i.e. internal and external) dependencies in the 534 | # given file (report RP0402 must not be disabled). 535 | import-graph= 536 | 537 | # Create a graph of internal dependencies in the given file (report RP0402 must 538 | # not be disabled). 539 | int-import-graph= 540 | 541 | # Force import order to recognize a module as part of the standard 542 | # compatibility libraries. 543 | known-standard-library= 544 | 545 | # Force import order to recognize a module as part of a third party library. 546 | known-third-party=enchant 547 | 548 | # Couples of modules and preferred modules, separated by a comma. 549 | preferred-modules= 550 | 551 | 552 | [DESIGN] 553 | 554 | # Maximum number of arguments for function / method. 555 | max-args=5 556 | 557 | # Maximum number of attributes for a class (see R0902). 558 | max-attributes=7 559 | 560 | # Maximum number of boolean expressions in an if statement (see R0916). 561 | max-bool-expr=5 562 | 563 | # Maximum number of branch for function / method body. 564 | max-branches=12 565 | 566 | # Maximum number of locals for function / method body. 567 | max-locals=15 568 | 569 | # Maximum number of parents for a class (see R0901). 570 | max-parents=7 571 | 572 | # Maximum number of public methods for a class (see R0904). 573 | max-public-methods=20 574 | 575 | # Maximum number of return / yield for function / method body. 576 | max-returns=6 577 | 578 | # Maximum number of statements in function / method body. 579 | max-statements=50 580 | 581 | # Minimum number of public methods for a class (see R0903). 582 | min-public-methods=2 583 | 584 | 585 | [EXCEPTIONS] 586 | 587 | # Exceptions that will emit a warning when being caught. Defaults to 588 | # "BaseException, Exception". 589 | overgeneral-exceptions=BaseException, 590 | Exception 591 | --------------------------------------------------------------------------------