├── trigger-kaitai_struct_python_docker ├── trigger-kaitai_struct_webide ├── runtime └── README.md ├── .gitmodules ├── README.md └── .github └── workflows └── main.yml /trigger-kaitai_struct_python_docker: -------------------------------------------------------------------------------- 1 | #!/bin/sh -ef 2 | 3 | # https://gitlab.com/kaitaiStructCompile.py/kaitai_struct_python_docker 4 | # build depends on fresh unstable .deb being published. This call 5 | # triggers GitLab CI pipeline that will build a new Docker image 6 | # including latest unstable compiler build .deb. 7 | 8 | echo Triggering build for kaitai_struct_python_docker project 9 | if [ -n "$KAITAI_STRUCT_PYTHON_DOCKER_TOKEN" ]; then 10 | curl -F "token=${KAITAI_STRUCT_PYTHON_DOCKER_TOKEN}" -F "ref=master" https://gitlab.com/api/v4/projects/10444125/trigger/pipeline || true 11 | fi 12 | -------------------------------------------------------------------------------- /trigger-kaitai_struct_webide: -------------------------------------------------------------------------------- 1 | #!/bin/sh -ef 2 | 3 | # ide.kaitai.io/devel needs to be rebuilt after we publish the latest JS build of KSC to npm. 4 | 5 | if [ "$#" -ne 1 ]; then 6 | echo "Usage: $0 " 7 | exit 1 8 | fi 9 | 10 | ksc_version=$1 11 | 12 | PROJECT='kaitai-io/kaitai_struct_webide' 13 | 14 | echo "Triggering build for $PROJECT project..." 15 | if [ -n "$KAITAI_STRUCT_WEBIDE_GITHUB_TOKEN" ]; then 16 | # `event_type` must match the name whitelisted in the CI workflow config in 17 | # https://github.com/kaitai-io/kaitai_struct_webide 18 | body=$(jq -n -c --arg ksc_version "$ksc_version" '{event_type: "rebuild", client_payload: {ksc_version: $ksc_version}}') 19 | 20 | printf '%s\n' "$body" 21 | 22 | curl -fsSL \ 23 | -H "Accept: application/vnd.github+json" \ 24 | -H "Authorization: Bearer $KAITAI_STRUCT_WEBIDE_GITHUB_TOKEN" \ 25 | -H "X-GitHub-Api-Version: 2022-11-28" \ 26 | -d "$body" \ 27 | "https://api.github.com/repos/$PROJECT/dispatches" 28 | else 29 | echo "No KAITAI_STRUCT_WEBIDE_GITHUB_TOKEN found!" 30 | exit 1 31 | fi 32 | -------------------------------------------------------------------------------- /runtime/README.md: -------------------------------------------------------------------------------- 1 | # Developers' memo for runtimes 2 | 3 | All runtimes should include at least the following methods in 4 | `KaitaiStream` class (or its equivalent). It's heavily preferred to 5 | maintain exactly this order of declarations and group headers. 6 | 7 | ## Stream positioning 8 | 9 | * `eof?` 10 | * `seek(n)` 11 | * `pos` 12 | * `size` 13 | 14 | ## Integer numbers 15 | 16 | ### Signed 17 | 18 | * `read_s1` 19 | 20 | #### Big-endian 21 | 22 | * `read_s2be` 23 | * `read_s4be` 24 | * `read_s8be` 25 | 26 | #### Little-endian 27 | 28 | * `read_s2le` 29 | * `read_s4le` 30 | * `read_s8le` 31 | 32 | ### Unsigned 33 | 34 | * `read_u1` 35 | 36 | #### Big-endian 37 | 38 | * `read_u2be` 39 | * `read_u4be` 40 | * `read_u8be` 41 | 42 | #### Little-endian 43 | 44 | * `read_u2le` 45 | * `read_u4le` 46 | * `read_u8le` 47 | 48 | ## Floating point numbers 49 | 50 | ### Big-endian 51 | 52 | * `read_f4be` 53 | * `read_f8be` 54 | 55 | ### Little-endian 56 | 57 | * `read_f4le` 58 | * `read_f8le` 59 | 60 | ## Unaligned bit values 61 | 62 | * `align_to_byte()` 63 | * `read_bits_int(n)` 64 | * `read_bits_array(n)` 65 | 66 | ## Byte arrays 67 | 68 | * `read_bytes(n)` 69 | * `read_bytes_full` 70 | * `read_bytes_term(String encoding, int term, boolean include_term, boolean consumeTerm, boolean eosError)` 71 | * `ensure_fixed_contents(expected)` 72 | * static `bytes_strip_right(bytes, pad_byte)` 73 | * static `bytes_terminate(bytes, term, include_term)` 74 | * static `bytes_to_str(bytes, encoding)` 75 | 76 | ## Byte array processing 77 | 78 | * `process_xor(data, key)` 79 | * `process_xor_one(data, key)` 80 | * `process_xor_many(data, key)` 81 | * `process_rotate_left(data, amount, group_size)` 82 | * `process_zlib(data)` 83 | 84 | ## Misc runtime operations 85 | 86 | * static `mod(a, b)` 87 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "runtime/java"] 2 | path = runtime/java 3 | url = https://github.com/kaitai-io/kaitai_struct_java_runtime.git 4 | [submodule "runtime/python"] 5 | path = runtime/python 6 | url = https://github.com/kaitai-io/kaitai_struct_python_runtime.git 7 | [submodule "runtime/ruby"] 8 | path = runtime/ruby 9 | url = https://github.com/kaitai-io/kaitai_struct_ruby_runtime.git 10 | [submodule "tests"] 11 | path = tests 12 | url = https://github.com/kaitai-io/kaitai_struct_tests.git 13 | [submodule "compiler"] 14 | path = compiler 15 | url = https://github.com/kaitai-io/kaitai_struct_compiler.git 16 | [submodule "runtime/javascript"] 17 | path = runtime/javascript 18 | url = https://github.com/kaitai-io/kaitai_struct_javascript_runtime.git 19 | [submodule "runtime/cpp_stl"] 20 | path = runtime/cpp_stl 21 | url = https://github.com/kaitai-io/kaitai_struct_cpp_stl_runtime.git 22 | [submodule "runtime/csharp"] 23 | path = runtime/csharp 24 | url = https://github.com/kaitai-io/kaitai_struct_csharp_runtime 25 | [submodule "formats"] 26 | path = formats 27 | url = https://github.com/kaitai-io/kaitai_struct_formats.git 28 | [submodule "visualizer"] 29 | path = visualizer 30 | url = https://github.com/kaitai-io/kaitai_struct_visualizer.git 31 | [submodule "runtime/php"] 32 | path = runtime/php 33 | url = https://github.com/kaitai-io/kaitai_struct_php_runtime 34 | [submodule "runtime/perl"] 35 | path = runtime/perl 36 | url = https://github.com/kaitai-io/kaitai_struct_perl_runtime 37 | [submodule "benchmarks"] 38 | path = benchmarks 39 | url = https://github.com/kaitai-io/kaitai_struct_benchmarks 40 | [submodule "doc"] 41 | path = doc 42 | url = https://github.com/kaitai-io/kaitai_struct_doc 43 | [submodule "runtime/go"] 44 | path = runtime/go 45 | url = https://github.com/kaitai-io/kaitai_struct_go_runtime 46 | [submodule "runtime/lua"] 47 | path = runtime/lua 48 | url = https://github.com/kaitai-io/kaitai_struct_lua_runtime 49 | [submodule "runtime/nim"] 50 | path = runtime/nim 51 | url = https://github.com/kaitai-io/kaitai_struct_nim_runtime 52 | [submodule "runtime/rust"] 53 | path = runtime/rust 54 | url = https://github.com/kaitai-io/kaitai_struct_rust_runtime 55 | [submodule "runtime/zig"] 56 | path = runtime/zig 57 | url = https://github.com/kaitai-io/kaitai_struct_zig_runtime.git 58 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kaitai Struct 2 | 3 | [![Join the chat at https://gitter.im/kaitai_struct/Lobby](https://badges.gitter.im/kaitai_struct/Lobby.svg)](https://gitter.im/kaitai_struct/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 4 | 5 | > **Note:** if you want to make changes to the project, do **not** fork this repository *kaitai_struct*. Instead, choose the component you want to modify in the file tree above and fork **that** individual component instead. 6 | > 7 | > This is an umbrella repository, containing the components only as submodules to make it easier to check out the entire project. Unless you want to modify this README, it is not the repo where you can make edits. 8 | 9 | ## What is Kaitai Struct? 10 | 11 | Kaitai Struct (KS) is a declarative language used to describe various 12 | binary data structures, laid out in files or in memory: i.e. binary 13 | file formats, network stream packet formats, etc. 14 | 15 | The main idea is that a particular format is described in Kaitai 16 | Struct language (`.ksy` file) only once and then can be compiled with 17 | `kaitai-struct-compiler` (or `ksc` for short) into source files in one 18 | of the supported programming languages. These modules will include 19 | generated code for a parser that can read the described data structure 20 | from a file or stream and provide access to it in a nice, 21 | easy-to-comprehend API. 22 | 23 | ## What is it used for? 24 | 25 | Have you ever found yourself writing repetitive, error-prone and 26 | hard-to-debug code that reads binary data structures from file / 27 | network stream and somehow represents them in memory for easier 28 | access? 29 | 30 | Kaitai Struct tries to make this job easier — you only have to 31 | describe the binary format once and then everybody can use it from their 32 | programming languages — cross-language, cross-platform. 33 | 34 | Kaitai Struct includes a growing collection of format descriptions, 35 | available in 36 | [formats](https://github.com/kaitai-io/kaitai_struct_formats) 37 | submodule repository. 38 | 39 | ## Using KS in your project 40 | 41 | Typically, using formats described in KS in your project involves the 42 | following steps: 43 | 44 | * Describe the format — i.e. create a `.ksy` file 45 | * Use a visualizer to debug the format and ensure that it parses data 46 | properly (official visualizers are [Web IDE](https://ide.kaitai.io/) 47 | and the console visualizer 48 | [ksv](https://github.com/kaitai-io/kaitai_struct_visualizer)) 49 | * Compile the `.ksy` file into a target language source file and include 50 | that file in your project 51 | * Add the KS runtime library for your particular language to your 52 | project (don't worry, it's small and it's there mostly to ensure 53 | readability of generated code) 54 | * Use the generated class(es) to parse your binary file or stream and 55 | access its components 56 | 57 | To see an example, a list of supported languages, download instructions 58 | and licensing information, visit https://kaitai.io/. 59 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: {} 8 | 9 | permissions: {} 10 | 11 | jobs: 12 | ksc: 13 | environment: unstable 14 | runs-on: ubuntu-latest 15 | permissions: 16 | contents: read 17 | steps: 18 | - uses: actions/checkout@v5 19 | with: 20 | persist-credentials: false 21 | submodules: recursive 22 | - uses: actions/setup-java@v5 23 | with: 24 | distribution: temurin 25 | java-version: '25' 26 | - uses: sbt/setup-sbt@v1 27 | - name: build-compiler 28 | run: | 29 | cd compiler 30 | export GIT_COMMIT=$(git log -1 --format=%h) 31 | export GIT_DATE_ISO=$(TZ=UTC git log -1 --date=iso-strict-local --format=%cd) 32 | export GIT_DATE=$(TZ=UTC git log -1 --date=format-local:%Y%m%d.%H%M%S --format=%cd) 33 | export KAITAI_STRUCT_VERSION=0.12-SNAPSHOT${GIT_DATE}.${GIT_COMMIT} 34 | 35 | echo "KAITAI_STRUCT_VERSION=$KAITAI_STRUCT_VERSION" 36 | 37 | cat /dev/null | sbt \ 38 | compile \ 39 | compilerJVM/stage \ 40 | fastOptJS \ 41 | buildNpmJsFile \ 42 | buildNpmPackage \ 43 | compilerJVM/debian:packageBin \ 44 | compilerJVM/universal:packageBin 45 | 46 | # TODO: add compilerJVM/rpm:packageBin 47 | 48 | - id: publish-ksc-to-npm 49 | name: publish JS compiler to npm 50 | working-directory: compiler 51 | run: ./publish_js_to_npm.sh 52 | env: 53 | NPM_API_KEY: ${{ secrets.NPM_TOKEN }} 54 | continue-on-error: true 55 | 56 | - name: trigger rebuild of ide.kaitai.io/devel 57 | run: | 58 | ./trigger-kaitai_struct_webide "$(jq -r .version compiler/js/npm/package.json)" 59 | env: 60 | KAITAI_STRUCT_WEBIDE_GITHUB_TOKEN: ${{ secrets.KAITAI_STRUCT_WEBIDE_GITHUB_TOKEN }} 61 | if: ${{ success() && steps.publish-ksc-to-npm.outcome == 'success' }} 62 | 63 | - name: publish ksc to artifacts 64 | uses: actions/upload-artifact@v5 65 | with: 66 | name: kaitai-struct-compiler 67 | path: | 68 | compiler/js/target/scala-*/kaitai-struct-compiler-fastopt.js 69 | compiler/js/npm 70 | compiler/jvm/target/kaitai-struct-compiler_*_all.deb 71 | compiler/jvm/target/universal/kaitai-struct-compiler-*.zip 72 | 73 | # TODO: add compiler/jvm/target/rpm/RPMS/noarch/kaitai-struct-compiler-*.noarch.rpm 74 | 75 | - name: publish deb to repo 76 | uses: kaitai-io/repo-apt-handle@v0.1 77 | with: 78 | az_storage_sas_token: ${{ secrets.AZURE_STORAGE_SAS_TOKEN }} 79 | az_storage_account: packageskaitai 80 | az_storage_container: unstable 81 | packages: compiler/jvm/target/kaitai-struct-compiler_*_all.deb 82 | gpg_priv_key: ${{ secrets.GPG_PRIV_KEY }} 83 | gpg_passphrase: ${{ secrets.GPG_PASSPHRASE }} 84 | if: github.event_name == 'push' && github.ref == 'refs/heads/master' 85 | continue-on-error: true 86 | 87 | - name: build-formats 88 | run: | 89 | cd tests 90 | ./build-formats 91 | 92 | - name: publish formats to ci_targets 93 | env: 94 | BOT_SSH_KEY: ${{secrets.BOT_SSH_KEY}} 95 | run: | 96 | cd tests 97 | ./push_artifacts/git_config_kaitai_bot 98 | ./push_artifacts/publish \ 99 | -o kaitai-io \ 100 | -r ci_targets \ 101 | -m "Regen ${GITHUB_REF#refs/heads/*} kaitai-io/kaitai_struct@$GITHUB_SHA" \ 102 | -- \ 103 | --exclude=.git \ 104 | --exclude=.github \ 105 | --exclude=.travis.yml \ 106 | compiled 107 | if: github.event_name == 'push' && github.ref == 'refs/heads/master' 108 | --------------------------------------------------------------------------------