├── .dockerignore ├── .github └── workflows │ └── tests.yml ├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── Dockerfile ├── README.md ├── celldb-migrate ├── CMakeLists.txt └── src │ ├── main.cpp │ └── main_copy_with_new_settings.cpp ├── cmake └── FindPostgreSQL.cmake ├── pgton ├── CMakeLists.txt ├── README.md ├── expected │ └── basic.out ├── pgton--0.1.sql ├── pgton.c └── sql │ └── basic.sql ├── scripts ├── add2systemd.sh ├── create_indexes.sql ├── drop_indexes.sql └── entrypoint.sh ├── symbolicate_crash.sh ├── ton-emulate-go ├── docs │ ├── docs.go │ ├── swagger.json │ └── swagger.yaml ├── go.mod ├── go.sum ├── main.go └── models │ ├── convert.go │ ├── models.go │ └── response.go ├── ton-index-clickhouse ├── CMakeLists.txt └── src │ ├── IndexScheduler.cpp │ ├── IndexScheduler.h │ ├── InsertManagerClickhouse.cpp │ ├── InsertManagerClickhouse.h │ └── main.cpp ├── ton-index-postgres-v2 ├── CMakeLists.txt └── src │ ├── BlockInterfacesDetector.h │ ├── IndexScheduler.cpp │ ├── IndexScheduler.h │ ├── InsertManagerPostgres.cpp │ ├── InsertManagerPostgres.h │ └── main.cpp ├── ton-index-postgres ├── CMakeLists.txt └── src │ ├── IndexScheduler.cpp │ ├── IndexScheduler.h │ ├── InsertManagerPostgres.cpp │ ├── InsertManagerPostgres.h │ └── main.cpp ├── ton-integrity-checker ├── CMakeLists.txt └── src │ ├── IntegrityChecker.cpp │ ├── IntegrityChecker.h │ └── main.cpp ├── ton-smc-scanner ├── CMakeLists.txt └── src │ ├── PostgreSQLInserter.cpp │ ├── PostgreSQLInserter.h │ ├── SmcScanner.cpp │ ├── SmcScanner.h │ └── main.cpp ├── ton-trace-emulator ├── CMakeLists.txt └── src │ ├── BlockEmulator.cpp │ ├── BlockEmulator.h │ ├── OverlayListener.cpp │ ├── OverlayListener.h │ ├── RedisListener.cpp │ ├── RedisListener.h │ ├── Serializer.hpp │ ├── TraceEmulator.cpp │ ├── TraceEmulator.h │ ├── TraceInserter.cpp │ ├── TraceInserter.h │ ├── TraceInterfaceDetector.cpp │ ├── TraceInterfaceDetector.h │ ├── TraceScheduler.cpp │ ├── TraceScheduler.h │ └── main.cpp ├── ton-trace-task-emulator ├── CMakeLists.txt └── src │ ├── RedisListener.cpp │ ├── RedisListener.h │ ├── TaskResultInserter.cpp │ ├── TaskResultInserter.h │ ├── TraceTaskScheduler.cpp │ ├── TraceTaskScheduler.h │ └── main.cpp └── tondb-scanner ├── CMakeLists.txt ├── src ├── DataParser.cpp ├── DataParser.h ├── DbScanner.cpp ├── DbScanner.h ├── EventProcessor.cpp ├── EventProcessor.h ├── IndexData.h ├── InsertManager.cpp ├── InsertManager.h ├── InsertManagerBase.cpp ├── InsertManagerBase.h ├── InterfaceDetectors.hpp ├── Statistics.cpp ├── Statistics.h ├── TraceAssembler.cpp ├── TraceAssembler.h ├── convert-utils.cpp ├── convert-utils.h ├── msgpack-utils.h ├── parse_token_data.cpp ├── parse_token_data.h ├── queue_state.cpp ├── queue_state.h ├── smc-interfaces │ ├── InterfacesDetector.h │ ├── NftSale.cpp │ ├── NftSale.h │ ├── Tokens.cpp │ ├── Tokens.h │ ├── execute-smc.cpp │ └── execute-smc.h └── tlb │ └── tokens.tlb └── test └── tests.cpp /.dockerignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | .idea/ 161 | 162 | # Prerequisites 163 | *.d 164 | 165 | # Compiled Object files 166 | *.slo 167 | *.lo 168 | *.o 169 | *.obj 170 | 171 | # Precompiled Headers 172 | *.gch 173 | *.pch 174 | 175 | # Compiled Dynamic libraries 176 | *.so 177 | *.dylib 178 | *.dll 179 | 180 | # Fortran module files 181 | *.mod 182 | *.smod 183 | 184 | # Compiled Static libraries 185 | *.lai 186 | *.la 187 | *.a 188 | *.lib 189 | 190 | # Executables 191 | *.exe 192 | *.out 193 | *.app 194 | 195 | # VS code 196 | .vscode/* 197 | !.vscode/settings.json 198 | !.vscode/tasks.json 199 | !.vscode/launch.json 200 | !.vscode/extensions.json 201 | !.vscode/*.code-snippets 202 | 203 | # Local History for Visual Studio Code 204 | .history/ 205 | 206 | # Built Visual Studio Code Extensions 207 | *.vsix 208 | 209 | # CMake 210 | CMakeLists.txt.user 211 | CMakeCache.txt 212 | CMakeFiles 213 | CMakeScripts 214 | Testing 215 | Makefile 216 | cmake_install.cmake 217 | install_manifest.txt 218 | compile_commands.json 219 | CTestTestfile.cmake 220 | _deps 221 | 222 | # build 223 | [Bb][Uu][Ii][Ll][Dd] 224 | sandbox/ 225 | 226 | # vs code 227 | .vscode/ 228 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Test pull request 2 | 3 | on: 4 | pull_request: 5 | branches: [ "main" ] 6 | 7 | permissions: 8 | contents: read 9 | 10 | env: 11 | BUILD_TYPE: Release 12 | 13 | jobs: 14 | build: 15 | runs-on: ubuntu-22.04 16 | 17 | steps: 18 | - name: Check out repository 19 | uses: actions/checkout@v3 20 | with: 21 | submodules: 'recursive' 22 | 23 | - name: Install system libraries 24 | run: | 25 | sudo apt-get update 26 | sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev 27 | wget https://apt.llvm.org/llvm.sh 28 | chmod +x llvm.sh 29 | sudo ./llvm.sh 16 all 30 | 31 | 32 | - name: Configure CMake 33 | run: cmake -B ${{github.workspace}}/build 34 | 35 | - name: Build 36 | # Build your program with the given configuration 37 | run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} 38 | 39 | - name: Test 40 | working-directory: ${{github.workspace}}/build 41 | run: ./tondb-scanner/test-tondb 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | .idea/ 161 | 162 | # Prerequisites 163 | *.d 164 | 165 | # Compiled Object files 166 | *.slo 167 | *.lo 168 | *.o 169 | *.obj 170 | 171 | # Precompiled Headers 172 | *.gch 173 | *.pch 174 | 175 | # Compiled Dynamic libraries 176 | *.so 177 | *.dylib 178 | *.dll 179 | 180 | # Compiled Static libraries 181 | *.lai 182 | *.la 183 | *.a 184 | *.lib 185 | 186 | # Executables 187 | *.exe 188 | *.app 189 | 190 | # VS code 191 | .vscode/* 192 | !.vscode/settings.json 193 | !.vscode/tasks.json 194 | !.vscode/launch.json 195 | !.vscode/extensions.json 196 | !.vscode/*.code-snippets 197 | 198 | # Local History for Visual Studio Code 199 | .history/ 200 | 201 | # Built Visual Studio Code Extensions 202 | *.vsix 203 | 204 | # CMake 205 | CMakeLists.txt.user 206 | CMakeCache.txt 207 | CMakeFiles 208 | CMakeScripts 209 | Testing 210 | Makefile 211 | cmake_install.cmake 212 | install_manifest.txt 213 | compile_commands.json 214 | CTestTestfile.cmake 215 | _deps 216 | 217 | # build 218 | [Bb][Uu][Ii][Ll][Dd] 219 | sandbox/ 220 | 221 | # vs code 222 | .vscode/ 223 | 224 | .DS_Store 225 | 226 | tondb-scanner/src/tokens-tlb.cpp 227 | tondb-scanner/src/tokens-tlb.h 228 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "external/ton"] 2 | path = external/ton 3 | url = https://github.com/dungeon-master-666/ton 4 | branch = db_secondary 5 | [submodule "external/libpqxx"] 6 | path = external/libpqxx 7 | url = https://github.com/jtv/libpqxx.git 8 | [submodule "external/clickhouse-cpp"] 9 | path = external/clickhouse-cpp 10 | url = https://github.com/kdimentionaltree/clickhouse-cpp.git 11 | branch = absl-fix 12 | [submodule "external/hiredis"] 13 | path = external/hiredis 14 | url = https://github.com/redis/hiredis.git 15 | [submodule "external/redis-plus-plus"] 16 | path = external/redis-plus-plus 17 | url = https://github.com/sewenew/redis-plus-plus.git 18 | [submodule "external/msgpack-c"] 19 | path = external/msgpack-c 20 | url = https://github.com/msgpack/msgpack-c 21 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.16) 2 | 3 | project(ton-index-cpp) 4 | 5 | option(PGTON "Enable adding the pgton subdirectory" OFF) 6 | option(TON_USE_JEMALLOC "Use \"ON\" to enable JeMalloc." OFF) 7 | 8 | # Find jemalloc 9 | if (TON_USE_JEMALLOC) 10 | list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/external/ton/CMake") 11 | find_package(jemalloc REQUIRED) 12 | 13 | if (JEMALLOC_FOUND) 14 | include_directories(${JEMALLOC_INCLUDE_DIR}) 15 | link_libraries(${JEMALLOC_LIBRARIES}) 16 | add_compile_definitions(TON_USE_JEMALLOC) 17 | endif() 18 | endif() 19 | 20 | list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake) 21 | add_subdirectory(external/ton EXCLUDE_FROM_ALL) 22 | add_subdirectory(external/libpqxx EXCLUDE_FROM_ALL) 23 | add_subdirectory(external/clickhouse-cpp EXCLUDE_FROM_ALL) 24 | add_subdirectory(external/hiredis EXCLUDE_FROM_ALL) 25 | set(HIREDIS_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/external CACHE PATH "hiredis header" FORCE) 26 | set(HIREDIS_LIB hiredis CACHE STRING "hiredis lib" FORCE) 27 | set(REDIS_PLUS_PLUS_BUILD_TEST OFF CACHE BOOL "disable test" FORCE) 28 | add_subdirectory(external/redis-plus-plus EXCLUDE_FROM_ALL) 29 | set(MSGPACK_USE_BOOST OFF CACHE BOOL "disable boost" FORCE) 30 | set(MSGPACK_USE_STD_VARIANT_ADAPTOR ON CACHE BOOL "enable std::variant" FORCE) 31 | add_subdirectory(external/msgpack-c EXCLUDE_FROM_ALL) 32 | 33 | add_subdirectory(tondb-scanner) 34 | add_subdirectory(ton-index-postgres) 35 | add_subdirectory(ton-index-postgres-v2) 36 | add_subdirectory(ton-index-clickhouse) 37 | add_subdirectory(ton-integrity-checker) 38 | add_subdirectory(ton-smc-scanner) 39 | add_subdirectory(ton-trace-emulator) 40 | add_subdirectory(ton-trace-task-emulator) 41 | add_subdirectory(celldb-migrate) 42 | 43 | if (PGTON) 44 | message("Building pgton") 45 | add_subdirectory(pgton) 46 | endif() 47 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:22.04 as builder 2 | RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get update && apt-get -y install tzdata && rm -rf /var/lib/{apt,dpkg,cache,log}/ 3 | RUN apt update -y \ 4 | && apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev \ 5 | gperf wget git curl ccache libmicrohttpd-dev liblz4-dev \ 6 | pkg-config libsecp256k1-dev libsodium-dev python3-dev libpq-dev \ 7 | && rm -rf /var/lib/{apt,dpkg,cache,log}/ 8 | 9 | # building 10 | COPY external/ /app/external/ 11 | COPY pgton/ /app/pgton/ 12 | COPY sandbox-cpp/ /app/sandbox-cpp/ 13 | COPY ton-index-clickhouse/ /app/ton-index-clickhouse/ 14 | COPY ton-index-postgres/ /app/ton-index-postgres/ 15 | COPY ton-index-postgres-v2/ /app/ton-index-postgres-v2/ 16 | COPY ton-integrity-checker/ /app/ton-integrity-checker/ 17 | COPY ton-smc-scanner/ /app/ton-smc-scanner/ 18 | COPY ton-trace-emulator/ /app/ton-trace-emulator/ 19 | COPY tondb-scanner/ /app/tondb-scanner/ 20 | COPY CMakeLists.txt /app/ 21 | 22 | WORKDIR /app/build 23 | RUN cmake -DCMAKE_BUILD_TYPE=Release .. 24 | RUN make -j$(nproc) 25 | 26 | FROM ubuntu:22.04 27 | RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get update && apt-get -y install tzdata && rm -rf /var/lib/{apt,dpkg,cache,log}/ 28 | RUN apt update -y \ 29 | && apt install -y dnsutils libpq-dev libsecp256k1-dev libsodium-dev \ 30 | && rm -rf /var/lib/{apt,dpkg,cache,log}/ 31 | 32 | COPY scripts/entrypoint.sh /entrypoint.sh 33 | COPY --from=builder /app/build/external/libpqxx/src/libpqxx.so /usr/lib/libpqxx.so 34 | COPY --from=builder /app/build/external/libpqxx/src/libpqxx-*.so /usr/lib/ 35 | COPY --from=builder /app/build/ton-index-postgres/ton-index-postgres /usr/bin/ton-index-postgres 36 | COPY --from=builder /app/build/ton-index-postgres-v2/ton-index-postgres-v2 /usr/bin/ton-index-postgres-v2 37 | COPY --from=builder /app/build/ton-index-clickhouse/ton-index-clickhouse /usr/bin/ton-index-clickhouse 38 | COPY --from=builder /app/build/ton-smc-scanner/ton-smc-scanner /usr/bin/ton-smc-scanner 39 | COPY --from=builder /app/build/ton-integrity-checker/ton-integrity-checker /usr/bin/ton-integrity-checker 40 | COPY --from=builder /app/build/ton-trace-emulator/ton-trace-emulator /usr/bin/ton-trace-emulator 41 | 42 | ENTRYPOINT [ "/entrypoint.sh" ] 43 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TON Index (C++ Worker) 2 | 3 | This is a C++ worker for [TON indexer](https://github.com/toncenter/ton-indexer/). This worker reads data from TON node files, parses TL-B schemas, and inserts data into PostgreSQL database. 4 | 5 | ## 1. How to install 6 | 7 | Before installing the worker, ensure the TON Index database is set up using the [instructions provided](https://github.com/toncenter/ton-indexer/tree/master). 8 | 9 | 10 | ### 1.1. Setup as systemd daemon 11 | To install as a daemon use the script below: 12 | 13 | ./scripts/add2systemd.sh --db /var/ton-work/db --host --port \ 14 | --user --password --dbname \ 15 | --from 1 --max-active-tasks $(nproc) --threads $(nproc) \ 16 | --max-insert-actors [--force] 17 | 18 | You may find the list of arguments in section [available arguments](#13-available-arguments). Use flag `--force` to force rebuild binary. 19 | 20 | **NOTE:** We strongly recommend to setup worker **separately** from the database server because of significant RAM and Disk IO usage. 21 | 22 | ### 1.2. Manual install 23 | 24 | Do the following steps to build and run index worker from source. 25 | 26 | 1. Install required packages: 27 | 28 | sudo apt-get update -y 29 | sudo apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git curl libreadline-dev ccache libmicrohttpd-dev pkg-config libsecp256k1-dev libsodium-dev python3-dev libpq-dev ninja-build 30 | 2. Build TON index worker binary: 31 | 32 | mkdir -p build 33 | cd build 34 | cmake -DCMAKE_BUILD_TYPE=Release -GNinja .. 35 | ninja -j$(nproc) ton-index-postgres-v2 36 | 37 | 3. Install binary to your system: 38 | 39 | sudo cmake --install 40 | 41 | 4. Increase maximum opened files limit: 42 | 43 | ulimit -n 1000000 44 | 45 | 5. Run TON index worker: 46 | 47 | ton-index-postgres-v2 48 | 49 | ### 1.3. Available arguments: 50 | * `--db ` - path to TON node directory. Pass `/var/ton-work/db`, if you have TON node installed by mytonctrl. **Required**. 51 | * `--host ` - PostgreSQL host. Only IPv4 is acceptable. Default: `127.0.0.1`. 52 | * `--port ` - PostgreSQL port. Default: `5432`. 53 | * `--user ` - PostgreSQL user. Default: `postgres`. 54 | * `--password ` - PostgreSQL password. Default: empty password. 55 | * `--dbname ` - PostgreSQL database name. Default: `ton_index`. 56 | * `--from ` - Masterchain seqno to start indexing from. Use value `1` to index the whole blockchain. 57 | * `--max-active-tasks ` - maximum parallel disk reading tasks. Recommended value is number of CPU cores. 58 | * `--max-queue-blocks ` - maximum blocks in queue (prefetched blocks from disk). 59 | * `--max-queue-txs ` - maximum transactions in queue. 60 | * `--max-queue-msgs ` - maximum messages in queue. 61 | * `--max-insert-actors ` - maximum concurrent INSERT queries. 62 | * `--max-batch-blocks ` - maximum blocks in batch (size of insert batch). 63 | * `--max-batch-txs ` - maximum transactions in batch. 64 | * `--max-batch-msgs ` - maximum messages in batch. 65 | * `--max-data-depth ` - maximum depth of data boc to index (use 0 to index all accounts). 66 | * `--threads ` - number of CPU threads. 67 | * `--stats-freq ` - frequency of printing a statistics. 68 | 69 | -------------------------------------------------------------------------------- /celldb-migrate/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.16) 2 | 3 | add_executable(celldb-migrate 4 | src/main.cpp 5 | ) 6 | target_include_directories(celldb-migrate 7 | PUBLIC src 8 | ) 9 | target_compile_features(celldb-migrate PRIVATE cxx_std_20) 10 | target_link_libraries(celldb-migrate tondb-scanner rocksdb) 11 | 12 | install(TARGETS celldb-migrate RUNTIME DESTINATION bin) 13 | -------------------------------------------------------------------------------- /celldb-migrate/src/main.cpp: -------------------------------------------------------------------------------- 1 | #include "td/utils/port/signals.h" 2 | #include "td/utils/OptionParser.h" 3 | #include "td/utils/format.h" 4 | #include "td/utils/logging.h" 5 | #include "td/utils/check.h" 6 | #include "td/utils/port/path.h" 7 | #include "td/actor/actor.h" 8 | #include "crypto/vm/cp0.h" 9 | #include "tddb/td/db/RocksDb.h" 10 | #include 11 | #include "rocksdb/utilities/optimistic_transaction_db.h" 12 | #include "crypto/vm/db/DynamicBagOfCellsDb.h" 13 | #include "crypto/vm/db/CellStorage.h" 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | 21 | 22 | int main(int argc, char* argv[]) { 23 | SET_VERBOSITY_LEVEL(verbosity_INFO); 24 | td::set_default_failure_signal_handler().ensure(); 25 | 26 | td::OptionParser p; 27 | std::string db_root; 28 | p.set_description("Migrate DB from block based to plain table"); 29 | p.add_option('\0', "help", "prints_help", [&]() { 30 | char b[10240]; 31 | td::StringBuilder sb(td::MutableSlice{b, 10000}); 32 | sb << p; 33 | std::cout << sb.as_cslice().c_str(); 34 | std::exit(2); 35 | }); 36 | p.add_option('\0', "db", "Path to DB folder", [&](td::Slice fname) { 37 | db_root = fname.str(); 38 | }); 39 | 40 | auto S = p.run(argc, argv); 41 | if (S.is_error()) { 42 | LOG(ERROR) << "failed to parse options: " << S.move_as_error(); 43 | std::_Exit(2); 44 | } 45 | if (db_root.empty()) { 46 | LOG(ERROR) << "db path is empty"; 47 | std::_Exit(2); 48 | } 49 | 50 | rocksdb::DB* db_raw; 51 | rocksdb::Options options; 52 | options.create_if_missing = false; 53 | rocksdb::BlockBasedTableOptions table_options; 54 | table_options.block_cache = rocksdb::NewLRUCache(32ULL << 30); // 8GB cache 55 | // table_options.cache_index_and_filter_blocks = true; 56 | table_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false)); // 10 bits per key 57 | table_options.index_type = rocksdb::BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; 58 | table_options.partition_filters = true; 59 | options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); 60 | 61 | 62 | rocksdb::Status status = rocksdb::DB::Open(options, db_root, &db_raw); 63 | if (!status.ok()) { 64 | LOG(FATAL) << "failed to open src db: " << status.ToString(); 65 | } 66 | auto db = std::shared_ptr(db_raw); 67 | // rocksdb::CompactRangeOptions compact_options; 68 | // compact_options.max_subcompactions = 24; 69 | // compact_options.exclusive_manual_compaction = true; // Bypass automatic compactions 70 | // compact_options.change_level = true; 71 | // compact_options.bottommost_level_compaction = rocksdb::BottommostLevelCompaction::kForce; 72 | // status = db->CompactRange(compact_options, nullptr, nullptr); // Compact entire key range 73 | // if (!status.ok()) { 74 | // LOG(FATAL) << "failed to compact db: " << status.ToString(); 75 | // } 76 | // status = db->Flush({}); 77 | // if (!status.ok()) { 78 | // LOG(FATAL) << "failed to flush db: " << status.ToString(); 79 | // } 80 | // status = db->WaitForCompact({}); 81 | // if (!status.ok()) { 82 | // LOG(FATAL) << "failed to wait for compaction: " << status.ToString(); 83 | // } 84 | // status = db->Close(); 85 | // if (!status.ok()) { 86 | // LOG(FATAL) << "failed to close db: " << status.ToString(); 87 | // } 88 | 89 | std::string block_cache_usage; 90 | db->GetProperty("rocksdb.block-cache-usage", &block_cache_usage); 91 | LOG(INFO) << "rocksdb.block_cache_usage: " << block_cache_usage; 92 | std::string table_readers_mem; 93 | db->GetProperty("rocksdb.estimate-table-readers-mem", &table_readers_mem); 94 | LOG(INFO) << "rocksdb.estimate-table-readers-mem: " << table_readers_mem; 95 | std::string memtables_mem; 96 | db->GetProperty("rocksdb.cur-size-all-mem-tables", &memtables_mem); 97 | LOG(INFO) << "rocksdb.cur-size-all-mem-tables: " << memtables_mem; 98 | std::string pinned_mem; 99 | db->GetProperty("rocksdb.block-cache-pinned-usage", &pinned_mem); 100 | LOG(INFO) << "rocksdb.block-cache-pinned-usage: " << pinned_mem; 101 | 102 | while (true) { 103 | std::this_thread::sleep_for(std::chrono::seconds(1)); 104 | } 105 | 106 | return 0; 107 | } -------------------------------------------------------------------------------- /pgton/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(pgton) 3 | 4 | find_package(PostgreSQL REQUIRED) 5 | add_postgresql_extension(pgton 6 | VERSION 0.1 7 | SOURCES pgton.c 8 | SCRIPTS pgton--0.1.sql 9 | REGRESS basic) 10 | 11 | enable_testing() 12 | -------------------------------------------------------------------------------- /pgton/README.md: -------------------------------------------------------------------------------- 1 | # PGTON extension 2 | 3 | This extension adds two custom types in PostgreSQL: 4 | * tonhash: acts like a base64 string of size 44 bytes, but is stored in 32 bytes. 5 | * tonaddr: acts like a TON address in raw-form string of size 66-67 bytes, but stored in 36 bytes. 6 | 7 | Note: the functionality may be extended later. 8 | 9 | ## Build extension 10 | 11 | Run the following commands on machine with PostgreSQL: 12 | * Install dependencies: `sudo apt update && sudo apt install postgresql-server-dev-all cmake`. 13 | * Create build directory: `mkdir build && cd build`. 14 | * Configure and build: `cmake -DCMAKE_BUILD_TYPE=Release .. && make -j32 pgton`. 15 | * Install binaries: `cd pgton && sudo make install`. 16 | * Drop existing database and start TON Index worker with flag `--custom-types` to create a scheme with custom types. 17 | -------------------------------------------------------------------------------- /pgton/expected/basic.out: -------------------------------------------------------------------------------- 1 | CREATE TABLE test(id SERIAL, h TONHASH, a TONADDR); 2 | INSERT INTO test(h, a) VALUES 3 | ('ANT/iLBgHDmV1FwjM/EWihj4////////AAAAAAAAAAA=', '0:934F64BE8E43994563C6FCAAAA18B772B74E7D314D3D87CAD992F8711D32C635'), 4 | ('ANT/iLBgHDlgMYBZXVUAABj4////////AAAAAAAAAAA=', '-1:79DCEFAE9F68AB8F4D8A2CDABCE377A94365881866540B0FF87860851493D2C4'), 5 | ('5+p73p1noxIgPdD2SFf2Jk7heaHA8i1lkB9HP53iMY0=', '-1001:0000000000000000011100000000000000000000000000000000aaaaaaaaaaaa'); 6 | SELECT * FROM test; 7 | id | h | a 8 | ----+----------------------------------------------+------------------------------------------------------------------------ 9 | 1 | ANT/iLBgHDmV1FwjM/EWihj4////////AAAAAAAAAAA= | 0:934F64BE8E43994563C6FCAAAA18B772B74E7D314D3D87CAD992F8711D32C635 10 | 2 | ANT/iLBgHDlgMYBZXVUAABj4////////AAAAAAAAAAA= | -1:79DCEFAE9F68AB8F4D8A2CDABCE377A94365881866540B0FF87860851493D2C4 11 | 3 | 5+p73p1noxIgPdD2SFf2Jk7heaHA8i1lkB9HP53iMY0= | -1001:0000000000000000011100000000000000000000000000000000AAAAAAAAAAAA 12 | (3 rows) 13 | 14 | CREATE INDEX test_index_1 ON test(h); 15 | CREATE INDEX test_index_2 ON test(id, h); 16 | CREATE INDEX test_index_3 ON test(h, a); 17 | CREATE INDEX test_index_4 ON test(a, h); 18 | CREATE INDEX test_index_5 ON test(a, id); 19 | SELECT * FROM test WHERE h = 'ANT/iLBgHDlgMYBZXVUAABj4////////AAAAAAAAAAA='; 20 | id | h | a 21 | ----+----------------------------------------------+--------------------------------------------------------------------- 22 | 2 | ANT/iLBgHDlgMYBZXVUAABj4////////AAAAAAAAAAA= | -1:79DCEFAE9F68AB8F4D8A2CDABCE377A94365881866540B0FF87860851493D2C4 23 | (1 row) 24 | 25 | SELECT * FROM test WHERE a = '0:934F64BE8E43994563C6FCAAAA18B772B74E7D314D3D87CAD992F8711D32C635'; 26 | id | h | a 27 | ----+----------------------------------------------+-------------------------------------------------------------------- 28 | 1 | ANT/iLBgHDmV1FwjM/EWihj4////////AAAAAAAAAAA= | 0:934F64BE8E43994563C6FCAAAA18B772B74E7D314D3D87CAD992F8711D32C635 29 | (1 row) 30 | 31 | INSERT INTO test(a) VALUES 32 | ('addr_none'), 33 | ('addr_extern'); 34 | SELECT * FROM test; 35 | id | h | a 36 | ----+----------------------------------------------+------------------------------------------------------------------------ 37 | 1 | ANT/iLBgHDmV1FwjM/EWihj4////////AAAAAAAAAAA= | 0:934F64BE8E43994563C6FCAAAA18B772B74E7D314D3D87CAD992F8711D32C635 38 | 2 | ANT/iLBgHDlgMYBZXVUAABj4////////AAAAAAAAAAA= | -1:79DCEFAE9F68AB8F4D8A2CDABCE377A94365881866540B0FF87860851493D2C4 39 | 3 | 5+p73p1noxIgPdD2SFf2Jk7heaHA8i1lkB9HP53iMY0= | -1001:0000000000000000011100000000000000000000000000000000AAAAAAAAAAAA 40 | 4 | | addr_none 41 | 5 | | addr_extern 42 | (5 rows) 43 | 44 | -------------------------------------------------------------------------------- /pgton/pgton--0.1.sql: -------------------------------------------------------------------------------- 1 | CREATE TYPE tonhash; 2 | 3 | CREATE OR REPLACE FUNCTION tonhash_in(cstring) 4 | RETURNS tonhash 5 | AS 'MODULE_PATHNAME', 'tonhash_in' 6 | LANGUAGE C IMMUTABLE STRICT; 7 | 8 | CREATE OR REPLACE FUNCTION tonhash_out(tonhash) 9 | RETURNS cstring 10 | AS 'MODULE_PATHNAME', 'tonhash_out' 11 | LANGUAGE C IMMUTABLE STRICT; 12 | 13 | CREATE FUNCTION tonhash_recv(internal) 14 | RETURNS tonhash 15 | AS 'MODULE_PATHNAME', 'tonhash_recv' 16 | LANGUAGE C IMMUTABLE STRICT; 17 | 18 | CREATE FUNCTION tonhash_send(tonhash) 19 | RETURNS bytea 20 | AS 'MODULE_PATHNAME', 'tonhash_send' 21 | LANGUAGE C IMMUTABLE STRICT; 22 | 23 | 24 | CREATE TYPE tonhash ( 25 | input = tonhash_in, 26 | output = tonhash_out, 27 | send = tonhash_send, 28 | receive = tonhash_recv, 29 | internallength = 32, 30 | alignment = int4 31 | ); 32 | 33 | CREATE FUNCTION tonhash_lt(tonhash, tonhash) RETURNS bool 34 | AS 'MODULE_PATHNAME', 'tonhash_lt' LANGUAGE C IMMUTABLE STRICT; 35 | CREATE FUNCTION tonhash_le(tonhash, tonhash) RETURNS bool 36 | AS 'MODULE_PATHNAME', 'tonhash_le' LANGUAGE C IMMUTABLE STRICT; 37 | CREATE FUNCTION tonhash_eq(tonhash, tonhash) RETURNS bool 38 | AS 'MODULE_PATHNAME', 'tonhash_eq' LANGUAGE C IMMUTABLE STRICT; 39 | CREATE FUNCTION tonhash_gt(tonhash, tonhash) RETURNS bool 40 | AS 'MODULE_PATHNAME', 'tonhash_gt' LANGUAGE C IMMUTABLE STRICT; 41 | CREATE FUNCTION tonhash_ge(tonhash, tonhash) RETURNS bool 42 | AS 'MODULE_PATHNAME', 'tonhash_ge' LANGUAGE C IMMUTABLE STRICT; 43 | CREATE FUNCTION tonhash_cmp(tonhash, tonhash) RETURNS int4 44 | AS 'MODULE_PATHNAME', 'tonhash_cmp' LANGUAGE C IMMUTABLE STRICT; 45 | 46 | CREATE OPERATOR < ( 47 | leftarg = tonhash, rightarg = tonhash, procedure = tonhash_lt, 48 | commutator = > , negator = >= , 49 | restrict = scalarltsel, join = scalarltjoinsel 50 | ); 51 | CREATE OPERATOR <= ( 52 | leftarg = tonhash, rightarg = tonhash, procedure = tonhash_le, 53 | commutator = >= , negator = > , 54 | restrict = scalarlesel, join = scalarlejoinsel 55 | ); 56 | CREATE OPERATOR = ( 57 | leftarg = tonhash, rightarg = tonhash, procedure = tonhash_eq, 58 | commutator = = , 59 | restrict = eqsel, join = eqjoinsel 60 | ); 61 | CREATE OPERATOR >= ( 62 | leftarg = tonhash, rightarg = tonhash, procedure = tonhash_ge, 63 | commutator = <= , negator = < , 64 | restrict = scalargesel, join = scalargejoinsel 65 | ); 66 | CREATE OPERATOR > ( 67 | leftarg = tonhash, rightarg = tonhash, procedure = tonhash_gt, 68 | commutator = < , negator = <= , 69 | restrict = scalargtsel, join = scalargtjoinsel 70 | ); 71 | 72 | CREATE OPERATOR CLASS tonhash_ops 73 | DEFAULT FOR TYPE tonhash USING btree AS 74 | OPERATOR 1 < , 75 | OPERATOR 2 <= , 76 | OPERATOR 3 = , 77 | OPERATOR 4 >= , 78 | OPERATOR 5 > , 79 | FUNCTION 1 tonhash_cmp(tonhash, tonhash); 80 | 81 | -- TonAddr type 82 | CREATE TYPE tonaddr; 83 | 84 | CREATE OR REPLACE FUNCTION tonaddr_in(cstring) 85 | RETURNS tonaddr 86 | AS 'MODULE_PATHNAME', 'tonaddr_in' 87 | LANGUAGE C IMMUTABLE STRICT; 88 | 89 | CREATE OR REPLACE FUNCTION tonaddr_out(tonaddr) 90 | RETURNS cstring 91 | AS 'MODULE_PATHNAME', 'tonaddr_out' 92 | LANGUAGE C IMMUTABLE STRICT; 93 | 94 | CREATE FUNCTION tonaddr_recv(internal) 95 | RETURNS tonaddr 96 | AS 'MODULE_PATHNAME', 'tonaddr_recv' 97 | LANGUAGE C IMMUTABLE STRICT; 98 | 99 | CREATE FUNCTION tonaddr_send(tonaddr) 100 | RETURNS bytea 101 | AS 'MODULE_PATHNAME', 'tonaddr_send' 102 | LANGUAGE C IMMUTABLE STRICT; 103 | 104 | 105 | CREATE TYPE tonaddr ( 106 | input = tonaddr_in, 107 | output = tonaddr_out, 108 | send = tonaddr_send, 109 | receive = tonaddr_recv, 110 | internallength = 36, 111 | alignment = int4 112 | ); 113 | 114 | CREATE FUNCTION tonaddr_lt(tonaddr, tonaddr) RETURNS bool 115 | AS 'MODULE_PATHNAME', 'tonaddr_lt' LANGUAGE C IMMUTABLE STRICT; 116 | CREATE FUNCTION tonaddr_le(tonaddr, tonaddr) RETURNS bool 117 | AS 'MODULE_PATHNAME', 'tonaddr_le' LANGUAGE C IMMUTABLE STRICT; 118 | CREATE FUNCTION tonaddr_eq(tonaddr, tonaddr) RETURNS bool 119 | AS 'MODULE_PATHNAME', 'tonaddr_eq' LANGUAGE C IMMUTABLE STRICT; 120 | CREATE FUNCTION tonaddr_gt(tonaddr, tonaddr) RETURNS bool 121 | AS 'MODULE_PATHNAME', 'tonaddr_gt' LANGUAGE C IMMUTABLE STRICT; 122 | CREATE FUNCTION tonaddr_ge(tonaddr, tonaddr) RETURNS bool 123 | AS 'MODULE_PATHNAME', 'tonaddr_ge' LANGUAGE C IMMUTABLE STRICT; 124 | CREATE FUNCTION tonaddr_cmp(tonaddr, tonaddr) RETURNS int4 125 | AS 'MODULE_PATHNAME', 'tonaddr_cmp' LANGUAGE C IMMUTABLE STRICT; 126 | 127 | CREATE OPERATOR < ( 128 | leftarg = tonaddr, rightarg = tonaddr, procedure = tonaddr_lt, 129 | commutator = > , negator = >= , 130 | restrict = scalarltsel, join = scalarltjoinsel 131 | ); 132 | CREATE OPERATOR <= ( 133 | leftarg = tonaddr, rightarg = tonaddr, procedure = tonaddr_le, 134 | commutator = >= , negator = > , 135 | restrict = scalarlesel, join = scalarlejoinsel 136 | ); 137 | CREATE OPERATOR = ( 138 | leftarg = tonaddr, rightarg = tonaddr, procedure = tonaddr_eq, 139 | commutator = = , 140 | restrict = eqsel, join = eqjoinsel 141 | ); 142 | CREATE OPERATOR >= ( 143 | leftarg = tonaddr, rightarg = tonaddr, procedure = tonaddr_ge, 144 | commutator = <= , negator = < , 145 | restrict = scalargesel, join = scalargejoinsel 146 | ); 147 | CREATE OPERATOR > ( 148 | leftarg = tonaddr, rightarg = tonaddr, procedure = tonaddr_gt, 149 | commutator = < , negator = <= , 150 | restrict = scalargtsel, join = scalargtjoinsel 151 | ); 152 | 153 | CREATE OPERATOR CLASS tonaddr_ops 154 | DEFAULT FOR TYPE tonaddr USING btree AS 155 | OPERATOR 1 < , 156 | OPERATOR 2 <= , 157 | OPERATOR 3 = , 158 | OPERATOR 4 >= , 159 | OPERATOR 5 > , 160 | FUNCTION 1 tonaddr_cmp(tonaddr, tonaddr); 161 | -------------------------------------------------------------------------------- /pgton/sql/basic.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE test(id SERIAL, h TONHASH, a TONADDR); 2 | INSERT INTO test(h, a) VALUES 3 | ('ANT/iLBgHDmV1FwjM/EWihj4////////AAAAAAAAAAA=', '0:934F64BE8E43994563C6FCAAAA18B772B74E7D314D3D87CAD992F8711D32C635'), 4 | ('ANT/iLBgHDlgMYBZXVUAABj4////////AAAAAAAAAAA=', '-1:79DCEFAE9F68AB8F4D8A2CDABCE377A94365881866540B0FF87860851493D2C4'), 5 | ('5+p73p1noxIgPdD2SFf2Jk7heaHA8i1lkB9HP53iMY0=', '-1001:0000000000000000011100000000000000000000000000000000aaaaaaaaaaaa'); 6 | 7 | SELECT * FROM test; 8 | 9 | CREATE INDEX test_index_1 ON test(h); 10 | CREATE INDEX test_index_2 ON test(id, h); 11 | CREATE INDEX test_index_3 ON test(h, a); 12 | CREATE INDEX test_index_4 ON test(a, h); 13 | CREATE INDEX test_index_5 ON test(a, id); 14 | 15 | SELECT * FROM test WHERE h = 'ANT/iLBgHDlgMYBZXVUAABj4////////AAAAAAAAAAA='; 16 | SELECT * FROM test WHERE a = '0:934F64BE8E43994563C6FCAAAA18B772B74E7D314D3D87CAD992F8711D32C635'; 17 | 18 | INSERT INTO test(a) VALUES 19 | ('addr_none'), 20 | ('addr_extern'); 21 | SELECT * FROM test; 22 | -------------------------------------------------------------------------------- /scripts/add2systemd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | function usage() { 5 | echo "Usage:" 6 | exit 7 | } 8 | 9 | 10 | POSITIONAL_ARGS=() 11 | FORCE_BUILD=0 12 | 13 | TASK_ARGS= 14 | 15 | while [[ $# -gt 0 ]]; do 16 | case $1 in 17 | -h|--help) 18 | usage 19 | ;; 20 | -f|--force) 21 | FORCE_BUILD=1 22 | shift;; 23 | -*|--*) 24 | echo "Adding argument '$1 $2' to daemon" 25 | TASK_ARGS="${TASK_ARGS} $1 $2" 26 | shift; shift;; 27 | *) 28 | POSITIONAL_ARGS+=($1) 29 | shift;; 30 | esac 31 | done 32 | 33 | # install libraries 34 | sudo apt update 35 | sudo apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev \ 36 | gperf wget git curl libreadline-dev ccache libmicrohttpd-dev liblz4-dev \ 37 | pkg-config libsecp256k1-dev libsodium-dev python3-dev libpq-dev ninja-build 38 | 39 | # build 40 | if [[ $FORCE_BUILD -eq "1" ]]; then 41 | echo "WARNING! Force building binary" 42 | rm -rf ./build 43 | fi 44 | 45 | if [[ -f "./build" ]]; then 46 | echo "Directory build exists" 47 | else 48 | mkdir -p build 49 | cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=off -GNinja -S . -B ./build 50 | ninja -C ./build -j$(nproc) ton-index-postgres-v2 51 | sudo cmake --install build/ 52 | fi 53 | 54 | # setup daemon 55 | echo "Task args: \'$TASK_ARGS\'" 56 | cat <&1' 66 | ExecStopPost = /bin/echo "index-worker service down" 67 | User = root 68 | Group = root 69 | LimitNOFILE = infinity 70 | LimitNPROC = infinity 71 | LimitMEMLOCK = infinity 72 | 73 | [Install] 74 | WantedBy = multi-user.target 75 | EOF 76 | 77 | # enable service 78 | sudo systemctl daemon-reload 79 | sudo systemctl enable index-worker.service 80 | sudo systemctl restart index-worker.service 81 | -------------------------------------------------------------------------------- /scripts/create_indexes.sql: -------------------------------------------------------------------------------- 1 | begin; 2 | 3 | -- blocks 4 | create index if not exists blocks_index_1 on blocks (gen_utime asc); 5 | create index if not exists blocks_index_2 on blocks (mc_block_seqno asc); 6 | create index if not exists blocks_index_3 on blocks (seqno asc) where (workchain = '-1'::integer); 7 | create index if not exists blocks_index_4 on blocks (start_lt asc); 8 | 9 | -- transactions 10 | create index if not exists transactions_index_1 on transactions (block_workchain, block_shard, block_seqno); 11 | create index if not exists transactions_index_2 on transactions (lt asc); 12 | create index if not exists transactions_index_3 on transactions (now asc, lt asc); 13 | create index if not exists transactions_index_4 on transactions (account, lt asc); 14 | create index if not exists transactions_index_5 on transactions (account, now asc, lt asc); 15 | create index if not exists transactions_index_6 on transactions (hash); 16 | create index if not exists transactions_index_7 on transactions (trace_id, lt asc); 17 | create index if not exists transactions_index_8 on transactions (mc_block_seqno asc, lt asc); 18 | 19 | -- messages 20 | create index if not exists messages_index_1 on messages (msg_hash); 21 | create index if not exists messages_index_2 on messages (source, created_lt asc); 22 | create index if not exists messages_index_3 on messages (destination, created_lt asc); 23 | create index if not exists messages_index_4 on messages (body_hash); 24 | create index if not exists messages_index_5 on messages (trace_id, tx_lt asc); 25 | create index if not exists messages_index_6 on messages (opcode, created_lt); 26 | 27 | -- account states 28 | create index if not exists latest_account_states_index_1 on latest_account_states (balance desc); 29 | create index if not exists latest_account_states_index_2 on latest_account_states (id asc); 30 | create index if not exists latest_account_states_address_book_index on latest_account_states (account) include (account_friendly, code_hash, account_status); 31 | 32 | -- jettons 33 | create index if not exists jetton_masters_index_1 on jetton_masters (admin_address, id asc); 34 | create index if not exists jetton_masters_index_2 on jetton_masters (id asc); 35 | 36 | create index if not exists jetton_wallets_index_1 on jetton_wallets (owner, id asc); 37 | create index if not exists jetton_wallets_index_2 on jetton_wallets (jetton, id asc); 38 | create index if not exists jetton_wallets_index_3 on jetton_wallets (id asc); 39 | create index if not exists jetton_wallets_index_4 on jetton_wallets (jetton, balance desc); 40 | create index if not exists jetton_wallets_index_5 on jetton_wallets (owner, balance desc); 41 | 42 | create index if not exists jetton_transfers_index_1 on jetton_transfers (source, tx_now asc); 43 | create index if not exists jetton_transfers_index_2 on jetton_transfers (source, tx_lt asc); 44 | create index if not exists jetton_transfers_index_3 on jetton_transfers (destination, tx_lt asc); 45 | create index if not exists jetton_transfers_index_4 on jetton_transfers (destination, tx_now asc); 46 | create index if not exists jetton_transfers_index_6 on jetton_transfers (jetton_wallet_address, tx_lt asc); 47 | create index if not exists jetton_transfers_index_7 on jetton_transfers (jetton_master_address, tx_now asc); 48 | create index if not exists jetton_transfers_index_8 on jetton_transfers (jetton_master_address, tx_lt asc); 49 | create index if not exists jetton_transfers_index_9 on jetton_transfers (tx_now asc, tx_lt asc); 50 | create index if not exists jetton_transfers_index_10 on jetton_transfers (tx_lt asc); 51 | 52 | create index if not exists jetton_burns_index_1 on jetton_burns (owner, tx_now asc, tx_lt asc); 53 | create index if not exists jetton_burns_index_2 on jetton_burns (owner, tx_lt asc); 54 | create index if not exists jetton_burns_index_3 on jetton_burns (jetton_wallet_address, tx_now asc, tx_lt asc); 55 | create index if not exists jetton_burns_index_4 on jetton_burns (jetton_wallet_address, tx_lt asc); 56 | create index if not exists jetton_burns_index_5 on jetton_burns (jetton_master_address, tx_now asc, tx_lt asc); 57 | create index if not exists jetton_burns_index_6 on jetton_burns (jetton_master_address, tx_lt asc); 58 | create index if not exists jetton_burns_index_7 on jetton_burns (tx_now asc, tx_lt asc); 59 | create index if not exists jetton_burns_index_8 on jetton_burns (tx_lt asc); 60 | 61 | -- nfts 62 | create index if not exists nft_collections_index_1 on nft_collections (owner_address, id asc); 63 | create index if not exists nft_collections_index_2 on nft_collections (id asc); 64 | 65 | create index if not exists nft_items_index_1 on nft_items (collection_address, index asc); 66 | create index if not exists nft_items_index_2 on nft_items (owner_address, collection_address asc, index asc); 67 | create index if not exists nft_items_index_3 on nft_items (id asc); 68 | 69 | -- create index if not exists nft_transfers_index_1 on nft_transfers (nft_item_address, tx_now asc, tx_lt asc); 70 | create index if not exists nft_transfers_index_2 on nft_transfers (nft_item_address, tx_lt asc); 71 | create index if not exists nft_transfers_index_3 on nft_transfers (nft_collection_address, tx_now asc); 72 | create index if not exists nft_transfers_index_4 on nft_transfers (nft_collection_address, tx_lt asc); 73 | create index if not exists nft_transfers_index_5 on nft_transfers (old_owner, tx_lt asc); 74 | -- create index if not exists nft_transfers_index_6 on nft_transfers (old_owner, tx_now asc, tx_lt asc); 75 | create index if not exists nft_transfers_index_7 on nft_transfers (new_owner, tx_lt asc); 76 | -- create index if not exists nft_transfers_index_8 on nft_transfers (new_owner, tx_now asc, tx_lt asc); 77 | create index if not exists nft_transfers_index_9 on nft_transfers (tx_lt asc); 78 | create index if not exists nft_transfers_index_10 on nft_transfers (tx_now asc, tx_lt asc); 79 | 80 | -- traces 81 | create index if not exists traces_index_1 on traces (state); 82 | create index if not exists trace_index_2a on traces (mc_seqno_end asc); 83 | -- create index if not exists traces_index_3 on traces (end_lt asc); 84 | -- create index if not exists traces_index_4 on traces (end_utime asc); 85 | -- create index if not exists traces_index_5 on traces (external_hash, end_lt asc); 86 | -- create index if not exists traces_index_6 on traces (external_hash, end_utime asc); 87 | create index if not exists traces_index_7 on traces (classification_state); 88 | 89 | create index if not exists trace_edges_index_1 on trace_edges (incomplete); 90 | -- create index if not exists trace_edges_index_2 on trace_edges (msg_hash); 91 | 92 | -- create index if not exists actions_index_1 on actions (trace_id, start_lt, end_lt); 93 | create index if not exists actions_index_2 on actions (action_id); 94 | commit; 95 | -------------------------------------------------------------------------------- /scripts/drop_indexes.sql: -------------------------------------------------------------------------------- 1 | begin; 2 | 3 | -- blocks 4 | drop index if exists blocks_index_1; 5 | drop index if exists blocks_index_2; 6 | drop index if exists blocks_index_3; 7 | drop index if exists blocks_index_4; 8 | 9 | -- transactions 10 | drop index if exists transactions_index_1; 11 | drop index if exists transactions_index_2; 12 | drop index if exists transactions_index_3; 13 | drop index if exists transactions_index_4; 14 | drop index if exists transactions_index_5; 15 | drop index if exists transactions_index_6; 16 | drop index if exists transactions_index_7; 17 | drop index if exists transactions_index_8; 18 | 19 | -- messages 20 | drop index if exists messages_index_1; 21 | drop index if exists messages_index_2; 22 | drop index if exists messages_index_3; 23 | drop index if exists messages_index_4; 24 | drop index if exists messages_index_5; 25 | drop index if exists messages_index_6; 26 | 27 | -- account states 28 | drop index if exists latest_account_states_index_1; 29 | drop index if exists latest_account_states_index_2; 30 | drop index if exists latest_account_states_address_book_index; 31 | 32 | -- jettons 33 | drop index if exists jetton_masters_index_1; 34 | drop index if exists jetton_masters_index_2; 35 | 36 | drop index if exists jetton_wallets_index_1; 37 | drop index if exists jetton_wallets_index_2; 38 | drop index if exists jetton_wallets_index_3; 39 | drop index if exists jetton_wallets_index_4; 40 | drop index if exists jetton_wallets_index_5; 41 | 42 | drop index if exists jetton_transfers_index_1; 43 | drop index if exists jetton_transfers_index_2; 44 | drop index if exists jetton_transfers_index_3; 45 | drop index if exists jetton_transfers_index_4; 46 | drop index if exists jetton_transfers_index_5; 47 | drop index if exists jetton_transfers_index_6; 48 | drop index if exists jetton_transfers_index_7; 49 | drop index if exists jetton_transfers_index_8; 50 | drop index if exists jetton_transfers_index_9; 51 | drop index if exists jetton_transfers_index_10; 52 | 53 | drop index if exists jetton_burns_index_1; 54 | drop index if exists jetton_burns_index_2; 55 | drop index if exists jetton_burns_index_3; 56 | drop index if exists jetton_burns_index_4; 57 | drop index if exists jetton_burns_index_5; 58 | drop index if exists jetton_burns_index_6; 59 | drop index if exists jetton_burns_index_7; 60 | drop index if exists jetton_burns_index_8; 61 | 62 | -- nfts 63 | drop index if exists nft_collections_index_1; 64 | drop index if exists nft_collections_index_2; 65 | 66 | drop index if exists nft_items_index_1; 67 | drop index if exists nft_items_index_2; 68 | drop index if exists nft_items_index_3; 69 | 70 | drop index if exists nft_transfers_index_1; 71 | drop index if exists nft_transfers_index_2; 72 | drop index if exists nft_transfers_index_3; 73 | drop index if exists nft_transfers_index_4; 74 | drop index if exists nft_transfers_index_5; 75 | drop index if exists nft_transfers_index_6; 76 | drop index if exists nft_transfers_index_7; 77 | drop index if exists nft_transfers_index_8; 78 | drop index if exists nft_transfers_index_9; 79 | drop index if exists nft_transfers_index_10; 80 | 81 | -- traces 82 | drop index if exists traces_index_1; 83 | drop index if exists traces_index_2; 84 | drop index if exists traces_index_3; 85 | drop index if exists traces_index_4; 86 | drop index if exists traces_index_5; 87 | drop index if exists traces_index_6; 88 | 89 | drop index if exists trace_edges_index_1; 90 | drop index if exists trace_edges_index_2; 91 | 92 | drop index if exists trace_unclassified_index; 93 | commit; 94 | -------------------------------------------------------------------------------- /scripts/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # postgres password 5 | if [ ! -z "$POSTGRES_PASSWORD_FILE" ]; then 6 | echo "Postgres password file: ${POSTGRES_PASSWORD_FILE}" 7 | if [ ! -f "${POSTGRES_PASSWORD_FILE}" ]; then 8 | echo "Password file specified, but not found" 9 | exit 1 10 | fi 11 | POSTGRES_PASSWORD=$(cat ${POSTGRES_PASSWORD_FILE}) 12 | elif [ ! -z "$POSTGRES_PASSWORD" ]; then 13 | echo "Postgres password specified" 14 | else 15 | echo "Postgres password file not specified!" 16 | exit 1 17 | fi 18 | 19 | POSTGRES_HOST_IP=$(dig +short ${POSTGRES_HOST}) 20 | if [[ -z "$POSTGRES_HOST_IP" ]]; then 21 | POSTGRES_HOST_IP=$POSTGRES_HOST 22 | echo "PostgreSQL host IP: $POSTGRES_HOST_IP" 23 | fi 24 | echo "Postgres host: $POSTGRES_HOST (ip: $POSTGRES_HOST_IP)" 25 | 26 | ulimit -n 1000000 27 | printenv 28 | echo "Running binary ${TON_WORKER_BINARY:-ton-index-postgres}" 29 | ${TON_WORKER_BINARY:-ton-index-postgres} --host $POSTGRES_HOST_IP \ 30 | --port $POSTGRES_PORT \ 31 | --user $POSTGRES_USER \ 32 | --password $POSTGRES_PASSWORD \ 33 | --dbname $POSTGRES_DBNAME \ 34 | --db ${TON_WORKER_DBROOT:-/tondb} $@ 35 | -------------------------------------------------------------------------------- /symbolicate_crash.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$#" -ne 2 ]; then 4 | echo "Usage: $0 " 5 | exit 1 6 | fi 7 | 8 | BINARY_PATH="$1" 9 | STACK_TRACE_FILE="$2" 10 | 11 | if [ ! -f "$BINARY_PATH" ]; then 12 | echo "Error: Binary file not found at $BINARY_PATH" 13 | exit 1 14 | fi 15 | 16 | if [ ! -f "$STACK_TRACE_FILE" ]; then 17 | echo "Error: Stack trace file not found at $STACK_TRACE_FILE" 18 | exit 1 19 | fi 20 | 21 | symbolicate_address() { 22 | ADDRESS="$1" 23 | gdb -batch -ex "file $BINARY_PATH" -ex "info symbol $ADDRESS" 24 | } 25 | 26 | while IFS= read -r line; do 27 | if [[ $line =~ \(\+0x([0-9a-fA-F]+)\) ]]; then 28 | symbolicate_address "0x${BASH_REMATCH[1]}" 29 | fi 30 | done < "$STACK_TRACE_FILE" 31 | -------------------------------------------------------------------------------- /ton-emulate-go/docs/docs.go: -------------------------------------------------------------------------------- 1 | // Package docs Code generated by swaggo/swag. DO NOT EDIT 2 | package docs 3 | 4 | import "github.com/swaggo/swag" 5 | 6 | const docTemplate = `{ 7 | "schemes": {{ marshal .Schemes }}, 8 | "swagger": "2.0", 9 | "info": { 10 | "description": "{{escape .Description}}", 11 | "title": "{{.Title}}", 12 | "contact": {}, 13 | "version": "{{.Version}}" 14 | }, 15 | "host": "{{.Host}}", 16 | "basePath": "{{.BasePath}}", 17 | "paths": { 18 | "/v1/emulateTrace": { 19 | "post": { 20 | "description": "Emulate trace by external message.", 21 | "consumes": [ 22 | "application/json" 23 | ], 24 | "produces": [ 25 | "application/json" 26 | ], 27 | "tags": [ 28 | "emulate" 29 | ], 30 | "summary": "Emulate trace by external message", 31 | "parameters": [ 32 | { 33 | "description": "External Message Request", 34 | "name": "request", 35 | "in": "body", 36 | "required": true, 37 | "schema": { 38 | "$ref": "#/definitions/main.EmulateRequest" 39 | } 40 | } 41 | ], 42 | "responses": {} 43 | } 44 | } 45 | }, 46 | "definitions": { 47 | "main.EmulateRequest": { 48 | "type": "object", 49 | "properties": { 50 | "boc": { 51 | "type": "string", 52 | "example": "te6ccgEBAQEAAgAAAA==" 53 | }, 54 | "ignore_chksig": { 55 | "type": "boolean", 56 | "example": false 57 | }, 58 | "include_code_data": { 59 | "type": "boolean", 60 | "example": false 61 | }, 62 | "with_actions": { 63 | "type": "boolean", 64 | "example": false 65 | } 66 | } 67 | } 68 | } 69 | }` 70 | 71 | // SwaggerInfo holds exported Swagger Info so clients can modify it 72 | var SwaggerInfo = &swag.Spec{ 73 | Version: "0.0.1", 74 | Host: "", 75 | BasePath: "/api/emulate/", 76 | Schemes: []string{}, 77 | Title: "TON Emulate API", 78 | Description: "TON Emulate API provides an endpoint to emulate transactions and traces before committing them to the blockchain.", 79 | InfoInstanceName: "swagger", 80 | SwaggerTemplate: docTemplate, 81 | LeftDelim: "{{", 82 | RightDelim: "}}", 83 | } 84 | 85 | func init() { 86 | swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo) 87 | } 88 | -------------------------------------------------------------------------------- /ton-emulate-go/docs/swagger.json: -------------------------------------------------------------------------------- 1 | { 2 | "swagger": "2.0", 3 | "info": { 4 | "description": "TON Emulate API provides an endpoint to emulate transactions and traces before committing them to the blockchain.", 5 | "title": "TON Emulate API", 6 | "contact": {}, 7 | "version": "0.0.1" 8 | }, 9 | "basePath": "/api/emulate/", 10 | "paths": { 11 | "/v1/emulateTrace": { 12 | "post": { 13 | "description": "Emulate trace by external message.", 14 | "consumes": [ 15 | "application/json" 16 | ], 17 | "produces": [ 18 | "application/json" 19 | ], 20 | "tags": [ 21 | "emulate" 22 | ], 23 | "summary": "Emulate trace by external message", 24 | "parameters": [ 25 | { 26 | "description": "External Message Request", 27 | "name": "request", 28 | "in": "body", 29 | "required": true, 30 | "schema": { 31 | "$ref": "#/definitions/main.EmulateRequest" 32 | } 33 | } 34 | ], 35 | "responses": {} 36 | } 37 | } 38 | }, 39 | "definitions": { 40 | "main.EmulateRequest": { 41 | "type": "object", 42 | "properties": { 43 | "boc": { 44 | "type": "string", 45 | "example": "te6ccgEBAQEAAgAAAA==" 46 | }, 47 | "ignore_chksig": { 48 | "type": "boolean", 49 | "example": false 50 | }, 51 | "include_code_data": { 52 | "type": "boolean", 53 | "example": false 54 | }, 55 | "with_actions": { 56 | "type": "boolean", 57 | "example": false 58 | } 59 | } 60 | } 61 | } 62 | } -------------------------------------------------------------------------------- /ton-emulate-go/docs/swagger.yaml: -------------------------------------------------------------------------------- 1 | basePath: /api/emulate/ 2 | definitions: 3 | main.EmulateRequest: 4 | properties: 5 | boc: 6 | example: te6ccgEBAQEAAgAAAA== 7 | type: string 8 | ignore_chksig: 9 | example: false 10 | type: boolean 11 | include_code_data: 12 | example: false 13 | type: boolean 14 | with_actions: 15 | example: false 16 | type: boolean 17 | type: object 18 | info: 19 | contact: {} 20 | description: TON Emulate API provides an endpoint to emulate transactions and traces 21 | before committing them to the blockchain. 22 | title: TON Emulate API 23 | version: 0.0.1 24 | paths: 25 | /v1/emulateTrace: 26 | post: 27 | consumes: 28 | - application/json 29 | description: Emulate trace by external message. 30 | parameters: 31 | - description: External Message Request 32 | in: body 33 | name: request 34 | required: true 35 | schema: 36 | $ref: '#/definitions/main.EmulateRequest' 37 | produces: 38 | - application/json 39 | responses: {} 40 | summary: Emulate trace by external message 41 | tags: 42 | - emulate 43 | swagger: "2.0" 44 | -------------------------------------------------------------------------------- /ton-emulate-go/go.mod: -------------------------------------------------------------------------------- 1 | module ton-emulate-go 2 | 3 | go 1.23.0 4 | 5 | require ( 6 | github.com/go-redis/redis/v8 v8.11.5 7 | github.com/gofiber/fiber/v2 v2.52.5 8 | github.com/gofiber/swagger v1.1.0 9 | github.com/kdimentionaltree/ton-index-go v0.0.0-20241229144241-5ae3b2976c07 10 | github.com/swaggo/swag v1.16.4 11 | github.com/vmihailenco/msgpack/v5 v5.4.1 12 | ) 13 | 14 | require ( 15 | github.com/KyleBanks/depth v1.2.1 // indirect 16 | github.com/andybalholm/brotli v1.1.1 // indirect 17 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 18 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 19 | github.com/go-openapi/jsonpointer v0.21.0 // indirect 20 | github.com/go-openapi/jsonreference v0.21.0 // indirect 21 | github.com/go-openapi/spec v0.21.0 // indirect 22 | github.com/go-openapi/swag v0.23.0 // indirect 23 | github.com/google/uuid v1.6.0 // indirect 24 | github.com/jackc/pgpassfile v1.0.0 // indirect 25 | github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect 26 | github.com/jackc/pgx/v5 v5.7.2 // indirect 27 | github.com/jackc/puddle/v2 v2.2.2 // indirect 28 | github.com/josharian/intern v1.0.0 // indirect 29 | github.com/klauspost/compress v1.17.11 // indirect 30 | github.com/mailru/easyjson v0.9.0 // indirect 31 | github.com/mattn/go-colorable v0.1.13 // indirect 32 | github.com/mattn/go-isatty v0.0.20 // indirect 33 | github.com/mattn/go-runewidth v0.0.16 // indirect 34 | github.com/onsi/gomega v1.36.2 // indirect 35 | github.com/redis/go-redis/v9 v9.7.0 // indirect 36 | github.com/rivo/uniseg v0.4.7 // indirect 37 | github.com/rogpeppe/go-internal v1.13.1 // indirect 38 | github.com/sigurn/crc16 v0.0.0-20240131213347-83fcde1e29d1 // indirect 39 | github.com/swaggo/files/v2 v2.0.1 // indirect 40 | github.com/valyala/bytebufferpool v1.0.0 // indirect 41 | github.com/valyala/fasthttp v1.58.0 // indirect 42 | github.com/valyala/tcplisten v1.0.0 // indirect 43 | github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect 44 | github.com/xssnick/tonutils-go v1.10.2 // indirect 45 | golang.org/x/crypto v0.31.0 // indirect 46 | golang.org/x/sync v0.10.0 // indirect 47 | golang.org/x/sys v0.28.0 // indirect 48 | golang.org/x/text v0.21.0 // indirect 49 | golang.org/x/tools v0.28.0 // indirect 50 | gopkg.in/yaml.v3 v3.0.1 // indirect 51 | ) 52 | 53 | replace github.com/kdimentionaltree/ton-index-go => github.com/verdigos/ton-index-go v0.0.0-20250128165830-bb9d9e383a5a 54 | -------------------------------------------------------------------------------- /ton-emulate-go/models/convert.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/hex" 6 | "fmt" 7 | "strconv" 8 | 9 | tonindexgo "github.com/kdimentionaltree/ton-index-go/index" 10 | emulated "github.com/kdimentionaltree/ton-index-go/index/emulated" 11 | msgpack "github.com/vmihailenco/msgpack/v5" 12 | ) 13 | 14 | func ConvertHsetToTraceNodeShort(hset map[string]string) (*TraceNodeShort, map[Hash]*Transaction, error) { 15 | trace_id := hset["root_node"] 16 | rootNodeBytes := []byte(hset[trace_id]) 17 | if len(rootNodeBytes) == 0 { 18 | return nil, nil, fmt.Errorf("root node not found") 19 | } 20 | 21 | txMap := make(map[Hash]*Transaction) 22 | node, err := convertNode(hset, rootNodeBytes, txMap) 23 | if err != nil { 24 | return nil, nil, fmt.Errorf("failed to convert root node: %w", err) 25 | } 26 | return node, txMap, nil 27 | } 28 | 29 | func convertNode(hset map[string]string, nodeBytes []byte, txMap map[Hash]*Transaction) (*TraceNodeShort, error) { 30 | var node TraceNode 31 | err := msgpack.Unmarshal(nodeBytes, &node) 32 | if err != nil { 33 | return nil, fmt.Errorf("failed to unmarshal node: %w", err) 34 | } 35 | 36 | txMap[node.Transaction.Hash] = &node.Transaction 37 | 38 | short := &TraceNodeShort{ 39 | TransactionHash: node.Transaction.Hash, 40 | InMsgHash: node.Transaction.InMsg.Hash, 41 | Children: make([]*TraceNodeShort, 0, len(node.Transaction.OutMsgs)), 42 | } 43 | 44 | for _, outMsg := range node.Transaction.OutMsgs { 45 | // Get child node by out message hash 46 | msgKey := base64.StdEncoding.EncodeToString(outMsg.Hash[:]) 47 | if childBytes, exists := hset[msgKey]; exists { 48 | childNode := []byte(childBytes) 49 | nodeShort, err := convertNode(hset, childNode, txMap) 50 | if err != nil { 51 | return nil, fmt.Errorf("failed to convert child node: %w", err) 52 | } 53 | short.Children = append(short.Children, nodeShort) 54 | } 55 | } 56 | 57 | return short, nil 58 | } 59 | 60 | func TransformToAPIResponse(hset map[string]string) (*EmulateTraceResponse, error) { 61 | shortTrace, txMap, err := ConvertHsetToTraceNodeShort(hset) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | var accountStates map[Hash]*AccountState 67 | err = msgpack.Unmarshal([]byte(hset["account_states"]), &accountStates) 68 | if err != nil { 69 | return nil, fmt.Errorf("failed to unmarshal account states: %w", err) 70 | } 71 | 72 | var actionsPointer *[]tonindexgo.Action = nil 73 | actionsData, ok := hset["actions"] 74 | if ok { 75 | actions := make([]emulated.Action, 0) 76 | err = msgpack.Unmarshal([]byte(actionsData), &actions) 77 | if err != nil { 78 | return nil, fmt.Errorf("failed to unmarshal actions: %w", err) 79 | } 80 | 81 | goActions := make([]tonindexgo.Action, 0, len(actions)) 82 | 83 | for _, a := range actions { 84 | // convert trace id from base64 to hex, because in ton-trace-task-emulator we use hex 85 | // todo: unify trace id format 86 | traceIdBytes, err := base64.StdEncoding.DecodeString(a.TraceId) 87 | if err != nil { 88 | return nil, fmt.Errorf("failed to decode trace id: %w", err) 89 | } 90 | a.TraceId = hex.EncodeToString(traceIdBytes) 91 | row, err := a.GetActionRow() 92 | if err != nil { 93 | return nil, fmt.Errorf("failed to get action row: %w", err) 94 | } 95 | pgx_row := emulated.NewRow(&row) 96 | 97 | rawAction, err := tonindexgo.ScanRawAction(pgx_row) 98 | if err != nil { 99 | return nil, fmt.Errorf("failed to scan raw action: %w", err) 100 | } 101 | 102 | goAction, err := tonindexgo.ParseRawAction(rawAction) 103 | if err != nil { 104 | return nil, fmt.Errorf("failed to parse action: %w", err) 105 | } 106 | goActions = append(goActions, *goAction) 107 | } 108 | actionsPointer = &goActions 109 | } 110 | 111 | mcBlockSeqno, err := strconv.ParseUint(hset["mc_block_seqno"], 10, 32) 112 | if err != nil { 113 | return nil, fmt.Errorf("failed to convert mc_block_seqno to int: %w", err) 114 | } 115 | 116 | var codeCellsPointer *map[Hash]string 117 | if codeCells, ok := hset["code_cells"]; ok { 118 | var codeCellsMap map[Hash]string 119 | err = msgpack.Unmarshal([]byte(codeCells), &codeCellsMap) 120 | if err != nil { 121 | return nil, fmt.Errorf("failed to unmarshal code cells: %w", err) 122 | } 123 | codeCellsPointer = &codeCellsMap 124 | } 125 | 126 | var dataCellsPointer *map[Hash]string 127 | if dataCells, ok := hset["data_cells"]; ok { 128 | var dataCellsMap map[Hash]string 129 | err = msgpack.Unmarshal([]byte(dataCells), &dataCellsMap) 130 | if err != nil { 131 | return nil, fmt.Errorf("failed to unmarshal data cells: %w", err) 132 | } 133 | dataCellsPointer = &dataCellsMap 134 | } 135 | 136 | response := EmulateTraceResponse{ 137 | McBlockSeqno: uint32(mcBlockSeqno), 138 | Trace: *shortTrace, 139 | Transactions: txMap, 140 | AccountStates: accountStates, 141 | Actions: actionsPointer, 142 | CodeCells: codeCellsPointer, 143 | DataCells: dataCellsPointer, 144 | RandSeed: hset["rand_seed"], 145 | } 146 | return &response, nil 147 | } 148 | -------------------------------------------------------------------------------- /ton-emulate-go/models/response.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import tonindexgo "github.com/kdimentionaltree/ton-index-go/index" 4 | 5 | type TraceNodeShort struct { 6 | TransactionHash Hash `json:"tx_hash"` 7 | InMsgHash Hash `json:"in_msg_hash"` 8 | Children []*TraceNodeShort `json:"children"` 9 | } 10 | 11 | type EmulateTraceResponse struct { 12 | McBlockSeqno uint32 `json:"mc_block_seqno"` 13 | Trace TraceNodeShort `json:"trace"` 14 | Transactions map[Hash]*Transaction `json:"transactions"` 15 | AccountStates map[Hash]*AccountState `json:"account_states"` 16 | Actions *[]tonindexgo.Action `json:"actions,omitempty"` 17 | CodeCells *map[Hash]string `json:"code_cells,omitempty"` 18 | DataCells *map[Hash]string `json:"data_cells,omitempty"` 19 | RandSeed string `json:"rand_seed"` 20 | } 21 | -------------------------------------------------------------------------------- /ton-index-clickhouse/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.16) 2 | 3 | add_executable(ton-index-clickhouse 4 | src/main.cpp 5 | src/InsertManagerClickhouse.cpp 6 | src/IndexScheduler.cpp 7 | ) 8 | 9 | target_compile_features(ton-index-clickhouse PRIVATE cxx_std_20) 10 | target_link_libraries(ton-index-clickhouse tondb-scanner clickhouse-cpp-lib) 11 | 12 | install(TARGETS ton-index-clickhouse RUNTIME DESTINATION bin) 13 | -------------------------------------------------------------------------------- /ton-index-clickhouse/src/IndexScheduler.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "td/actor/actor.h" 4 | 5 | #include "IndexData.h" 6 | #include "DbScanner.h" 7 | #include "EventProcessor.h" 8 | #include "InsertManager.h" 9 | #include "DataParser.h" 10 | 11 | 12 | class IndexScheduler: public td::actor::Actor { 13 | private: 14 | std::queue queued_seqnos_; 15 | std::set processing_seqnos_; 16 | std::set existing_seqnos_; 17 | 18 | td::actor::ActorId db_scanner_; 19 | td::actor::ActorId insert_manager_; 20 | td::actor::ActorId parse_manager_; 21 | td::actor::ActorOwn event_processor_; 22 | std::shared_ptr watcher_; 23 | 24 | std::uint32_t max_active_tasks_{32}; 25 | std::int32_t last_known_seqno_{0}; 26 | std::int32_t last_indexed_seqno_{0}; 27 | std::int32_t from_seqno_{0}; 28 | std::int32_t to_seqno_{0}; 29 | bool force_index_{false}; 30 | 31 | std::double_t avg_tps_{0}; 32 | std::int64_t last_existing_seqno_count_{0}; 33 | 34 | QueueState max_queue_{30000, 30000, 500000, 500000}; 35 | QueueState cur_queue_state_; 36 | 37 | std::int32_t stats_timeout_{10}; 38 | td::Timestamp next_print_stats_; 39 | public: 40 | IndexScheduler(td::actor::ActorId db_scanner, td::actor::ActorId insert_manager, 41 | td::actor::ActorId parse_manager, std::int32_t from_seqno = 0, std::int32_t to_seqno = 0, bool force_index = false, 42 | std::uint32_t max_active_tasks = 32, QueueState max_queue = QueueState{30000, 30000, 500000, 500000}, std::int32_t stats_timeout = 10, 43 | std::shared_ptr watcher = nullptr) 44 | : db_scanner_(db_scanner), insert_manager_(insert_manager), parse_manager_(parse_manager), 45 | from_seqno_(from_seqno), to_seqno_(to_seqno), force_index_(force_index), max_active_tasks_(max_active_tasks), 46 | max_queue_(std::move(max_queue)), stats_timeout_(stats_timeout), watcher_(watcher) {}; 47 | 48 | void start_up() override; 49 | void alarm() override; 50 | void run(); 51 | private: 52 | void schedule_next_seqnos(); 53 | 54 | void schedule_seqno(std::uint32_t mc_seqno); 55 | void reschedule_seqno(std::uint32_t mc_seqno); 56 | void seqno_fetched(std::uint32_t mc_seqno, MasterchainBlockDataState block_data_state); 57 | void seqno_parsed(std::uint32_t mc_seqno, ParsedBlockPtr parsed_block); 58 | void seqno_interfaces_processed(std::uint32_t mc_seqno, ParsedBlockPtr parsed_block); 59 | void seqno_queued_to_insert(std::uint32_t mc_seqno, QueueState status); 60 | void seqno_inserted(std::uint32_t mc_seqno, td::Unit result); 61 | 62 | void got_existing_seqnos(td::Result> R); 63 | void got_last_known_seqno(std::uint32_t last_known_seqno); 64 | 65 | void got_insert_queue_state(QueueState status); 66 | 67 | void print_stats(); 68 | }; 69 | -------------------------------------------------------------------------------- /ton-index-clickhouse/src/InsertManagerClickhouse.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include "InsertManagerBase.h" 6 | 7 | 8 | class InsertBatchClickhouse; 9 | 10 | class InsertManagerClickhouse: public InsertManagerBase { 11 | public: 12 | struct Credential { 13 | std::string host = "127.0.0.1"; 14 | int port = 9000; 15 | std::string user = "default"; 16 | std::string password = ""; 17 | std::string dbname = "default"; 18 | 19 | clickhouse::ClientOptions get_clickhouse_options(); 20 | }; 21 | struct Options {}; 22 | private: 23 | InsertManagerClickhouse::Credential credential_; 24 | public: 25 | InsertManagerClickhouse(Credential credential) : credential_(credential) {} 26 | 27 | void start_up() override; 28 | 29 | void create_insert_actor(std::vector insert_tasks, td::Promise promise) override; 30 | void get_existing_seqnos(td::Promise> promise, std::int32_t from_seqno = 0, std::int32_t to_seqno = 0) override; 31 | }; 32 | 33 | 34 | class InsertBatchClickhouse: public td::actor::Actor { 35 | public: 36 | InsertBatchClickhouse(clickhouse::ClientOptions client_options, std::vector insert_tasks, td::Promise promise) 37 | : client_options_(std::move(client_options)), insert_tasks_(std::move(insert_tasks)), promise_(std::move(promise)) {} 38 | 39 | void start_up() override; 40 | private: 41 | InsertManagerClickhouse::Credential credential_; 42 | clickhouse::ClientOptions client_options_; 43 | std::vector insert_tasks_; 44 | td::Promise promise_; 45 | 46 | struct MsgBody { 47 | td::Bits256 hash; 48 | std::string body; 49 | }; 50 | 51 | void insert_jettons(clickhouse::Client& client); 52 | void insert_nfts(clickhouse::Client& client); 53 | void insert_transactions(clickhouse::Client& client); 54 | void insert_messages(clickhouse::Client& client); 55 | void insert_account_states(clickhouse::Client& client); 56 | void insert_shard_state(clickhouse::Client& client); 57 | void insert_blocks(clickhouse::Client& client); 58 | }; 59 | -------------------------------------------------------------------------------- /ton-index-postgres-v2/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.16) 2 | 3 | add_executable(ton-index-postgres-v2 4 | src/main.cpp 5 | src/InsertManagerPostgres.cpp 6 | src/IndexScheduler.cpp 7 | ) 8 | 9 | target_compile_features(ton-index-postgres-v2 PRIVATE cxx_std_20) 10 | target_link_libraries(ton-index-postgres-v2 tondb-scanner pqxx) 11 | target_link_options(ton-index-postgres-v2 PUBLIC -rdynamic) 12 | 13 | install(TARGETS ton-index-postgres-v2 RUNTIME DESTINATION bin) 14 | -------------------------------------------------------------------------------- /ton-index-postgres-v2/src/IndexScheduler.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "td/actor/actor.h" 4 | 5 | #include "IndexData.h" 6 | #include "DbScanner.h" 7 | #include "EventProcessor.h" 8 | #include "TraceAssembler.h" 9 | #include "InsertManager.h" 10 | #include "DataParser.h" 11 | #include "smc-interfaces/InterfacesDetector.h" 12 | 13 | using Detector = InterfacesDetector; 16 | 17 | class IndexScheduler: public td::actor::Actor { 18 | private: 19 | std::queue queued_seqnos_; 20 | std::set processing_seqnos_; 21 | std::set existing_seqnos_; 22 | 23 | td::actor::ActorId db_scanner_; 24 | td::actor::ActorId insert_manager_; 25 | td::actor::ActorId parse_manager_; 26 | td::actor::ActorOwn trace_assembler_; 27 | std::shared_ptr watcher_; 28 | 29 | std::string working_dir_; 30 | 31 | std::uint32_t max_active_tasks_{32}; 32 | std::int32_t last_known_seqno_{0}; 33 | std::int32_t last_indexed_seqno_{0}; 34 | std::int32_t from_seqno_{0}; 35 | std::int32_t to_seqno_{0}; 36 | bool force_index_{false}; 37 | 38 | std::double_t avg_tps_{0}; 39 | std::int64_t last_existing_seqno_count_{0}; 40 | 41 | QueueState max_queue_{30000, 30000, 500000, 500000}; 42 | QueueState cur_queue_state_; 43 | 44 | std::int32_t stats_timeout_{10}; 45 | td::Timestamp next_print_stats_; 46 | td::Timestamp next_statistics_flush_; 47 | 48 | std::unordered_map timers_; 49 | public: 50 | IndexScheduler(td::actor::ActorId db_scanner, td::actor::ActorId insert_manager, 51 | td::actor::ActorId parse_manager, std::string working_dir, std::int32_t from_seqno = 0, std::int32_t to_seqno = 0, bool force_index = false, 52 | std::uint32_t max_active_tasks = 32, QueueState max_queue = QueueState{30000, 30000, 500000, 500000}, std::int32_t stats_timeout = 10, 53 | std::shared_ptr watcher = nullptr) 54 | : db_scanner_(db_scanner), insert_manager_(insert_manager), parse_manager_(parse_manager), working_dir_(std::move(working_dir)), 55 | from_seqno_(from_seqno), to_seqno_(to_seqno), force_index_(force_index), max_active_tasks_(max_active_tasks), 56 | max_queue_(std::move(max_queue)), stats_timeout_(stats_timeout), watcher_(watcher) {}; 57 | 58 | void start_up() override; 59 | void alarm() override; 60 | void run(); 61 | private: 62 | void schedule_next_seqnos(); 63 | 64 | void schedule_seqno(std::uint32_t mc_seqno); 65 | void reschedule_seqno(std::uint32_t mc_seqno); 66 | void seqno_fetched(std::uint32_t mc_seqno, MasterchainBlockDataState block_data_state); 67 | void seqno_parsed(std::uint32_t mc_seqno, ParsedBlockPtr parsed_block); 68 | void seqno_traces_assembled(std::uint32_t mc_seqno, ParsedBlockPtr parsed_block); 69 | void seqno_interfaces_processed(std::uint32_t mc_seqno, ParsedBlockPtr parsed_block); 70 | void seqno_actions_processed(std::uint32_t mc_seqno, ParsedBlockPtr parsed_block); 71 | void seqno_queued_to_insert(std::uint32_t mc_seqno, QueueState status); 72 | void seqno_inserted(std::uint32_t mc_seqno, td::Unit result); 73 | 74 | void got_existing_seqnos(td::Result> R); 75 | void got_trace_assembler_last_state_seqno(ton::BlockSeqno last_state_seqno); 76 | void got_last_known_seqno(std::uint32_t last_known_seqno); 77 | 78 | void got_insert_queue_state(QueueState status); 79 | 80 | void print_stats(); 81 | }; 82 | -------------------------------------------------------------------------------- /ton-index-postgres-v2/src/InsertManagerPostgres.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include "InsertManagerBase.h" 5 | 6 | 7 | class InsertBatchPostgres; 8 | 9 | class InsertManagerPostgres: public InsertManagerBase { 10 | public: 11 | struct Credential { 12 | std::string host = "127.0.0.1"; 13 | int port = 5432; 14 | std::string user; 15 | std::string password; 16 | std::string dbname = "ton_index"; 17 | 18 | std::string get_connection_string(std::string dbname = "") const; 19 | }; 20 | private: 21 | InsertManagerPostgres::Credential credential_; 22 | bool custom_types_{false}; 23 | bool create_indexes_{false}; 24 | bool run_migrations_{false}; 25 | std::int32_t max_data_depth_{0}; 26 | std::int32_t out_of_sync_seqno_{0}; 27 | public: 28 | InsertManagerPostgres(InsertManagerPostgres::Credential credential, bool custom_types, bool create_indexes, bool run_migrations) : 29 | credential_(credential), custom_types_(custom_types), create_indexes_(create_indexes), run_migrations_(run_migrations) {} 30 | 31 | void start_up() override; 32 | 33 | void set_max_data_depth(std::int32_t value); 34 | 35 | void create_insert_actor(std::vector insert_tasks, td::Promise promise) override; 36 | void get_existing_seqnos(td::Promise> promise, std::int32_t from_seqno = 0, std::int32_t to_seqno = 0) override; 37 | }; 38 | 39 | 40 | class InsertBatchPostgres: public td::actor::Actor { 41 | public: 42 | InsertBatchPostgres(InsertManagerPostgres::Credential credential, std::vector insert_tasks, td::Promise promise, std::int32_t max_data_depth = 12) : 43 | credential_(std::move(credential)), insert_tasks_(std::move(insert_tasks)), promise_(std::move(promise)), max_data_depth_(max_data_depth) { 44 | // sorting in descending seqno order for easier processing of interfaces 45 | std::sort(insert_tasks_.begin(), insert_tasks_.end(), [](const auto& a, const auto& b) { 46 | return a.mc_seqno_ > b.mc_seqno_; 47 | }); 48 | } 49 | 50 | void start_up() override; 51 | void alarm() override; 52 | private: 53 | InsertManagerPostgres::Credential credential_; 54 | std::string connection_string_; 55 | std::vector insert_tasks_; 56 | td::Promise promise_; 57 | std::int32_t max_data_depth_; 58 | bool with_copy_{true}; 59 | 60 | std::string stringify(schema::ComputeSkipReason compute_skip_reason); 61 | std::string stringify(schema::AccStatusChange acc_status_change); 62 | std::string stringify(schema::AccountStatus account_status); 63 | std::string stringify(schema::Trace::State state); 64 | 65 | void insert_blocks(pqxx::work &txn, bool with_copy); 66 | void insert_shard_state(pqxx::work &txn, bool with_copy); 67 | void insert_transactions(pqxx::work &txn, bool with_copy); 68 | void insert_messages(pqxx::work &txn, bool with_copy); 69 | void insert_account_states(pqxx::work &txn, bool with_copy); 70 | std::string insert_latest_account_states(pqxx::work &txn); 71 | void insert_jetton_transfers(pqxx::work &txn, bool with_copy); 72 | void insert_jetton_burns(pqxx::work &txn, bool with_copy); 73 | void insert_nft_transfers(pqxx::work &txn, bool with_copy); 74 | std::string insert_jetton_masters(pqxx::work &txn); 75 | std::string insert_jetton_wallets(pqxx::work &txn); 76 | std::string insert_nft_collections(pqxx::work &txn); 77 | std::string insert_nft_items(pqxx::work &txn); 78 | std::string insert_getgems_nft_auctions(pqxx::work &txn); 79 | std::string insert_getgems_nft_sales(pqxx::work &txn); 80 | void insert_traces(pqxx::work &txn, bool with_copy); 81 | }; 82 | -------------------------------------------------------------------------------- /ton-index-postgres/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.16) 2 | 3 | add_executable(ton-index-postgres 4 | src/main.cpp 5 | src/InsertManagerPostgres.cpp 6 | src/IndexScheduler.cpp 7 | ) 8 | 9 | target_compile_features(ton-index-postgres PRIVATE cxx_std_20) 10 | target_link_libraries(ton-index-postgres tondb-scanner pqxx) 11 | target_link_options(ton-index-postgres PUBLIC -rdynamic) 12 | 13 | install(TARGETS ton-index-postgres RUNTIME DESTINATION bin) 14 | -------------------------------------------------------------------------------- /ton-index-postgres/src/IndexScheduler.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "td/actor/actor.h" 4 | 5 | #include "IndexData.h" 6 | #include "DbScanner.h" 7 | #include "EventProcessor.h" 8 | #include "InsertManager.h" 9 | #include "DataParser.h" 10 | 11 | 12 | class IndexScheduler: public td::actor::Actor { 13 | private: 14 | td::actor::ActorId db_scanner_; 15 | td::actor::ActorId insert_manager_; 16 | td::actor::ActorId parse_manager_; 17 | td::actor::ActorOwn event_processor_; 18 | 19 | const uint32_t max_active_tasks_{32}; 20 | const uint32_t from_seqno_{0}; 21 | 22 | uint32_t last_known_seqno_{0}; 23 | uint32_t last_indexed_seqno_{0}; 24 | 25 | std::queue queued_seqnos_; 26 | std::set processing_seqnos_; 27 | std::set existing_seqnos_; 28 | 29 | double avg_tps_{0}; 30 | uint32_t last_existing_seqno_count_{0}; 31 | 32 | const QueueState max_queue_{30000, 30000, 500000, 500000}; 33 | QueueState cur_queue_state_; 34 | 35 | const uint32_t stats_timeout_{10}; 36 | td::Timestamp next_print_stats_; 37 | public: 38 | IndexScheduler(td::actor::ActorId db_scanner, td::actor::ActorId insert_manager, 39 | td::actor::ActorId parse_manager, uint32_t from_seqno = 0, 40 | uint32_t max_active_tasks = 32, QueueState max_queue = QueueState{30000, 30000, 500000, 500000}, int32_t stats_timeout = 10) 41 | : db_scanner_(db_scanner), insert_manager_(insert_manager), parse_manager_(parse_manager), 42 | from_seqno_(from_seqno), last_known_seqno_(from_seqno), max_active_tasks_(max_active_tasks), 43 | max_queue_(std::move(max_queue)), stats_timeout_(stats_timeout) { 44 | event_processor_ = td::actor::create_actor("event_processor", insert_manager_); 45 | }; 46 | 47 | void alarm() override; 48 | void run(); 49 | private: 50 | void schedule_next_seqnos(); 51 | 52 | void schedule_seqno(uint32_t mc_seqno); 53 | void reschedule_seqno(uint32_t mc_seqno); 54 | void seqno_fetched(uint32_t mc_seqno, MasterchainBlockDataState block_data_state); 55 | void seqno_parsed(uint32_t mc_seqno, ParsedBlockPtr parsed_block); 56 | void seqno_interfaces_processed(uint32_t mc_seqno, ParsedBlockPtr parsed_block); 57 | void seqno_queued_to_insert(uint32_t mc_seqno, QueueState status); 58 | void seqno_inserted(uint32_t mc_seqno, td::Unit result); 59 | 60 | void got_existing_seqnos(td::Result> R); 61 | void got_last_known_seqno(uint32_t last_known_seqno); 62 | 63 | void got_insert_queue_state(QueueState status); 64 | 65 | void print_stats(); 66 | }; 67 | -------------------------------------------------------------------------------- /ton-index-postgres/src/InsertManagerPostgres.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include "InsertManagerBase.h" 5 | 6 | 7 | class InsertBatchPostgres; 8 | 9 | class InsertManagerPostgres: public InsertManagerBase { 10 | public: 11 | struct Credential { 12 | std::string host = "127.0.0.1"; 13 | int port = 5432; 14 | std::string user; 15 | std::string password; 16 | std::string dbname = "ton_index"; 17 | 18 | std::string get_connection_string(); 19 | }; 20 | private: 21 | InsertManagerPostgres::Credential credential_; 22 | public: 23 | InsertManagerPostgres(InsertManagerPostgres::Credential credential) : credential_(credential) {} 24 | 25 | void create_insert_actor(std::vector insert_tasks, td::Promise promise) override; 26 | void get_existing_seqnos(td::Promise> promise, std::int32_t from_seqno = 0, std::int32_t to_seqno = 0) override; 27 | }; 28 | 29 | 30 | class InsertBatchPostgres: public td::actor::Actor { 31 | public: 32 | InsertBatchPostgres(InsertManagerPostgres::Credential credential, std::vector insert_tasks, td::Promise promise) : 33 | credential_(std::move(credential)), insert_tasks_(std::move(insert_tasks)), promise_(std::move(promise)) {} 34 | 35 | void start_up() override; 36 | private: 37 | InsertManagerPostgres::Credential credential_; 38 | std::string connection_string_; 39 | std::vector insert_tasks_; 40 | td::Promise promise_; 41 | 42 | struct TxMsg { 43 | std::string tx_hash; 44 | std::string msg_hash; 45 | std::string direction; // in or out 46 | }; 47 | 48 | struct MsgBody { 49 | td::Bits256 hash; 50 | std::string body; 51 | }; 52 | 53 | std::string stringify(schema::ComputeSkipReason compute_skip_reason); 54 | std::string stringify(schema::AccStatusChange acc_status_change); 55 | std::string stringify(schema::AccountStatus account_status); 56 | std::string jsonify(const schema::SplitMergeInfo& info); 57 | std::string jsonify(const schema::StorageUsedShort& s); 58 | std::string jsonify(const schema::TrStoragePhase& s); 59 | std::string jsonify(const schema::TrCreditPhase& c); 60 | std::string jsonify(const schema::TrActionPhase& action); 61 | std::string jsonify(const schema::TrBouncePhase& bounce); 62 | std::string jsonify(const schema::TrComputePhase& compute); 63 | std::string jsonify(schema::TransactionDescr descr); 64 | std::string jsonify(const schema::BlockReference& block_ref); 65 | std::string jsonify(const std::vector& prev_blocks); 66 | 67 | void insert_blocks(pqxx::work &transaction, const std::vector& insert_tasks_); 68 | void insert_shard_state(pqxx::work &transaction, const std::vector& insert_tasks_); 69 | void insert_transactions(pqxx::work &transaction, const std::vector& insert_tasks_); 70 | void insert_messsages(pqxx::work &transaction, const std::vector &messages, const std::vector& msg_bodies, const std::vector &tx_msgs); 71 | void insert_messages_contents(const std::vector& msg_bodies, pqxx::work& transaction); 72 | void insert_messages_impl(const std::vector& messages, pqxx::work& transaction); 73 | void insert_messages_txs(const std::vector& messages, pqxx::work& transaction); 74 | void insert_account_states(pqxx::work &transaction, const std::vector& insert_tasks_); 75 | void insert_latest_account_states(pqxx::work &transaction, const std::vector& insert_tasks_); 76 | void insert_jetton_transfers(pqxx::work &transaction, const std::vector& insert_tasks_); 77 | void insert_jetton_burns(pqxx::work &transaction, const std::vector& insert_tasks_); 78 | void insert_nft_transfers(pqxx::work &transaction, const std::vector& insert_tasks_); 79 | void insert_jetton_masters(pqxx::work &transaction, const std::vector& insert_tasks); 80 | void insert_jetton_wallets(pqxx::work &transaction, const std::vector& insert_tasks); 81 | void insert_nft_collections(pqxx::work &transaction, const std::vector& insert_tasks); 82 | void insert_nft_items(pqxx::work &transaction, const std::vector& insert_tasks); 83 | }; 84 | -------------------------------------------------------------------------------- /ton-integrity-checker/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.16) 2 | 3 | add_executable(ton-integrity-checker 4 | src/main.cpp 5 | src/IntegrityChecker.cpp 6 | ) 7 | 8 | target_compile_features(ton-integrity-checker PRIVATE cxx_std_20) 9 | target_link_libraries(ton-integrity-checker tondb-scanner) 10 | 11 | install(TARGETS ton-integrity-checker RUNTIME DESTINATION bin) 12 | -------------------------------------------------------------------------------- /ton-integrity-checker/src/IntegrityChecker.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "td/actor/actor.h" 4 | #include "DbScanner.h" 5 | 6 | 7 | // Stub parser that does nothing for now. 8 | class IntegrityParser: public td::actor::Actor { 9 | public: 10 | IntegrityParser() {} 11 | 12 | void parse(int mc_seqno, MasterchainBlockDataState mc_block, td::Promise promise) { 13 | promise.set_result(td::Unit()); 14 | } 15 | }; 16 | 17 | 18 | class IntegrityChecker : public td::actor::Actor { 19 | private: 20 | td::actor::ActorId db_scanner_; 21 | td::actor::ActorId parse_manager_; 22 | std::string checkpoint_path_; 23 | size_t fetch_parallelism_; 24 | size_t parse_parallelism_; 25 | std::uint32_t stats_timeout_; 26 | size_t min_free_memory_; 27 | std::shared_ptr watcher_; 28 | 29 | std::uint32_t from_seqno_{0}; 30 | std::uint32_t to_seqno_{0}; 31 | std::queue queued_seqnos_; 32 | td::Timestamp next_print_stats_; 33 | std::unordered_set seqnos_fetching_; 34 | std::unordered_set seqnos_parsing_; 35 | std::set seqnos_processed_; 36 | std::queue> blocks_to_parse_; 37 | std::uint32_t blocks_to_parse_queue_max_size_{1000}; 38 | 39 | td::Timestamp last_tps_calc_ts_ = td::Timestamp::now(); 40 | uint32_t last_tps_calc_processed_count_{0}; 41 | float tps_{0}; 42 | 43 | std::uint32_t checkpoint_seqno_{0}; 44 | 45 | public: 46 | IntegrityChecker(td::actor::ActorId db_scanner, td::actor::ActorId parse_manager, std::string checkpoint_path, 47 | std::size_t fetch_parallelism = 1, std::size_t parse_parallelism = 1, std::uint32_t stats_timeout = 60, size_t min_free_memory = 3, std::shared_ptr watcher = nullptr) : 48 | db_scanner_(db_scanner), parse_manager_(parse_manager), checkpoint_path_(checkpoint_path), fetch_parallelism_(fetch_parallelism), 49 | parse_parallelism_(parse_parallelism), stats_timeout_(stats_timeout), min_free_memory_(min_free_memory), watcher_(watcher) {}; 50 | virtual ~IntegrityChecker() = default; 51 | 52 | virtual void start_up() override; 53 | 54 | void got_last_mc_seqno(std::uint32_t last_known_seqno); 55 | void got_block_handle(ton::validator::ConstBlockHandle handle); 56 | void got_min_mc_seqno(ton::BlockSeqno min_known_seqno); 57 | void got_oldest_mc_seqno_with_state(ton::BlockSeqno oldest_seqno_with_state); 58 | void find_oldest_seqno_with_state(uint32_t min_seqno, uint32_t max_seqno); 59 | void fetch_next_seqnos(); 60 | void fetch_error(std::uint32_t seqno, td::Status error); 61 | void seqno_fetched(std::uint32_t seqno, MasterchainBlockDataState state); 62 | void parse_next_seqnos(); 63 | void parse_error(std::uint32_t seqno, td::Status error); 64 | void seqno_parsed(std::uint32_t seqno); 65 | 66 | void print_stats(); 67 | void fail_if_low_memory(); 68 | 69 | void alarm(); 70 | }; 71 | -------------------------------------------------------------------------------- /ton-integrity-checker/src/main.cpp: -------------------------------------------------------------------------------- 1 | #include "td/utils/port/signals.h" 2 | #include "td/utils/OptionParser.h" 3 | #include "td/utils/format.h" 4 | #include "td/utils/logging.h" 5 | #include "td/utils/check.h" 6 | 7 | #include "crypto/vm/cp0.h" 8 | 9 | // #include "InsertManagerPostgres.h" 10 | #include "DataParser.h" 11 | #include "DbScanner.h" 12 | #include "EventProcessor.h" 13 | #include "IntegrityChecker.h" 14 | 15 | 16 | int main(int argc, char *argv[]) { 17 | SET_VERBOSITY_LEVEL(verbosity_INFO); 18 | td::set_default_failure_signal_handler().ensure(); 19 | 20 | CHECK(vm::init_op_cp0()); 21 | 22 | // options 23 | td::uint32 threads = 7; 24 | td::int32 stats_timeout = 10; 25 | std::string db_root; 26 | td::uint32 last_known_seqno = 0; 27 | std::string checkpoint_path; 28 | 29 | std::uint32_t max_active_tasks = 7; 30 | size_t min_free_memory = 10; 31 | 32 | 33 | td::OptionParser p; 34 | p.set_description("Parse TON DB and insert data into Postgres"); 35 | p.add_option('\0', "help", "prints_help", [&]() { 36 | char b[10240]; 37 | td::StringBuilder sb(td::MutableSlice{b, 10000}); 38 | sb << p; 39 | std::cout << sb.as_cslice().c_str(); 40 | std::exit(2); 41 | }); 42 | p.add_option('D', "db", "Path to TON DB folder", [&](td::Slice fname) { 43 | db_root = fname.str(); 44 | }); 45 | 46 | p.add_checked_option('f', "from", "Masterchain seqno to start indexing from", [&](td::Slice fname) { 47 | int v; 48 | try { 49 | v = std::stoi(fname.str()); 50 | } catch (...) { 51 | return td::Status::Error(ton::ErrorCode::error, "bad value for --from: not a number"); 52 | } 53 | last_known_seqno = v; 54 | return td::Status::OK(); 55 | }); 56 | p.add_checked_option('\0', "max-active-tasks", "Max active reading tasks", [&](td::Slice fname) { 57 | int v; 58 | try { 59 | v = std::stoi(fname.str()); 60 | } catch (...) { 61 | return td::Status::Error(ton::ErrorCode::error, "bad value for --max-active-tasks: not a number"); 62 | } 63 | max_active_tasks = v; 64 | return td::Status::OK(); 65 | }); 66 | 67 | p.add_checked_option('\0', "min-free-memory", "Minimum percentage of free RAM left (integer). When less RAM is left the program will exit with code 100.", [&](td::Slice fname) { 68 | int v; 69 | try { 70 | v = std::stoi(fname.str()); 71 | } catch (...) { 72 | return td::Status::Error(ton::ErrorCode::error, "bad value for --max-active-tasks: not a number"); 73 | } 74 | min_free_memory = v; 75 | return td::Status::OK(); 76 | }); 77 | 78 | p.add_checked_option('\0', "stats-freq", "Pause between printing stats in seconds", [&](td::Slice fname) { 79 | int v; 80 | try { 81 | v = std::stoi(fname.str()); 82 | } catch (...) { 83 | return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: not a number"); 84 | } 85 | stats_timeout = v; 86 | return td::Status::OK(); 87 | }); 88 | 89 | p.add_option('\0', "checkpoint", "Path to checkpoint file, used for saving progress.", [&](td::Slice arg) { 90 | checkpoint_path = arg.str(); 91 | }); 92 | 93 | p.add_checked_option('t', "threads", "Scheduler threads (default: 7)", [&](td::Slice fname) { 94 | int v; 95 | try { 96 | v = std::stoi(fname.str()); 97 | } catch (...) { 98 | return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: not a number"); 99 | } 100 | threads = v; 101 | return td::Status::OK(); 102 | }); 103 | 104 | auto S = p.run(argc, argv); 105 | if (S.is_error()) { 106 | LOG(ERROR) << "failed to parse options: " << S.move_as_error(); 107 | std::_Exit(2); 108 | } 109 | 110 | td::actor::Scheduler scheduler({threads}); 111 | 112 | td::actor::ActorOwn parse_manager; 113 | td::actor::ActorOwn db_scanner; 114 | 115 | auto watcher = td::create_shared_destructor([] { 116 | td::actor::SchedulerContext::get()->stop(); 117 | }); 118 | 119 | scheduler.run_in_context([&, watcher = std::move(watcher)] { 120 | parse_manager = td::actor::create_actor("parsemanager"); 121 | db_scanner = td::actor::create_actor("scanner", db_root, dbs_readonly); 122 | td::actor::create_actor("integritychecker", 123 | db_scanner.get(), parse_manager.get(), checkpoint_path, max_active_tasks, max_active_tasks, stats_timeout, min_free_memory, watcher).release(); 124 | }); 125 | 126 | scheduler.run(); 127 | 128 | LOG(INFO) << "TON DB integrity check finished successfully"; 129 | 130 | return 0; 131 | } 132 | -------------------------------------------------------------------------------- /ton-smc-scanner/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.16) 2 | 3 | add_executable(ton-smc-scanner 4 | src/main.cpp 5 | src/SmcScanner.cpp 6 | src/PostgreSQLInserter.cpp 7 | ) 8 | target_include_directories(ton-smc-scanner 9 | PUBLIC src 10 | ) 11 | target_compile_features(ton-smc-scanner PRIVATE cxx_std_20) 12 | target_link_libraries(ton-smc-scanner tondb-scanner pqxx) 13 | 14 | install(TARGETS ton-smc-scanner RUNTIME DESTINATION bin) 15 | -------------------------------------------------------------------------------- /ton-smc-scanner/src/PostgreSQLInserter.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | // #include 8 | #include "IndexData.h" 9 | 10 | 11 | 12 | using InsertData = std::variant; 13 | 14 | class PostgreSQLInserter : public td::actor::Actor { 15 | public: 16 | PostgreSQLInserter(std::string connection_string, std::vector data, td::Promise promise) 17 | : connection_string_(connection_string), data_(std::move(data)), promise_(std::move(promise)) {} 18 | 19 | void start_up() override; 20 | private: 21 | void insert_latest_account_states(pqxx::work &transaction); 22 | void insert_jetton_masters(pqxx::work &transaction); 23 | void insert_jetton_wallets(pqxx::work &transaction); 24 | void insert_nft_items(pqxx::work &transaction); 25 | void insert_nft_collections(pqxx::work &transaction); 26 | 27 | std::string connection_string_; 28 | std::vector data_; 29 | td::Promise promise_; 30 | }; 31 | 32 | class PostgreSQLInsertManager : public td::actor::Actor { 33 | public: 34 | PostgreSQLInsertManager(std::string connection_string, std::int32_t batch_size) 35 | : connection_string_(connection_string), batch_size_(batch_size) {} 36 | void start_up() override; 37 | void alarm() override; 38 | void insert_data(std::vector data); 39 | void insert_done(size_t cnt); 40 | void checkpoint(ton::ShardIdFull shard, td::Bits256 cur_addr_); 41 | void checkpoint_read(ton::ShardIdFull shard, td::Promise promise); 42 | void checkpoint_reset(ton::ShardIdFull shard); 43 | private: 44 | void check_queue(bool force = false); 45 | 46 | std::string connection_string_; 47 | std::int32_t batch_size_; 48 | std::vector queue_; 49 | 50 | std::int32_t inserted_count_{0}; 51 | std::int32_t in_progress_{0}; 52 | }; 53 | -------------------------------------------------------------------------------- /ton-smc-scanner/src/SmcScanner.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "td/actor/actor.h" 3 | #include "DbScanner.h" 4 | #include "smc-interfaces/InterfacesDetector.h" 5 | #include 6 | 7 | 8 | using Detector = InterfacesDetector; 10 | 11 | struct ShardStateData { 12 | AllShardStates shard_states_; 13 | block::gen::ShardStateUnsplit::Record sstate_; 14 | std::shared_ptr config_; 15 | }; 16 | 17 | using ShardStateDataPtr = std::shared_ptr; 18 | 19 | struct Options { 20 | std::uint32_t seqno_; 21 | td::actor::ActorId insert_manager_; 22 | std::int32_t batch_size_{5000}; 23 | bool index_interfaces_{false}; 24 | bool from_checkpoint{true}; 25 | }; 26 | 27 | class ShardStateScanner; 28 | 29 | class StateBatchParser: public td::actor::Actor { 30 | private: 31 | std::vector> data_; 32 | ShardStateDataPtr shard_state_data_; 33 | td::actor::ActorId shard_state_scanner_; 34 | Options options_; 35 | 36 | std::unordered_map> interfaces_; 37 | std::vector result_; 38 | public: 39 | StateBatchParser(std::vector> data, ShardStateDataPtr shard_state_data, td::actor::ActorId shard_state_scanner, Options options) 40 | : data_(std::move(data)), shard_state_data_(std::move(shard_state_data)), shard_state_scanner_(shard_state_scanner), options_(options) {} 41 | void start_up() override; 42 | void processing_finished(); 43 | private: 44 | void interfaces_detected(block::StdAddress address, std::vector interfaces, 45 | td::Bits256 code_hash, td::Bits256 data_hash, uint64_t last_trans_lt, uint32_t last_trans_now, td::Promise promise); 46 | void process_account_states(std::vector account_states); 47 | }; 48 | 49 | class ShardStateScanner: public td::actor::Actor { 50 | private: 51 | td::Ref shard_state_; 52 | MasterchainBlockDataState mc_block_ds_; 53 | 54 | ShardStateDataPtr shard_state_data_; 55 | Options options_; 56 | 57 | td::Bits256 cur_addr_{td::Bits256::zero()}; 58 | 59 | ton::ShardIdFull shard_; 60 | bool allow_same_{true}; 61 | bool finished_{false}; 62 | uint32_t in_progress_{0}; 63 | uint32_t processed_{0}; 64 | 65 | std::unordered_map no_interface_count_; 66 | std::unordered_set code_hashes_to_skip_; 67 | std::mutex code_hashes_to_skip_mutex_; 68 | public: 69 | ShardStateScanner(td::Ref shard_state, MasterchainBlockDataState mc_block_ds, Options options); 70 | 71 | void schedule_next(); 72 | void start_up() override; 73 | void alarm() override; 74 | void batch_inserted(); 75 | 76 | void got_checkpoint(td::Bits256 cur_addr); 77 | }; 78 | 79 | class SmcScanner: public td::actor::Actor { 80 | private: 81 | td::actor::ActorId db_scanner_; 82 | Options options_; 83 | public: 84 | SmcScanner(td::actor::ActorId db_scanner, Options options) : 85 | db_scanner_(db_scanner), options_(options) {}; 86 | 87 | void start_up() override; 88 | void got_block(MasterchainBlockDataState block); 89 | }; 90 | -------------------------------------------------------------------------------- /ton-smc-scanner/src/main.cpp: -------------------------------------------------------------------------------- 1 | #include "td/utils/port/signals.h" 2 | #include "td/utils/OptionParser.h" 3 | #include "td/utils/format.h" 4 | #include "td/utils/logging.h" 5 | #include "td/utils/check.h" 6 | #include "td/actor/actor.h" 7 | 8 | #include "crypto/vm/cp0.h" 9 | #include "DbScanner.h" 10 | #include "SmcScanner.h" 11 | 12 | 13 | int main(int argc, char *argv[]) { 14 | SET_VERBOSITY_LEVEL(verbosity_INFO); 15 | td::set_default_failure_signal_handler().ensure(); 16 | 17 | CHECK(vm::init_op_cp0()); 18 | 19 | td::uint32 threads = 7; 20 | std::string db_root = "/var/lib/ton-work/db"; 21 | std::string pg_dsn = "postgresql://localhost:5432/ton_index"; 22 | Options options_; 23 | bool is_testnet = false; 24 | 25 | td::OptionParser p; 26 | p.set_description("Scan all accounts at some seqno, detect interfaces and save them to postgres"); 27 | p.add_option('\0', "help", "prints_help", [&]() { 28 | char b[10240]; 29 | td::StringBuilder sb(td::MutableSlice{b, 10000}); 30 | sb << p; 31 | std::cout << sb.as_cslice().c_str(); 32 | std::exit(2); 33 | }); 34 | p.add_checked_option('S', "seqno", "Masterchain seqno to start indexing from", [&](td::Slice fname) { 35 | int v; 36 | try { 37 | v = std::stoi(fname.str()); 38 | } catch (...) { 39 | return td::Status::Error("bad value for --seqno: not a number"); 40 | } 41 | options_.seqno_ = v; 42 | return td::Status::OK(); 43 | }); 44 | p.add_checked_option('t', "threads", "Scheduler threads (default: 7)", [&](td::Slice fname) { 45 | int v; 46 | try { 47 | v = std::stoi(fname.str()); 48 | } catch (...) { 49 | return td::Status::Error("bad value for --threads: not a number"); 50 | } 51 | threads = v; 52 | return td::Status::OK(); 53 | }); 54 | p.add_checked_option('b', "batch-size", "Insert batch size", [&](td::Slice fname) { 55 | int v; 56 | try { 57 | v = std::stoi(fname.str()); 58 | } catch (...) { 59 | return td::Status::Error("bad value for --batch-size: not a number"); 60 | } 61 | options_.batch_size_ = v; 62 | return td::Status::OK(); 63 | }); 64 | p.add_option('D', "db", "Path to TON DB folder", [&](td::Slice fname) { 65 | db_root = fname.str(); 66 | }); 67 | p.add_option('d', "pg", "PostgreSQL connection string", [&](td::Slice value) { 68 | pg_dsn = value.str(); 69 | }); 70 | p.add_option('i', "interfaces", "Detect interfaces", [&] { 71 | options_.index_interfaces_ = true; 72 | }); 73 | p.add_option('f', "force", "Reset checkpoints", [&]() { 74 | options_.from_checkpoint = false; 75 | }); 76 | p.add_option('\0', "testnet", "Use for testnet. It is used for correct indexing of .ton DNS entries (in testnet .ton collection has a different address)", [&]() { 77 | is_testnet = true; 78 | }); 79 | 80 | auto S = p.run(argc, argv); 81 | if (S.is_error()) { 82 | LOG(ERROR) << "failed to parse options: " << S.move_as_error(); 83 | std::_Exit(2); 84 | } 85 | 86 | NftItemDetectorR::is_testnet = is_testnet; 87 | 88 | td::actor::Scheduler scheduler({threads}); 89 | td::actor::ActorOwn db_scanner; 90 | td::actor::ActorOwn insert_manager; 91 | 92 | // auto watcher = td::create_shared_destructor([] { 93 | // td::actor::SchedulerContext::get()->stop(); 94 | // }); 95 | scheduler.run_in_context([&] { 96 | db_scanner = td::actor::create_actor("scanner", db_root, dbs_readonly); 97 | insert_manager = td::actor::create_actor("insert_manager", pg_dsn, options_.batch_size_); 98 | options_.insert_manager_ = insert_manager.get(); 99 | td::actor::create_actor("smcscanner", db_scanner.get(), options_).release(); 100 | }); 101 | 102 | scheduler.run(); 103 | LOG(INFO) << "TON DB integrity check finished successfully"; 104 | return 0; 105 | } 106 | 107 | 108 | -------------------------------------------------------------------------------- /ton-trace-emulator/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.16) 2 | 3 | add_library(ton-trace-emulator-core STATIC 4 | src/TraceEmulator.cpp 5 | src/OverlayListener.cpp 6 | src/BlockEmulator.cpp 7 | src/TraceInterfaceDetector.cpp 8 | ) 9 | target_include_directories(ton-trace-emulator-core 10 | PUBLIC external/ton 11 | PUBLIC src/ 12 | PUBLIC tondb-scanner/src 13 | ) 14 | target_link_directories(ton-trace-emulator-core 15 | PUBLIC external/ton 16 | ) 17 | 18 | target_link_libraries(ton-trace-emulator-core PUBLIC tondb-scanner overlay emulator) 19 | target_compile_features(ton-trace-emulator-core PRIVATE cxx_std_20) 20 | 21 | add_executable(ton-trace-emulator 22 | src/main.cpp 23 | src/TraceScheduler.cpp 24 | src/TraceInserter.cpp 25 | src/RedisListener.cpp 26 | ) 27 | target_include_directories(ton-trace-emulator 28 | PUBLIC ${CMAKE_BINARY_DIR}/external/redis-plus-plus/src 29 | ) 30 | target_compile_features(ton-trace-emulator PRIVATE cxx_std_20) 31 | target_compile_definitions(ton-trace-emulator PRIVATE -DMSGPACK_USE_DEFINE_MAP) 32 | target_link_libraries(ton-trace-emulator ton-trace-emulator-core hiredis redis++ msgpack-cxx) 33 | target_link_options(ton-trace-emulator PUBLIC -rdynamic) 34 | install(TARGETS ton-trace-emulator RUNTIME DESTINATION bin) 35 | -------------------------------------------------------------------------------- /ton-trace-emulator/src/BlockEmulator.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "IndexData.h" 5 | #include "TraceEmulator.h" 6 | 7 | 8 | struct OutMsgInfo { 9 | td::Bits256 hash; 10 | td::Ref root; 11 | }; 12 | 13 | struct TransactionInfo { 14 | block::StdAddress account; 15 | td::Bits256 hash; 16 | ton::LogicalTime lt; 17 | td::Ref root; 18 | td::Bits256 in_msg_hash; 19 | bool is_first; 20 | std::vector out_msgs; 21 | std::optional initial_msg_hash{}; // hash of initial transaction in block that caused this transaction. 22 | // This is not necessarily ext in message, because ext in could happen in prev block. 23 | }; 24 | 25 | class McBlockEmulator: public td::actor::Actor { 26 | private: 27 | MasterchainBlockDataState mc_data_state_; 28 | std::function)> trace_processor_; 29 | td::Promise<> promise_; 30 | size_t blocks_left_to_parse_; 31 | std::vector txs_; 32 | 33 | std::vector> shard_states_; 34 | 35 | std::unordered_map tx_by_in_msg_hash_; 36 | 37 | std::unordered_set trace_ids_in_progress_; 38 | 39 | int traces_cnt_{0}; 40 | 41 | td::Timestamp start_time_; 42 | 43 | // static map for matching in-out msgs between blocks to propagate trace ids. TODO: clean up old entries. 44 | // TODO: care of thread safety 45 | inline static std::unordered_map interblock_trace_ids_; 46 | 47 | void parse_error(ton::BlockId blkid, td::Status error); 48 | void block_parsed(ton::BlockId blkid, std::vector txs); 49 | void process_txs(); 50 | void emulate_traces(); 51 | void trace_error(td::Bits256 tx_hash, TraceId trace_id, td::Status error); 52 | void trace_received(td::Bits256 tx_hash, Trace trace); 53 | void trace_interfaces_error(TraceId trace_id, td::Status error); 54 | void trace_emulated(Trace trace); 55 | void trace_finished(TraceId trace_id); 56 | 57 | td::Result fetch_account(const block::StdAddress& addr, ton::UnixTime now); 58 | 59 | public: 60 | McBlockEmulator(MasterchainBlockDataState mc_data_state, std::function)> trace_processor, td::Promise<> promise); 61 | 62 | virtual void start_up() override; 63 | }; -------------------------------------------------------------------------------- /ton-trace-emulator/src/OverlayListener.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "IndexData.h" 10 | #include "TraceEmulator.h" 11 | 12 | 13 | class OverlayListener : public td::actor::Actor { 14 | private: 15 | std::string global_config_path_; 16 | std::string inet_addr_; 17 | std::function)> trace_processor_; 18 | 19 | td::actor::ActorOwn overlays_; 20 | td::actor::ActorOwn adnl_; 21 | td::actor::ActorOwn keyring_; 22 | td::actor::ActorOwn dht_; 23 | td::actor::ActorOwn adnl_network_manager_; 24 | 25 | MasterchainBlockDataState mc_data_state_; 26 | std::unordered_set known_ext_msgs_; // this set grows infinitely. TODO: remove old messages 27 | 28 | int traces_cnt_{0}; 29 | 30 | void process_external_message(td::Ref message); 31 | void trace_error(TraceId trace_id, td::Status error); 32 | void trace_received(TraceId trace_id, Trace trace); 33 | void trace_interfaces_error(TraceId trace_id, td::Status error); 34 | void finish_processing(Trace trace); 35 | 36 | public: 37 | OverlayListener(std::string global_config_path, std::string inet_addr, std::function)> trace_processor) 38 | : global_config_path_(std::move(global_config_path)), inet_addr_(std::move(inet_addr)), trace_processor_(std::move(trace_processor)) {}; 39 | 40 | virtual void start_up() override; 41 | 42 | void set_mc_data_state(MasterchainBlockDataState mc_data_state) { 43 | mc_data_state_ = std::move(mc_data_state); 44 | } 45 | }; -------------------------------------------------------------------------------- /ton-trace-emulator/src/RedisListener.cpp: -------------------------------------------------------------------------------- 1 | #include "RedisListener.h" 2 | 3 | 4 | void RedisListener::start_up() { 5 | alarm_timestamp() = td::Timestamp::now() ; 6 | } 7 | 8 | void RedisListener::alarm() { 9 | if (mc_data_state_.config_ == nullptr) { 10 | alarm_timestamp() = td::Timestamp::in(0.1); 11 | return; 12 | } 13 | 14 | while (auto value = redis_.rpop(queue_name_)) { 15 | auto boc_decoded = td::base64_decode(td::Slice(value.value())); 16 | if (boc_decoded.is_error()) { 17 | LOG(ERROR) << "Can't decode base64 boc: " << boc_decoded.move_as_error(); 18 | continue; 19 | } 20 | auto msg_cell_r = vm::std_boc_deserialize(boc_decoded.move_as_ok()); 21 | if (msg_cell_r.is_error()) { 22 | LOG(ERROR) << "Can't deserialize message boc: " << msg_cell_r.move_as_error(); 23 | continue; 24 | } 25 | auto msg_cell = msg_cell_r.move_as_ok(); 26 | 27 | auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), msg_hash = td::Bits256(msg_cell->get_hash().bits())](td::Result R) mutable { 28 | if (R.is_error()) { 29 | td::actor::send_closure(SelfId, &RedisListener::trace_error, msg_hash, R.move_as_error()); 30 | } else { 31 | td::actor::send_closure(SelfId, &RedisListener::trace_received, msg_hash, R.move_as_ok()); 32 | } 33 | }); 34 | 35 | td::actor::create_actor("TraceEmu", mc_data_state_, msg_cell, false, std::move(P)).release(); 36 | } 37 | 38 | alarm_timestamp() = td::Timestamp::in(0.1); 39 | } 40 | 41 | void RedisListener::set_mc_data_state(MasterchainBlockDataState mc_data_state) { 42 | shard_states_.clear(); 43 | for (const auto& shard_state : mc_data_state.shard_blocks_) { 44 | shard_states_.push_back(shard_state.block_state); 45 | } 46 | 47 | mc_data_state_ = std::move(mc_data_state); 48 | } 49 | 50 | void RedisListener::trace_error(TraceId trace_id, td::Status error) { 51 | LOG(ERROR) << "Failed to emulate trace " << td::base64_encode(trace_id.as_slice()) << ": " << error; 52 | known_ext_msgs_.erase(trace_id); 53 | } 54 | 55 | void RedisListener::trace_received(TraceId trace_id, Trace trace) { 56 | LOG(INFO) << "Emulated trace " << td::base64_encode(trace_id.as_slice()) << ": " 57 | << trace.transactions_count() << " transactions, " << trace.depth() << " depth"; 58 | if constexpr (std::variant_size_v > 0) { 59 | auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), trace_id = trace.id](td::Result R) { 60 | if (R.is_error()) { 61 | td::actor::send_closure(SelfId, &RedisListener::trace_interfaces_error, trace_id, R.move_as_error()); 62 | return; 63 | } 64 | td::actor::send_closure(SelfId, &RedisListener::finish_processing, R.move_as_ok()); 65 | }); 66 | 67 | td::actor::create_actor("TraceInterfaceDetector", shard_states_, mc_data_state_.config_, std::move(trace), std::move(P)).release(); 68 | } else { 69 | finish_processing(std::move(trace)); 70 | } 71 | } 72 | 73 | void RedisListener::trace_interfaces_error(TraceId trace_id, td::Status error) { 74 | LOG(ERROR) << "Failed to detect interfaces on trace " << td::base64_encode(trace_id.as_slice()) << ": " << error; 75 | } 76 | 77 | void RedisListener::finish_processing(Trace trace) { 78 | auto P = td::PromiseCreator::lambda([trace_id = trace.id](td::Result R) { 79 | if (R.is_error()) { 80 | LOG(ERROR) << "Failed to insert trace " << td::base64_encode(trace_id.as_slice()) << ": " << R.move_as_error(); 81 | return; 82 | } 83 | LOG(DEBUG) << "Successfully inserted trace " << td::base64_encode(trace_id.as_slice()); 84 | }); 85 | trace_processor_(std::move(trace), std::move(P)); 86 | } -------------------------------------------------------------------------------- /ton-trace-emulator/src/RedisListener.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "IndexData.h" 7 | #include "TraceEmulator.h" 8 | #include "TraceInterfaceDetector.h" 9 | 10 | 11 | class RedisListener : public td::actor::Actor { 12 | private: 13 | sw::redis::Redis redis_; 14 | std::string queue_name_; 15 | std::function)> trace_processor_; 16 | 17 | MasterchainBlockDataState mc_data_state_; 18 | std::vector> shard_states_; 19 | 20 | std::unordered_set known_ext_msgs_; // this set grows infinitely. TODO: remove old messages 21 | 22 | public: 23 | RedisListener(std::string redis_dsn, std::string queue_name, std::function)> trace_processor) 24 | : redis_(sw::redis::Redis(redis_dsn)), queue_name_(queue_name), trace_processor_(std::move(trace_processor)) {}; 25 | 26 | virtual void start_up() override; 27 | 28 | void alarm() override; 29 | void set_mc_data_state(MasterchainBlockDataState mc_data_state); 30 | 31 | private: 32 | void trace_error(TraceId trace_id, td::Status error); 33 | void trace_received(TraceId trace_id, Trace trace); 34 | void trace_interfaces_error(TraceId trace_id, td::Status error); 35 | void finish_processing(Trace trace); 36 | }; 37 | -------------------------------------------------------------------------------- /ton-trace-emulator/src/TraceEmulator.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include "DbScanner.h" 6 | 7 | #include "smc-interfaces/InterfacesDetector.h" 8 | 9 | using TraceId = td::Bits256; 10 | 11 | struct AddrCmp { 12 | bool operator()(const block::StdAddress& lhs, const block::StdAddress& rhs) const { 13 | if (lhs.workchain != rhs.workchain) { 14 | return lhs.workchain < rhs.workchain; 15 | } 16 | return lhs.addr < rhs.addr; 17 | } 18 | }; 19 | 20 | struct TraceNode { 21 | td::Bits256 node_id; // hash of cur tx in msg 22 | block::StdAddress address; 23 | std::vector> children; 24 | td::Ref transaction_root; 25 | bool emulated; 26 | 27 | int depth() const { 28 | int res = 0; 29 | for (const auto& child : children) { 30 | res = std::max(res, child->depth()); 31 | } 32 | return res + 1; 33 | } 34 | 35 | int transactions_count() const { 36 | int res = transaction_root.not_null() ? 1 : 0; 37 | for (const auto& child : children) { 38 | res += child->transactions_count(); 39 | } 40 | return res; 41 | } 42 | 43 | std::string to_string(int tabs = 0) const { 44 | std::stringstream ss; 45 | ss << std::endl; 46 | for (int i = 0; i < tabs; i++) { 47 | ss << "--"; 48 | } 49 | // print account, lt, status, bounced, in_msg opcode 50 | block::gen::Transaction::Record trans; 51 | block::gen::TransactionDescr::Record_trans_ord descr; 52 | tlb::unpack_cell(transaction_root, trans); 53 | tlb::unpack_cell(trans.description, descr); 54 | ss << "TX acc=" << trans.account_addr.to_hex() << " lt=" << trans.lt << " outmsg_cnt=" << trans.outmsg_cnt << " aborted=" << descr.aborted << std::endl; 55 | 56 | for (const auto& child : children) { 57 | ss << child->to_string(tabs + 1); 58 | } 59 | return ss.str(); 60 | } 61 | }; 62 | 63 | struct Trace { 64 | TraceId id; 65 | std::unique_ptr root; 66 | td::Bits256 rand_seed; 67 | 68 | using Detector = InterfacesDetector; 71 | 72 | std::multimap emulated_accounts; 73 | std::unordered_map> interfaces; 74 | 75 | int depth() const { 76 | return root->depth(); 77 | } 78 | 79 | int transactions_count() const { 80 | return root->transactions_count(); 81 | } 82 | 83 | std::string to_string() const { 84 | return root->to_string(); 85 | } 86 | }; 87 | 88 | td::Result fetch_msg_dest_address(td::Ref msg, int& type); 89 | 90 | class TraceEmulatorImpl: public td::actor::Actor { 91 | private: 92 | std::shared_ptr emulator_; 93 | std::vector> shard_states_; 94 | std::multimap& emulated_accounts_; 95 | std::mutex& emulated_accounts_mutex_; 96 | std::unordered_map>& emulator_actors_; 97 | std::unordered_map, td::Promise>>> result_promises_; 98 | 99 | uint32_t utime_{0}; 100 | public: 101 | TraceEmulatorImpl(std::shared_ptr emulator, std::vector> shard_states, 102 | std::multimap& emulated_accounts, std::mutex& emulated_accounts_mutex, 103 | std::unordered_map>& emulator_actors) 104 | : emulator_(std::move(emulator)), shard_states_(std::move(shard_states)), 105 | emulated_accounts_(emulated_accounts), emulated_accounts_mutex_(emulated_accounts_mutex), emulator_actors_(emulator_actors) { 106 | } 107 | 108 | void emulate(td::Ref in_msg, block::StdAddress address, size_t depth, td::Promise> promise); 109 | 110 | private: 111 | td::Result unpack_account(vm::AugmentedDictionary& accounts_dict, const block::StdAddress& account_addr, uint32_t utime); 112 | void emulate_transaction(block::Account account, block::StdAddress address, 113 | td::Ref in_msg, size_t depth, td::Promise> promise); 114 | void child_emulated(TraceNode *parent_node_raw, std::unique_ptr child, size_t ind); 115 | void child_error(TraceNode *parent_node_raw, td::Status error); 116 | }; 117 | 118 | // Emulates whole trace, in_msg is external inbound message 119 | class TraceEmulator: public td::actor::Actor { 120 | private: 121 | MasterchainBlockDataState mc_data_state_; 122 | td::Ref in_msg_; 123 | bool ignore_chksig_; 124 | td::Promise promise_; 125 | td::Bits256 rand_seed_; 126 | 127 | std::shared_ptr emulator_; 128 | std::multimap emulated_accounts_; 129 | std::mutex emulated_accounts_mutex_; 130 | std::unordered_map> emulator_actors_; 131 | public: 132 | TraceEmulator(MasterchainBlockDataState mc_data_state, td::Ref in_msg, bool ignore_chksig, td::Promise promise); 133 | 134 | void start_up() override; 135 | void finish(td::Result> root); 136 | }; 137 | 138 | -------------------------------------------------------------------------------- /ton-trace-emulator/src/TraceInserter.cpp: -------------------------------------------------------------------------------- 1 | #include "TraceInserter.h" 2 | #include "Serializer.hpp" 3 | 4 | class TraceInserter: public td::actor::Actor { 5 | private: 6 | sw::redis::Transaction transaction_; 7 | Trace trace_; 8 | td::Promise promise_; 9 | 10 | public: 11 | TraceInserter(sw::redis::Transaction&& transaction, Trace trace, td::Promise promise) : 12 | transaction_(std::move(transaction)), trace_(std::move(trace)), promise_(std::move(promise)) { 13 | } 14 | 15 | void start_up() override { 16 | try { 17 | std::queue> queue; 18 | 19 | std::vector tx_keys_to_delete; 20 | std::vector> addr_keys_to_delete; 21 | std::vector flattened_trace; 22 | 23 | queue.push(*trace_.root); 24 | 25 | while (!queue.empty()) { 26 | TraceNode& current = queue.front(); 27 | 28 | for (auto& child : current.children) { 29 | queue.push(*child); 30 | } 31 | 32 | auto tx_r = parse_tx(current.transaction_root, current.address.workchain); 33 | if (tx_r.is_error()) { 34 | promise_.set_error(tx_r.move_as_error_prefix("Failed to parse transaction: ")); 35 | stop(); 36 | return; 37 | } 38 | auto tx = tx_r.move_as_ok(); 39 | 40 | if (!current.emulated) { 41 | delete_db_subtree(td::base64_encode(tx.in_msg.value().hash.as_slice()), tx_keys_to_delete, addr_keys_to_delete); 42 | } 43 | 44 | flattened_trace.push_back(RedisTraceNode{std::move(tx), current.emulated}); 45 | 46 | queue.pop(); 47 | } 48 | 49 | // delete previously emulated trace 50 | for (const auto& key : tx_keys_to_delete) { 51 | transaction_.hdel(td::base64_encode(trace_.id.as_slice()), key); 52 | } 53 | for (const auto& [addr, by_addr_key] : addr_keys_to_delete) { 54 | transaction_.zrem(addr, by_addr_key); 55 | } 56 | 57 | // insert new trace 58 | for (const auto& node : flattened_trace) { 59 | std::stringstream buffer; 60 | msgpack::pack(buffer, std::move(node)); 61 | 62 | transaction_.hset(td::base64_encode(trace_.id.as_slice()), td::base64_encode(node.transaction.in_msg.value().hash.as_slice()), buffer.str()); 63 | 64 | auto addr_raw = std::to_string(node.transaction.account.workchain) + ":" + node.transaction.account.addr.to_hex(); 65 | auto by_addr_key = td::base64_encode(trace_.id.as_slice()) + ":" + td::base64_encode(node.transaction.in_msg.value().hash.as_slice()); 66 | transaction_.zadd(addr_raw, by_addr_key, node.transaction.lt); 67 | } 68 | 69 | // insert interfaces 70 | for (const auto& [addr, interfaces] : trace_.interfaces) { 71 | auto interfaces_redis = parse_interfaces(interfaces); 72 | std::stringstream buffer; 73 | msgpack::pack(buffer, interfaces_redis); 74 | auto addr_raw = std::to_string(addr.workchain) + ":" + addr.addr.to_hex(); 75 | transaction_.hset(td::base64_encode(trace_.id.as_slice()), addr_raw, buffer.str()); 76 | } 77 | 78 | transaction_.publish("new_trace", td::base64_encode(trace_.id.as_slice())); 79 | transaction_.exec(); 80 | 81 | promise_.set_value(td::Unit()); 82 | } catch (const vm::VmError &e) { 83 | promise_.set_error(td::Status::Error("Got VmError while inserting trace: " + std::string(e.get_msg()))); 84 | } catch (const std::exception &e) { 85 | promise_.set_error(td::Status::Error("Got exception while inserting trace: " + std::string(e.what()))); 86 | } 87 | stop(); 88 | } 89 | void delete_db_subtree(std::string key, std::vector& tx_keys, std::vector>& addr_keys) { 90 | auto emulated_in_db = transaction_.redis().hget(td::base64_encode(trace_.id.as_slice()), key); 91 | if (emulated_in_db) { 92 | auto serialized = emulated_in_db.value(); 93 | RedisTraceNode node; 94 | msgpack::unpacked result; 95 | msgpack::unpack(result, serialized.data(), serialized.size()); 96 | result.get().convert(node); 97 | for (const auto& out_msg : node.transaction.out_msgs) { 98 | delete_db_subtree(td::base64_encode(out_msg.hash.as_slice()), tx_keys, addr_keys); 99 | } 100 | tx_keys.push_back(key); 101 | 102 | auto addr_raw = std::to_string(node.transaction.account.workchain) + ":" + node.transaction.account.addr.to_hex(); 103 | auto by_addr_key = td::base64_encode(trace_.id.as_slice()) + ":" + td::base64_encode(node.transaction.in_msg.value().hash.as_slice()); 104 | addr_keys.push_back(std::make_pair(addr_raw, by_addr_key)); 105 | } 106 | } 107 | }; 108 | 109 | void RedisInsertManager::insert(Trace trace, td::Promise promise) { 110 | td::actor::create_actor("TraceInserter", redis_.transaction(), std::move(trace), std::move(promise)).release(); 111 | } -------------------------------------------------------------------------------- /ton-trace-emulator/src/TraceInserter.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "td/actor/actor.h" 4 | #include "crypto/common/bitstring.h" 5 | #include "TraceEmulator.h" 6 | 7 | 8 | class ITraceInsertManager : public td::actor::Actor { 9 | public: 10 | virtual void insert(Trace trace, td::Promise promise) = 0; 11 | }; 12 | 13 | class RedisInsertManager: public ITraceInsertManager { 14 | private: 15 | sw::redis::Redis redis_; 16 | 17 | public: 18 | RedisInsertManager(std::string redis_dsn) : 19 | redis_(sw::redis::Redis(redis_dsn)) {} 20 | 21 | void insert(Trace trace, td::Promise promise); 22 | }; -------------------------------------------------------------------------------- /ton-trace-emulator/src/TraceInterfaceDetector.cpp: -------------------------------------------------------------------------------- 1 | #include "TraceInterfaceDetector.h" 2 | #include "smc-interfaces/InterfacesDetector.h" 3 | 4 | void TraceInterfaceDetector::start_up() { 5 | td::MultiPromise mp; 6 | auto ig = mp.init_guard(); 7 | auto P = td::PromiseCreator::lambda([&, SelfId=actor_id(this)](td::Result res) mutable { 8 | td::actor::send_closure(SelfId, &TraceInterfaceDetector::finish, std::move(res)); 9 | }); 10 | ig.add_promise(std::move(P)); 11 | 12 | // Detect interfaces for final state of each account 13 | std::unordered_set processed_addresses; 14 | for (auto it = trace_.emulated_accounts.rbegin(); it != trace_.emulated_accounts.rend(); it++) { 15 | const auto& [address, account] = *it; 16 | if (processed_addresses.count(address)) { 17 | continue; 18 | } 19 | processed_addresses.insert(address); 20 | trace_.interfaces[address] = {}; 21 | td::actor::create_actor 22 | ("InterfacesDetector", address, account.code, account.data, shard_states_, config_, 23 | td::PromiseCreator::lambda([SelfId = actor_id(this), address, promise = ig.get_promise()](std::vector interfaces) mutable { 24 | td::actor::send_closure(SelfId, &TraceInterfaceDetector::got_interfaces, address, std::move(interfaces), std::move(promise)); 25 | })).release(); 26 | } 27 | 28 | } 29 | 30 | void TraceInterfaceDetector::got_interfaces(block::StdAddress address, std::vector interfaces, td::Promise promise) { 31 | trace_.interfaces[address] = std::move(interfaces); 32 | promise.set_value(td::Unit()); 33 | } 34 | 35 | void TraceInterfaceDetector::finish(td::Result status) { 36 | if (status.is_error()) { 37 | promise_.set_error(status.move_as_error_prefix("Failed to detect interfaces: ")); 38 | } else { 39 | promise_.set_value(std::move(trace_)); 40 | } 41 | stop(); 42 | } -------------------------------------------------------------------------------- /ton-trace-emulator/src/TraceInterfaceDetector.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "TraceEmulator.h" 4 | 5 | class TraceInterfaceDetector: public td::actor::Actor { 6 | private: 7 | AllShardStates shard_states_; 8 | std::shared_ptr config_; 9 | Trace trace_; 10 | td::Promise promise_; 11 | public: 12 | TraceInterfaceDetector(AllShardStates shard_states, std::shared_ptr config, 13 | Trace trace, td::Promise promise) : 14 | shard_states_(shard_states), config_(config), trace_(std::move(trace)), promise_(std::move(promise)) { 15 | 16 | } 17 | 18 | void start_up() override; 19 | 20 | private: 21 | void got_interfaces(block::StdAddress address, std::vector interfaces, td::Promise promise); 22 | void finish(td::Result status); 23 | }; -------------------------------------------------------------------------------- /ton-trace-emulator/src/TraceScheduler.cpp: -------------------------------------------------------------------------------- 1 | #include "TraceScheduler.h" 2 | #include "BlockEmulator.h" 3 | #include "TraceInserter.h" 4 | 5 | 6 | void TraceEmulatorScheduler::start_up() { 7 | alarm_timestamp() = td::Timestamp::in(0.1); 8 | 9 | if (global_config_path_.empty() || inet_addr_.empty()) { 10 | LOG(WARNING) << "Global config path or inet addr is empty. OverlayListener was not started."; 11 | } else { 12 | overlay_listener_ = td::actor::create_actor("OverlayListener", global_config_path_, inet_addr_, insert_trace_); 13 | } 14 | 15 | if (input_redis_queue_.empty()) { 16 | LOG(WARNING) << "Input redis queue name is empty. RedisListener was not started."; 17 | } else { 18 | redis_listener_ = td::actor::create_actor("RedisListener", redis_dsn_, input_redis_queue_, insert_trace_); 19 | } 20 | } 21 | 22 | void TraceEmulatorScheduler::got_last_mc_seqno(ton::BlockSeqno new_last_known_seqno) { 23 | if (new_last_known_seqno == last_known_seqno_) { 24 | return; 25 | } 26 | 27 | LOG(INFO) << "New masterchain block " << new_last_known_seqno; 28 | 29 | if (last_known_seqno_ == 0) { 30 | last_known_seqno_ = new_last_known_seqno; 31 | last_fetched_seqno_ = new_last_known_seqno; 32 | return; 33 | } 34 | 35 | if (new_last_known_seqno > last_known_seqno_ + 1) { 36 | LOG(WARNING) << "More than one new masterchain block appeared. Skipping to the newest one, from " << last_known_seqno_ << " to " << new_last_known_seqno; 37 | } 38 | 39 | for (auto seqno = last_known_seqno_ + 1; seqno <= new_last_known_seqno; seqno++) { 40 | seqnos_to_fetch_.insert(seqno); 41 | } 42 | 43 | last_known_seqno_ = new_last_known_seqno; 44 | } 45 | 46 | void TraceEmulatorScheduler::fetch_seqnos() { 47 | for (auto it = seqnos_to_fetch_.begin(); it != seqnos_to_fetch_.end(); ) { 48 | auto seqno = *it; 49 | 50 | auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), seqno](td::Result R) { 51 | if (R.is_error()) { 52 | td::actor::send_closure(SelfId, &TraceEmulatorScheduler::fetch_error, seqno, R.move_as_error()); 53 | return; 54 | } 55 | auto mc_block_ds = R.move_as_ok(); 56 | for (auto &block_ds : mc_block_ds.shard_blocks_) { 57 | if (block_ds.block_data->block_id().is_masterchain()) { 58 | mc_block_ds.config_ = block::ConfigInfo::extract_config(block_ds.block_state, block::ConfigInfo::needCapabilities | block::ConfigInfo::needLibraries | block::ConfigInfo::needWorkchainInfo | block::ConfigInfo::needSpecialSmc).move_as_ok(); 59 | break; 60 | } 61 | } 62 | td::actor::send_closure(SelfId, &TraceEmulatorScheduler::seqno_fetched, seqno, std::move(mc_block_ds)); 63 | }); 64 | td::actor::send_closure(db_scanner_, &DbScanner::fetch_seqno, seqno, std::move(P)); 65 | 66 | it = seqnos_to_fetch_.erase(it); 67 | } 68 | } 69 | 70 | void TraceEmulatorScheduler::fetch_error(std::uint32_t seqno, td::Status error) { 71 | LOG(ERROR) << "Failed to fetch seqno " << seqno << ": " << std::move(error); 72 | seqnos_to_fetch_.insert(seqno); 73 | alarm_timestamp() = td::Timestamp::in(0.1); 74 | } 75 | 76 | void TraceEmulatorScheduler::seqno_fetched(std::uint32_t seqno, MasterchainBlockDataState mc_data_state) { 77 | LOG(INFO) << "Fetched seqno " << seqno; 78 | 79 | if (seqno > last_fetched_seqno_) { 80 | LOG(INFO) << "Setting last fetched seqno to " << seqno; 81 | last_fetched_seqno_ = seqno; 82 | 83 | if (!overlay_listener_.empty()) { 84 | td::actor::send_closure(overlay_listener_, &OverlayListener::set_mc_data_state, mc_data_state); 85 | } 86 | 87 | if (!redis_listener_.empty()) { 88 | td::actor::send_closure(redis_listener_, &RedisListener::set_mc_data_state, mc_data_state); 89 | } 90 | } 91 | 92 | blocks_to_emulate_[seqno] = mc_data_state; 93 | emulate_blocks(); 94 | } 95 | 96 | void TraceEmulatorScheduler::emulate_blocks() { 97 | if (last_emulated_seqno_ == 0) { 98 | last_emulated_seqno_ = last_fetched_seqno_; 99 | } 100 | 101 | auto it = blocks_to_emulate_.find(last_emulated_seqno_ + 1); 102 | while(it != blocks_to_emulate_.end()) { 103 | LOG(ERROR) << "Emulating mc block " << last_emulated_seqno_ + 1; 104 | auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), blkid = it->second.shard_blocks_[0].block_data->block_id().id](td::Result<> R) { 105 | if (R.is_error()) { 106 | LOG(ERROR) << "Error emulating mc block " << blkid.to_str(); 107 | return; 108 | } 109 | LOG(INFO) << "Success emulating mc block " << blkid.to_str(); 110 | }); 111 | td::actor::create_actor("McBlockEmulator", it->second, insert_trace_, std::move(P)).release(); 112 | 113 | blocks_to_emulate_.erase(it); 114 | last_emulated_seqno_++; 115 | it = blocks_to_emulate_.find(last_emulated_seqno_ + 1); 116 | } 117 | } 118 | 119 | // int seqno = 37786481; 120 | 121 | void TraceEmulatorScheduler::alarm() { 122 | auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R){ 123 | if (R.is_error()) { 124 | LOG(ERROR) << "Failed to update last seqno: " << R.move_as_error(); 125 | std::_Exit(2); 126 | return; 127 | } 128 | td::actor::send_closure(SelfId, &TraceEmulatorScheduler::got_last_mc_seqno, R.move_as_ok()); 129 | // td::actor::send_closure(SelfId, &TraceEmulatorScheduler::got_last_mc_seqno, seqno++); // for debugging 130 | }); 131 | td::actor::send_closure(db_scanner_, &DbScanner::get_last_mc_seqno, std::move(P)); 132 | 133 | fetch_seqnos(); 134 | 135 | alarm_timestamp() = td::Timestamp::in(0.3); 136 | } 137 | -------------------------------------------------------------------------------- /ton-trace-emulator/src/TraceScheduler.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "td/actor/actor.h" 4 | #include "DbScanner.h" 5 | #include "OverlayListener.h" 6 | #include "RedisListener.h" 7 | #include "TraceEmulator.h" 8 | #include "TraceInserter.h" 9 | 10 | 11 | class TraceEmulatorScheduler : public td::actor::Actor { 12 | private: 13 | td::actor::ActorId db_scanner_; 14 | std::string global_config_path_; 15 | std::string inet_addr_; 16 | std::string redis_dsn_; 17 | std::string input_redis_queue_; 18 | std::function)> insert_trace_; 19 | 20 | ton::BlockSeqno last_known_seqno_{0}; 21 | ton::BlockSeqno last_fetched_seqno_{0}; 22 | ton::BlockSeqno last_emulated_seqno_{0}; 23 | 24 | std::unordered_set seqnos_to_fetch_; 25 | std::map blocks_to_emulate_; 26 | 27 | td::actor::ActorOwn overlay_listener_; 28 | td::actor::ActorOwn redis_listener_; 29 | td::actor::ActorOwn insert_manager_; 30 | 31 | void got_last_mc_seqno(ton::BlockSeqno last_known_seqno); 32 | void fetch_seqnos(); 33 | void fetch_error(std::uint32_t seqno, td::Status error); 34 | void seqno_fetched(std::uint32_t seqno, MasterchainBlockDataState mc_data_state); 35 | void emulate_blocks(); 36 | 37 | void alarm(); 38 | 39 | public: 40 | TraceEmulatorScheduler(td::actor::ActorId db_scanner, td::actor::ActorId insert_manager, 41 | std::string global_config_path, std::string inet_addr, 42 | std::string redis_dsn, std::string input_redis_queue) : 43 | db_scanner_(db_scanner), insert_manager_(insert_manager), global_config_path_(global_config_path), inet_addr_(inet_addr), redis_dsn_(redis_dsn), input_redis_queue_(input_redis_queue) { 44 | insert_trace_ = [insert_manager = insert_manager_.get()](Trace trace, td::Promise promise) { 45 | td::actor::send_closure(insert_manager, &ITraceInsertManager::insert, std::move(trace), std::move(promise)); 46 | }; 47 | }; 48 | 49 | virtual void start_up() override; 50 | }; 51 | -------------------------------------------------------------------------------- /ton-trace-emulator/src/main.cpp: -------------------------------------------------------------------------------- 1 | #include "td/utils/port/signals.h" 2 | #include "td/utils/OptionParser.h" 3 | #include "td/utils/format.h" 4 | #include "td/utils/logging.h" 5 | #include "td/utils/check.h" 6 | 7 | #include "crypto/vm/cp0.h" 8 | 9 | #include "DbScanner.h" 10 | #include "TraceScheduler.h" 11 | #include "TraceInserter.h" 12 | 13 | 14 | int main(int argc, char *argv[]) { 15 | SET_VERBOSITY_LEVEL(verbosity_INFO); 16 | td::set_default_failure_signal_handler().ensure(); 17 | 18 | CHECK(vm::init_op_cp0()); 19 | 20 | // options 21 | std::string db_root; 22 | std::string working_dir; 23 | td::uint32 threads = 7; 24 | std::string redis_dsn = "tcp://127.0.0.1:6379"; 25 | std::string redis_queue_name = ""; 26 | 27 | std::string global_config_path; 28 | std::string inet_addr; 29 | 30 | td::OptionParser p; 31 | p.set_description("Emulate TON traces"); 32 | p.add_option('\0', "help", "prints_help", [&]() { 33 | char b[10240]; 34 | td::StringBuilder sb(td::MutableSlice{b, 10000}); 35 | sb << p; 36 | std::cout << sb.as_cslice().c_str(); 37 | std::exit(2); 38 | }); 39 | p.add_option('D', "db", "Path to TON DB folder", [&](td::Slice fname) { 40 | db_root = fname.str(); 41 | }); 42 | p.add_option('W', "working-dir", "Path to index working dir for secondary rocksdb logs", [&](td::Slice fname) { 43 | working_dir = fname.str(); 44 | }); 45 | 46 | p.add_checked_option('t', "threads", "Scheduler threads (default: 7)", [&](td::Slice fname) { 47 | int v; 48 | try { 49 | v = std::stoi(fname.str()); 50 | } catch (...) { 51 | return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: not a number"); 52 | } 53 | threads = v; 54 | return td::Status::OK(); 55 | }); 56 | 57 | p.add_option('\0', "redis", "Redis URI (default: 'tcp://127.0.0.1:6379')", [&](td::Slice fname) { 58 | redis_dsn = fname.str(); 59 | }); 60 | 61 | p.add_option('\0', "redis-queue", "Redis queue name for input msgs", [&](td::Slice fname) { 62 | redis_queue_name = fname.str(); 63 | }); 64 | 65 | p.add_option('\0', "global-config", "Path to global config json file (for listening overlay)", [&](td::Slice fname) { 66 | global_config_path = fname.str(); 67 | }); 68 | 69 | p.add_option('\0', "addr", "ip:port of this machine (for listening overlay)", [&](td::Slice fname) { 70 | inet_addr = fname.str(); 71 | }); 72 | 73 | 74 | auto S = p.run(argc, argv); 75 | if (S.is_error()) { 76 | LOG(ERROR) << "failed to parse options: " << S.move_as_error(); 77 | std::_Exit(2); 78 | } 79 | 80 | if (db_root.size() == 0) { 81 | std::cerr << "'--db' option missing" << std::endl; 82 | std::_Exit(2); 83 | } 84 | 85 | if (working_dir.size() == 0) { 86 | working_dir = PSTRING() << "/tmp/index_worker_" << getpid(); 87 | LOG(WARNING) << "Working dir not specified, using " << working_dir; 88 | } 89 | 90 | if (global_config_path.empty() ^ inet_addr.empty()) { 91 | std::cerr << "'--global-config' must be present with '--addr'" << std::endl; 92 | std::_Exit(2); 93 | } 94 | 95 | td::actor::Scheduler scheduler({threads}); 96 | td::actor::ActorOwn db_scanner; 97 | td::actor::ActorOwn insert_manager; 98 | 99 | scheduler.run_in_context([&] { 100 | db_scanner = td::actor::create_actor("scanner", db_root, dbs_secondary, working_dir, 0.5); 101 | insert_manager = td::actor::create_actor("RedisInsertManager", redis_dsn); 102 | td::actor::create_actor("integritychecker", db_scanner.get(), insert_manager.get(), global_config_path, inet_addr, redis_dsn, redis_queue_name).release(); 103 | }); 104 | 105 | scheduler.run(); 106 | 107 | return 0; 108 | } 109 | -------------------------------------------------------------------------------- /ton-trace-task-emulator/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.16) 2 | 3 | add_executable(ton-trace-task-emulator 4 | src/main.cpp 5 | src/TraceTaskScheduler.cpp 6 | src/TaskResultInserter.cpp 7 | src/RedisListener.cpp 8 | ) 9 | target_include_directories(ton-trace-task-emulator 10 | PUBLIC ${CMAKE_BINARY_DIR}/external/redis-plus-plus/src 11 | ) 12 | target_compile_features(ton-trace-task-emulator PRIVATE cxx_std_20) 13 | target_link_libraries(ton-trace-task-emulator ton-trace-emulator-core hiredis redis++ msgpack-cxx) 14 | target_link_options(ton-trace-task-emulator PUBLIC -rdynamic) 15 | target_compile_definitions(ton-trace-task-emulator PRIVATE -DMSGPACK_USE_DEFINE_MAP) 16 | install(TARGETS ton-trace-task-emulator RUNTIME DESTINATION bin) -------------------------------------------------------------------------------- /ton-trace-task-emulator/src/RedisListener.cpp: -------------------------------------------------------------------------------- 1 | #include "RedisListener.h" 2 | 3 | 4 | void RedisListener::start_up() { 5 | alarm_timestamp() = td::Timestamp::now() ; 6 | } 7 | 8 | void RedisListener::alarm() { 9 | while (auto buffer = redis_.rpop(queue_name_)) { 10 | TraceTask task; 11 | try { 12 | size_t offset = 0; 13 | msgpack::unpacked res; 14 | msgpack::unpack(res, buffer.value().data(), buffer.value().size(), offset); 15 | msgpack::object obj = res.get(); 16 | 17 | obj.convert(task); 18 | } catch (const std::exception &e) { 19 | LOG(ERROR) << "Failed to unpack trace task: " << e.what(); 20 | continue; 21 | } 22 | 23 | auto boc_decoded = td::base64_decode(task.boc); 24 | if (boc_decoded.is_error()) { 25 | auto error = td::Status::Error(PSLICE() << "Can't decode base64 boc: " << boc_decoded.move_as_error()); 26 | LOG(ERROR) << error; 27 | TraceEmulationResult res{std::move(task), std::move(error), current_mc_block_id_}; 28 | trace_processor_(std::move(res), td::PromiseCreator::lambda([](td::Result R) {})); 29 | continue; 30 | } 31 | auto msg_cell_r = vm::std_boc_deserialize(boc_decoded.move_as_ok()); 32 | if (msg_cell_r.is_error()) { 33 | auto error = td::Status::Error(PSLICE() << "Can't deserialize message boc: " << msg_cell_r.move_as_error()); 34 | LOG(ERROR) << error; 35 | TraceEmulationResult res{std::move(task), std::move(error), current_mc_block_id_}; 36 | trace_processor_(std::move(res), td::PromiseCreator::lambda([](td::Result R) {})); 37 | continue; 38 | } 39 | auto msg_cell = msg_cell_r.move_as_ok(); 40 | 41 | if (mc_data_state_.config_ == nullptr) { 42 | trace_error(std::move(task), current_mc_block_id_, td::Status::Error("RedisListener not ready")); 43 | alarm_timestamp() = td::Timestamp::in(0.1); 44 | return; 45 | } 46 | 47 | auto ignore_chksig = task.ignore_chksig; 48 | auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), task = std::move(task), mc_blkid = current_mc_block_id_](td::Result R) mutable { 49 | if (R.is_error()) { 50 | td::actor::send_closure(SelfId, &RedisListener::trace_error, std::move(task), std::move(mc_blkid), R.move_as_error()); 51 | } else { 52 | td::actor::send_closure(SelfId, &RedisListener::trace_received, std::move(task), std::move(mc_blkid), R.move_as_ok()); 53 | } 54 | }); 55 | td::actor::create_actor("TraceEmu", mc_data_state_, msg_cell, ignore_chksig, std::move(P)).release(); 56 | } 57 | 58 | alarm_timestamp() = td::Timestamp::now(); 59 | } 60 | 61 | void RedisListener::set_mc_data_state(MasterchainBlockDataState mc_data_state) { 62 | shard_states_.clear(); 63 | for (const auto& shard_state : mc_data_state.shard_blocks_) { 64 | shard_states_.push_back(shard_state.block_state); 65 | } 66 | 67 | current_mc_block_id_ = mc_data_state.shard_blocks_[0].block_data->block_id().id; 68 | 69 | mc_data_state_ = std::move(mc_data_state); 70 | } 71 | 72 | void RedisListener::trace_error(TraceTask task, ton::BlockId mc_block_id, td::Status error) { 73 | LOG(ERROR) << "Failed to emulate trace " << task.id << ": " << error; 74 | TraceEmulationResult res{std::move(task), error.move_as_error(), mc_block_id}; 75 | trace_processor_(std::move(res), td::PromiseCreator::lambda([](td::Result R) {})); 76 | } 77 | 78 | void RedisListener::trace_received(TraceTask task, ton::BlockId mc_block_id, Trace trace) { 79 | LOG(INFO) << "Emulated trace " << task.id << ": " << trace.transactions_count() << " transactions, " << trace.depth() << " depth"; 80 | if (task.detect_interfaces) { 81 | auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), task = std::move(task), mc_block_id = std::move(mc_block_id), trace_id = trace.id](td::Result R) { 82 | if (R.is_error()) { 83 | td::actor::send_closure(SelfId, &RedisListener::trace_interfaces_error, std::move(task), std::move(mc_block_id), R.move_as_error()); 84 | return; 85 | } 86 | td::actor::send_closure(SelfId, &RedisListener::finish_processing, std::move(task), std::move(mc_block_id), R.move_as_ok()); 87 | }); 88 | 89 | td::actor::create_actor("TraceInterfaceDetector", shard_states_, mc_data_state_.config_, std::move(trace), std::move(P)).release(); 90 | } else { 91 | finish_processing(std::move(task), std::move(mc_block_id), std::move(trace)); 92 | } 93 | } 94 | 95 | void RedisListener::trace_interfaces_error(TraceTask task, ton::BlockId mc_block_id, td::Status error) { 96 | LOG(ERROR) << "Failed to detect interfaces on task " << task.id << ": " << error; 97 | TraceEmulationResult res{std::move(task), error.move_as_error(), mc_block_id}; 98 | trace_processor_(std::move(res), td::PromiseCreator::lambda([](td::Result R) {})); 99 | } 100 | 101 | void RedisListener::finish_processing(TraceTask task, ton::BlockId mc_block_id, Trace trace) { 102 | LOG(INFO) << "Finished emulating trace " << task.id; 103 | auto P = td::PromiseCreator::lambda([task_id = task.id](td::Result R) { 104 | if (R.is_error()) { 105 | LOG(ERROR) << "Failed to insert trace task " << task_id << ": " << R.move_as_error(); 106 | return; 107 | } 108 | LOG(DEBUG) << "Successfully inserted trace task" << task_id; 109 | }); 110 | TraceEmulationResult res{std::move(task), std::move(trace), mc_block_id}; 111 | trace_processor_(std::move(res), std::move(P)); 112 | } -------------------------------------------------------------------------------- /ton-trace-task-emulator/src/RedisListener.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "IndexData.h" 8 | #include "TraceEmulator.h" 9 | #include "TraceInterfaceDetector.h" 10 | 11 | struct TraceTask { 12 | std::string id; 13 | std::string boc; 14 | bool ignore_chksig; 15 | bool detect_interfaces; 16 | bool include_code_data; 17 | 18 | MSGPACK_DEFINE(id, boc, ignore_chksig, detect_interfaces, include_code_data); 19 | }; 20 | 21 | struct TraceEmulationResult { 22 | TraceTask task; 23 | td::Result trace; 24 | ton::BlockId mc_block_id; 25 | }; 26 | 27 | class RedisListener : public td::actor::Actor { 28 | private: 29 | sw::redis::Redis redis_; 30 | std::string queue_name_; 31 | std::function)> trace_processor_; 32 | 33 | MasterchainBlockDataState mc_data_state_; 34 | std::vector> shard_states_; 35 | ton::BlockId current_mc_block_id_; 36 | 37 | public: 38 | RedisListener(std::string redis_dsn, std::string queue_name, typeof(trace_processor_) trace_processor) 39 | : redis_(sw::redis::Redis(redis_dsn)), queue_name_(queue_name), trace_processor_(std::move(trace_processor)) {}; 40 | 41 | virtual void start_up() override; 42 | 43 | void alarm() override; 44 | void set_mc_data_state(MasterchainBlockDataState mc_data_state); 45 | 46 | private: 47 | void trace_error(TraceTask task, ton::BlockId mc_block_id, td::Status error); 48 | void trace_received(TraceTask task, ton::BlockId mc_block_id, Trace trace); 49 | void trace_interfaces_error(TraceTask task, ton::BlockId mc_block_id, td::Status error); 50 | void finish_processing(TraceTask task, ton::BlockId mc_block_id, Trace trace); 51 | }; -------------------------------------------------------------------------------- /ton-trace-task-emulator/src/TaskResultInserter.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "td/actor/actor.h" 4 | #include "crypto/common/bitstring.h" 5 | #include "TraceEmulator.h" 6 | #include "RedisListener.h" 7 | 8 | class ITaskResultInserter : public td::actor::Actor { 9 | public: 10 | virtual void insert(TraceEmulationResult result, td::Promise promise) = 0; 11 | }; 12 | 13 | class RedisTaskResultInsertManager: public ITaskResultInserter { 14 | private: 15 | sw::redis::Redis redis_; 16 | 17 | public: 18 | RedisTaskResultInsertManager(std::string redis_dsn) : 19 | redis_(sw::redis::Redis(redis_dsn)) {} 20 | 21 | void insert(TraceEmulationResult result, td::Promise promise); 22 | }; -------------------------------------------------------------------------------- /ton-trace-task-emulator/src/TraceTaskScheduler.cpp: -------------------------------------------------------------------------------- 1 | #include "TraceTaskScheduler.h" 2 | #include "TaskResultInserter.h" 3 | 4 | 5 | void TraceTaskScheduler::start_up() { 6 | alarm_timestamp() = td::Timestamp::in(0.1); 7 | } 8 | 9 | void TraceTaskScheduler::got_last_mc_seqno(ton::BlockSeqno new_last_known_seqno) { 10 | if (new_last_known_seqno == last_known_seqno_) { 11 | return; 12 | } 13 | 14 | LOG(INFO) << "New masterchain block " << new_last_known_seqno; 15 | 16 | if (last_known_seqno_ == 0) { 17 | last_known_seqno_ = new_last_known_seqno; 18 | last_fetched_seqno_ = new_last_known_seqno; 19 | return; 20 | } 21 | 22 | if (new_last_known_seqno > last_known_seqno_ + 1) { 23 | LOG(WARNING) << "More than one new masterchain block appeared. Skipping to the newest one, from " << last_known_seqno_ << " to " << new_last_known_seqno; 24 | } 25 | 26 | for (auto seqno = last_known_seqno_ + 1; seqno <= new_last_known_seqno; seqno++) { 27 | seqnos_to_fetch_.insert(seqno); 28 | } 29 | 30 | last_known_seqno_ = new_last_known_seqno; 31 | } 32 | 33 | void TraceTaskScheduler::fetch_seqnos() { 34 | for (auto it = seqnos_to_fetch_.begin(); it != seqnos_to_fetch_.end(); ) { 35 | auto seqno = *it; 36 | 37 | auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), seqno](td::Result R) { 38 | if (R.is_error()) { 39 | td::actor::send_closure(SelfId, &TraceTaskScheduler::fetch_error, seqno, R.move_as_error()); 40 | return; 41 | } 42 | auto mc_block_ds = R.move_as_ok(); 43 | for (auto &block_ds : mc_block_ds.shard_blocks_) { 44 | if (block_ds.block_data->block_id().is_masterchain()) { 45 | mc_block_ds.config_ = block::ConfigInfo::extract_config(block_ds.block_state, block::ConfigInfo::needCapabilities | block::ConfigInfo::needLibraries | block::ConfigInfo::needWorkchainInfo | block::ConfigInfo::needSpecialSmc).move_as_ok(); 46 | break; 47 | } 48 | } 49 | td::actor::send_closure(SelfId, &TraceTaskScheduler::seqno_fetched, seqno, std::move(mc_block_ds)); 50 | }); 51 | td::actor::send_closure(db_scanner_, &DbScanner::fetch_seqno, seqno, std::move(P)); 52 | 53 | it = seqnos_to_fetch_.erase(it); 54 | } 55 | } 56 | 57 | void TraceTaskScheduler::fetch_error(std::uint32_t seqno, td::Status error) { 58 | if (error.code() != ton::ErrorCode::notready) { 59 | LOG(ERROR) << "Failed to fetch seqno " << seqno << ": " << std::move(error); 60 | } 61 | seqnos_to_fetch_.insert(seqno); 62 | alarm_timestamp() = td::Timestamp::in(0.1); 63 | } 64 | 65 | void TraceTaskScheduler::seqno_fetched(std::uint32_t seqno, MasterchainBlockDataState mc_data_state) { 66 | LOG(INFO) << "Fetched seqno " << seqno; 67 | 68 | if (seqno > last_fetched_seqno_) { 69 | auto time_diff = td::Clocks::system() - mc_data_state.shard_blocks_[0].handle->unix_time(); 70 | LOG(INFO) << "Setting last fetched seqno to " << seqno << ", created " << td::StringBuilder::FixedDouble(time_diff, 2) << "s ago"; 71 | last_fetched_seqno_ = seqno; 72 | 73 | if (!redis_listener_.empty()) { 74 | td::actor::send_closure(redis_listener_, &RedisListener::set_mc_data_state, mc_data_state); 75 | } 76 | } 77 | } 78 | 79 | // static uint32_t seqno = 43074152; 80 | // static bool changed = false; 81 | 82 | void TraceTaskScheduler::alarm() { 83 | auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R){ 84 | if (R.is_error()) { 85 | LOG(ERROR) << "Failed to update last seqno: " << R.move_as_error(); 86 | std::_Exit(2); 87 | return; 88 | } 89 | td::actor::send_closure(SelfId, &TraceTaskScheduler::got_last_mc_seqno, R.move_as_ok()); 90 | 91 | // td::actor::send_closure(SelfId, &TraceTaskScheduler::got_last_mc_seqno, seqno - 1); 92 | // if (!changed) { 93 | // seqno += 1; 94 | // changed = true; 95 | // } 96 | }); 97 | td::actor::send_closure(db_scanner_, &DbScanner::get_last_mc_seqno, std::move(P)); 98 | 99 | fetch_seqnos(); 100 | 101 | alarm_timestamp() = td::Timestamp::in(0.3); 102 | } 103 | -------------------------------------------------------------------------------- /ton-trace-task-emulator/src/TraceTaskScheduler.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "td/actor/actor.h" 4 | #include "DbScanner.h" 5 | #include "RedisListener.h" 6 | #include "TraceEmulator.h" 7 | #include "TaskResultInserter.h" 8 | 9 | 10 | class TraceTaskScheduler : public td::actor::Actor { 11 | private: 12 | td::actor::ActorId db_scanner_; 13 | std::string redis_dsn_; 14 | std::string input_redis_queue_; 15 | std::function)> insert_trace_; 16 | 17 | ton::BlockSeqno last_known_seqno_{0}; 18 | ton::BlockSeqno last_fetched_seqno_{0}; 19 | ton::BlockSeqno last_emulated_seqno_{0}; 20 | 21 | std::unordered_set seqnos_to_fetch_; 22 | 23 | td::actor::ActorOwn redis_listener_; 24 | td::actor::ActorOwn insert_manager_; 25 | 26 | void got_last_mc_seqno(ton::BlockSeqno last_known_seqno); 27 | void fetch_seqnos(); 28 | void fetch_error(std::uint32_t seqno, td::Status error); 29 | void seqno_fetched(std::uint32_t seqno, MasterchainBlockDataState mc_data_state); 30 | 31 | void alarm(); 32 | 33 | public: 34 | TraceTaskScheduler(td::actor::ActorId db_scanner, td::actor::ActorId insert_manager, 35 | std::string redis_dsn, std::string input_redis_queue) : 36 | db_scanner_(db_scanner), insert_manager_(insert_manager), redis_dsn_(redis_dsn), input_redis_queue_(input_redis_queue) { 37 | insert_trace_ = [insert_manager = insert_manager_.get()](TraceEmulationResult result, td::Promise promise) { 38 | td::actor::send_closure(insert_manager, &ITaskResultInserter::insert, std::move(result), std::move(promise)); 39 | }; 40 | redis_listener_ = td::actor::create_actor("RedisListener", redis_dsn_, input_redis_queue_, insert_trace_); 41 | }; 42 | 43 | virtual void start_up() override; 44 | }; 45 | -------------------------------------------------------------------------------- /ton-trace-task-emulator/src/main.cpp: -------------------------------------------------------------------------------- 1 | #include "td/utils/port/signals.h" 2 | #include "td/utils/OptionParser.h" 3 | #include "td/utils/format.h" 4 | #include "td/utils/logging.h" 5 | #include "td/utils/check.h" 6 | 7 | #include "crypto/vm/cp0.h" 8 | 9 | #include "DbScanner.h" 10 | #include "TraceTaskScheduler.h" 11 | #include "TaskResultInserter.h" 12 | 13 | 14 | int main(int argc, char *argv[]) { 15 | SET_VERBOSITY_LEVEL(verbosity_INFO); 16 | td::set_default_failure_signal_handler().ensure(); 17 | 18 | CHECK(vm::init_op_cp0()); 19 | 20 | // options 21 | std::string db_root; 22 | std::string working_dir; 23 | td::uint32 threads = 7; 24 | std::string redis_dsn = "tcp://127.0.0.1:6379"; 25 | std::string redis_queue_name = "emulatorqueue"; 26 | 27 | td::OptionParser p; 28 | p.set_description("Emulate TON traces"); 29 | p.add_option('\0', "help", "prints_help", [&]() { 30 | char b[10240]; 31 | td::StringBuilder sb(td::MutableSlice{b, 10000}); 32 | sb << p; 33 | std::cout << sb.as_cslice().c_str(); 34 | std::exit(2); 35 | }); 36 | p.add_option('D', "db", "Path to TON DB folder", [&](td::Slice fname) { 37 | db_root = fname.str(); 38 | }); 39 | p.add_option('W', "working-dir", "Path to index working dir for secondary rocksdb logs", [&](td::Slice fname) { 40 | working_dir = fname.str(); 41 | }); 42 | 43 | p.add_checked_option('t', "threads", "Scheduler threads (default: 7)", [&](td::Slice fname) { 44 | int v; 45 | try { 46 | v = std::stoi(fname.str()); 47 | } catch (...) { 48 | return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: not a number"); 49 | } 50 | threads = v; 51 | return td::Status::OK(); 52 | }); 53 | 54 | p.add_option('\0', "redis", "Redis URI (default: 'tcp://127.0.0.1:6379')", [&](td::Slice fname) { 55 | redis_dsn = fname.str(); 56 | }); 57 | 58 | p.add_option('\0', "redis-queue", "Redis queue name for input msgs", [&](td::Slice fname) { 59 | redis_queue_name = fname.str(); 60 | }); 61 | 62 | auto S = p.run(argc, argv); 63 | if (S.is_error()) { 64 | LOG(ERROR) << "failed to parse options: " << S.move_as_error(); 65 | std::_Exit(2); 66 | } 67 | 68 | if (db_root.size() == 0) { 69 | LOG(ERROR) << "'--db' option missing"; 70 | std::_Exit(2); 71 | } 72 | 73 | if (working_dir.size() == 0) { 74 | working_dir = PSTRING() << "/tmp/index_worker_" << getpid(); 75 | LOG(WARNING) << "Working dir not specified, using " << working_dir; 76 | } 77 | if (redis_queue_name.size() == 0) { 78 | LOG(ERROR) << "'--redis-queue' option missing"; 79 | std::_Exit(2); 80 | } 81 | if (redis_dsn.size() == 0) { 82 | LOG(ERROR) << "'--redis' option missing"; 83 | std::_Exit(2); 84 | } 85 | 86 | td::actor::Scheduler scheduler({threads}); 87 | td::actor::ActorOwn db_scanner; 88 | td::actor::ActorOwn insert_manager; 89 | 90 | scheduler.run_in_context([&] { 91 | db_scanner = td::actor::create_actor("scanner", db_root, dbs_secondary, working_dir, 0.5); 92 | insert_manager = td::actor::create_actor("OnDemandRedisInsertManager", redis_dsn); 93 | td::actor::create_actor("integritychecker", db_scanner.get(), insert_manager.get(), redis_dsn, redis_queue_name).release(); 94 | }); 95 | 96 | scheduler.run(); 97 | 98 | return 0; 99 | } 100 | -------------------------------------------------------------------------------- /tondb-scanner/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.16) 2 | 3 | add_library(tondb-scanner STATIC 4 | src/InsertManager.cpp 5 | src/InsertManagerBase.cpp 6 | src/DbScanner.cpp 7 | src/DataParser.cpp 8 | src/TraceAssembler.cpp 9 | src/EventProcessor.cpp 10 | src/queue_state.cpp 11 | src/parse_token_data.cpp 12 | src/convert-utils.cpp 13 | src/tokens-tlb.cpp 14 | src/Statistics.cpp 15 | src/smc-interfaces/Tokens.cpp 16 | src/smc-interfaces/NftSale.cpp 17 | src/smc-interfaces/execute-smc.cpp 18 | ) 19 | 20 | target_include_directories(tondb-scanner 21 | PUBLIC src/ 22 | ) 23 | 24 | target_compile_features(tondb-scanner PRIVATE cxx_std_20) 25 | target_link_libraries(tondb-scanner PUBLIC validator-disk ton_validator tddb smc-envelope msgpack-cxx) 26 | 27 | set(TLB_TOKENS 28 | ${CMAKE_CURRENT_SOURCE_DIR}/src/tokens-tlb.cpp 29 | ${CMAKE_CURRENT_SOURCE_DIR}/src/tokens-tlb.h 30 | ) 31 | 32 | add_custom_command( 33 | WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src 34 | COMMAND tlbc -o tokens-tlb -n tokens::gen -z tlb/tokens.tlb 35 | COMMENT "Generate tokes tlb source files" 36 | OUTPUT ${TLB_TOKENS} 37 | DEPENDS tlbc src/tlb/tokens.tlb 38 | ) 39 | 40 | add_custom_target(tlb_generate_tokens DEPENDS ${TLB_TOKENS}) 41 | add_dependencies(tondb-scanner tlb_generate_tokens) 42 | 43 | set(TONDB_SCANNER_TEST_SOURCE 44 | ${CMAKE_CURRENT_SOURCE_DIR}/test/tests.cpp 45 | PARENT_SCOPE 46 | ) 47 | 48 | add_executable(test-tondb test/tests.cpp ${TONDB_SCANNER_TEST_SOURCE}) 49 | target_include_directories(test-tondb 50 | PUBLIC external/ton 51 | PUBLIC src/ 52 | ) 53 | 54 | target_link_libraries(test-tondb PRIVATE tdutils tdactor smc-envelope tondb-scanner ton_block) 55 | add_dependencies(test-tondb test-tdutils) -------------------------------------------------------------------------------- /tondb-scanner/src/DataParser.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "IndexData.h" 3 | 4 | 5 | class ParseQuery: public td::actor::Actor { 6 | private: 7 | const int mc_seqno_; 8 | MasterchainBlockDataState mc_block_; 9 | std::shared_ptr cell_db_reader_; 10 | ParsedBlockPtr result; 11 | td::Promise promise_; 12 | public: 13 | ParseQuery(int mc_seqno, MasterchainBlockDataState mc_block, std::shared_ptr cell_db_reader, td::Promise promise) 14 | : mc_seqno_(mc_seqno), mc_block_(std::move(mc_block)), cell_db_reader_(std::move(cell_db_reader)), result(std::make_shared()), promise_(std::move(promise)) {} 15 | 16 | void start_up() override; 17 | 18 | private: 19 | td::Status parse_impl(); 20 | 21 | schema::Block parse_block(const td::Ref& root_cell, const ton::BlockIdExt& blk_id, block::gen::Block::Record& blk, const block::gen::BlockInfo::Record& info, 22 | const block::gen::BlockExtra::Record& extra, const td::optional &mc_block); 23 | schema::MasterchainBlockShard parse_shard_state(const schema::Block& mc_block, const ton::BlockIdExt& shard_blk_id); 24 | static td::Result parse_currency_collection(td::Ref csr); 25 | td::Result parse_message(td::Ref msg_cell); 26 | td::Result parse_tr_storage_phase(vm::CellSlice& cs); 27 | td::Result parse_tr_credit_phase(vm::CellSlice& cs); 28 | td::Result parse_tr_compute_phase(vm::CellSlice& cs); 29 | td::Result parse_storage_used_short(vm::CellSlice& cs); 30 | td::Result parse_tr_action_phase(vm::CellSlice& cs); 31 | td::Result parse_tr_bounce_phase(vm::CellSlice& cs); 32 | td::Result parse_split_merge_info(td::Ref& cs); 33 | td::Result process_transaction_descr(vm::CellSlice& td_cs); 34 | 35 | struct AccountStateShort { 36 | td::Bits256 account_cell_hash; 37 | uint64_t last_transaction_lt; 38 | td::Bits256 last_transaction_hash; 39 | }; 40 | td::Result> parse_transactions(const ton::BlockIdExt& blk_id, const block::gen::Block::Record &block, 41 | const block::gen::BlockInfo::Record &info, const block::gen::BlockExtra::Record &extra, 42 | std::map &account_states); 43 | td::Result> parse_account_states_new(ton::WorkchainId workchain_id, uint32_t gen_utime, std::map &account_states); 44 | td::Result parse_none_account(td::Ref account_root, block::StdAddress address, uint32_t gen_utime, td::Bits256 last_trans_hash, uint64_t last_trans_lt); 45 | 46 | public: //TODO: refactor 47 | static td::Result parse_account(td::Ref account_root, uint32_t gen_utime, td::Bits256 last_trans_hash, uint64_t last_trans_lt); 48 | }; 49 | 50 | 51 | class ParseManager: public td::actor::Actor { 52 | public: 53 | ParseManager() {} 54 | 55 | void parse(int mc_seqno, MasterchainBlockDataState mc_block, std::shared_ptr cell_db_reader, td::Promise promise) { 56 | td::actor::create_actor("parsequery", mc_seqno, std::move(mc_block), cell_db_reader, std::move(promise)).release(); 57 | } 58 | }; -------------------------------------------------------------------------------- /tondb-scanner/src/DbScanner.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "validator/manager-disk.h" 4 | #include "validator/db/rootdb.hpp" 5 | 6 | #include "IndexData.h" 7 | #include "DataParser.h" 8 | #include "EventProcessor.h" 9 | 10 | enum ScannerMode { dbs_readonly, dbs_secondary }; 11 | 12 | class DbScanner: public td::actor::Actor { 13 | private: 14 | std::string db_root_; 15 | ScannerMode mode_; 16 | std::optional secondary_working_dir_; 17 | float catch_up_interval_; 18 | bool is_ready_ = false; 19 | ton::BlockSeqno last_known_seqno_{0}; 20 | 21 | td::actor::ActorOwn validator_manager_; 22 | td::actor::ActorOwn db_; 23 | public: 24 | DbScanner(std::string db_root, ScannerMode mode, std::optional secondary_working_dir = std::nullopt, float catch_up_interval = 1.0) 25 | : db_root_(db_root), mode_(mode), secondary_working_dir_(secondary_working_dir), catch_up_interval_(catch_up_interval) {} 26 | 27 | void start_up() override; 28 | void alarm() override; 29 | 30 | void fetch_seqno(std::uint32_t mc_seqno, td::Promise promise); 31 | void get_last_mc_seqno(td::Promise promise); 32 | void get_oldest_mc_seqno(td::Promise promise); 33 | void get_mc_block_handle(ton::BlockSeqno seqno, td::Promise promise); 34 | void get_cell_db_reader(td::Promise> promise); 35 | private: 36 | void catch_up_with_primary(); 37 | }; 38 | 39 | struct BlockIdExtHasher { 40 | std::size_t operator()(const ton::BlockIdExt& k) const { 41 | return std::hash()(k.to_str()); 42 | } 43 | }; 44 | -------------------------------------------------------------------------------- /tondb-scanner/src/InsertManager.cpp: -------------------------------------------------------------------------------- 1 | #include "InsertManager.h" 2 | -------------------------------------------------------------------------------- /tondb-scanner/src/InsertManager.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "td/actor/actor.h" 3 | #include "queue_state.h" 4 | #include "IndexData.h" 5 | 6 | 7 | enum ErrorCode { 8 | DB_ERROR = 500, 9 | CACHED_CODE_HASH_NO_ENTITY = 501, 10 | DATA_PARSING_ERROR = 502, 11 | GET_METHOD_WRONG_RESULT = 503, 12 | ADDITIONAL_CHECKS_FAILED = 504, 13 | EVENT_PARSING_ERROR = 505, 14 | 15 | CODE_HASH_NOT_FOUND = 600, 16 | ENTITY_NOT_FOUND = 601 17 | }; 18 | 19 | 20 | class InsertManagerInterface: public td::actor::Actor { 21 | public: 22 | virtual void insert(std::uint32_t mc_seqno, ParsedBlockPtr block_ds, td::Promise queued_promise, td::Promise inserted_promise) = 0; 23 | virtual void get_insert_queue_state(td::Promise promise) = 0; 24 | virtual void get_existing_seqnos(td::Promise> promise, std::int32_t from_seqno = 0, std::int32_t to_seqno = 0) = 0; 25 | 26 | // // helper template functions 27 | // template 28 | // void upsert_entity(T entity, td::Promise promise); 29 | }; 30 | -------------------------------------------------------------------------------- /tondb-scanner/src/InsertManagerBase.cpp: -------------------------------------------------------------------------------- 1 | #include "td/utils/StringBuilder.h" 2 | #include "InsertManagerBase.h" 3 | 4 | 5 | QueueState InsertTaskStruct::get_queue_state() { 6 | QueueState status = {1, static_cast(parsed_block_->blocks_.size()), 0, 0, static_cast(parsed_block_->traces_.size())}; 7 | for(const auto& blk : parsed_block_->blocks_) { 8 | status.txs_ += blk.transactions.size(); 9 | for(const auto& tx : blk.transactions) { 10 | status.msgs_ += tx.out_msgs.size() + (tx.in_msg ? 1 : 0); 11 | } 12 | } 13 | return status; 14 | } 15 | 16 | 17 | void InsertManagerBase::start_up() { 18 | LOG(INFO) << "InsertManagerBase::start_up called"; 19 | alarm_timestamp() = td::Timestamp::in(1.0); 20 | } 21 | 22 | void InsertManagerBase::alarm() { 23 | alarm_timestamp() = td::Timestamp::in(1.0); 24 | td::actor::send_closure(actor_id(this), &InsertManagerBase::schedule_next_insert_batches, false); 25 | } 26 | 27 | void InsertManagerBase::print_info() { 28 | LOG(INFO) << "Insert manager(parallel=" << max_parallel_insert_actors_ 29 | << ", max_batch_mc_blocks=" << batch_size_.mc_blocks_ 30 | << ", max_batch_blocks=" << batch_size_.blocks_ 31 | << ", max_batch_txs=" << batch_size_.txs_ 32 | << ", max_batch_msgs=" << batch_size_.msgs_ 33 | << ")"; 34 | } 35 | 36 | 37 | void InsertManagerBase::insert(std::uint32_t mc_seqno, ParsedBlockPtr block_ds, td::Promise queued_promise, td::Promise inserted_promise) { 38 | auto task = InsertTaskStruct{mc_seqno, std::move(block_ds), std::move(inserted_promise)}; 39 | auto status_delta = task.get_queue_state(); 40 | insert_queue_.push(std::move(task)); 41 | queue_state_ += status_delta; 42 | queued_promise.set_result(queue_state_); 43 | } 44 | 45 | 46 | void InsertManagerBase::get_insert_queue_state(td::Promise promise) { 47 | promise.set_result(queue_state_); 48 | } 49 | 50 | 51 | bool InsertManagerBase::check_batch_size(QueueState &batch_state) 52 | { 53 | return batch_state < batch_size_; 54 | } 55 | 56 | void InsertManagerBase::schedule_next_insert_batches(bool full_batch = false) 57 | { 58 | while(!insert_queue_.empty() && (parallel_insert_actors_ < max_parallel_insert_actors_)) { 59 | if (check_batch_size(queue_state_) && full_batch) 60 | break; 61 | 62 | std::vector batch; 63 | QueueState batch_state{0, 0, 0, 0}; 64 | while(!insert_queue_.empty() && check_batch_size(batch_state)) { 65 | auto task = std::move(insert_queue_.front()); 66 | insert_queue_.pop(); 67 | 68 | QueueState loc_state = task.get_queue_state(); 69 | batch_state += loc_state; 70 | queue_state_ -= loc_state; 71 | batch.push_back(std::move(task)); 72 | } 73 | auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R){ 74 | if(R.is_error()) { 75 | LOG(ERROR) << "Failed to insert batch: " << R.move_as_error(); 76 | } 77 | td::actor::send_closure(SelfId, &InsertManagerBase::insert_batch_finished); 78 | }); 79 | 80 | ++parallel_insert_actors_; 81 | LOG(INFO) << "Inserting batch[mb=" << batch_state.mc_blocks_ 82 | << ", b=" << batch_state.blocks_ 83 | << ", txs=" << batch_state.txs_ 84 | << ", msgs=" << batch_state.msgs_ 85 | << ", traces=" << batch_state.traces_ << "]"; 86 | create_insert_actor(std::move(batch), std::move(P)); 87 | } 88 | } 89 | 90 | void InsertManagerBase::insert_batch_finished() { 91 | --parallel_insert_actors_; 92 | td::actor::send_closure(actor_id(this), &InsertManagerBase::schedule_next_insert_batches, true); 93 | } 94 | -------------------------------------------------------------------------------- /tondb-scanner/src/InsertManagerBase.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include "InsertManager.h" 5 | 6 | 7 | struct InsertTaskStruct{ 8 | std::uint32_t mc_seqno_; 9 | ParsedBlockPtr parsed_block_; 10 | td::Promise promise_; 11 | 12 | QueueState get_queue_state(); 13 | }; 14 | 15 | class InsertManagerBase: public InsertManagerInterface { 16 | public: 17 | void set_parallel_inserts_actors(int value) { max_parallel_insert_actors_ = value; } 18 | void set_insert_batch_size(QueueState batch_size) { batch_size_ = batch_size; } 19 | void set_insert_blocks(int value) { batch_size_.blocks_ = value; } 20 | void set_insert_txs(int value) { batch_size_.txs_ = value; } 21 | void set_insert_msgs(int value) { batch_size_.msgs_ = value; } 22 | 23 | void print_info(); 24 | 25 | void start_up() override; 26 | void alarm() override; 27 | void insert(std::uint32_t mc_seqno, ParsedBlockPtr block_ds, td::Promise queued_promise, td::Promise inserted_promise) override; 28 | void get_insert_queue_state(td::Promise promise) override; 29 | 30 | virtual void create_insert_actor(std::vector insert_tasks, td::Promise promise) = 0; 31 | private: 32 | std::queue insert_queue_; 33 | QueueState queue_state_{0, 0, 0, 0}; 34 | 35 | td::int32 max_parallel_insert_actors_{32}; 36 | td::int32 parallel_insert_actors_{0}; 37 | 38 | QueueState batch_size_{10000, 10000, 100000, 100000}; 39 | 40 | bool check_batch_size(QueueState& batch_state); 41 | void schedule_next_insert_batches(bool full_batch); 42 | void insert_batch_finished(); 43 | }; 44 | -------------------------------------------------------------------------------- /tondb-scanner/src/Statistics.cpp: -------------------------------------------------------------------------------- 1 | #include "Statistics.h" 2 | #include 3 | #include 4 | 5 | 6 | void HistogramImpl::add(uint64_t duration, size_t count) { 7 | count_.fetch_add(count, std::memory_order_relaxed); 8 | sum_.fetch_add(duration, std::memory_order_relaxed); 9 | update_max(max_, duration); 10 | 11 | size_t index = bucketMapper.IndexForValue(duration); 12 | buckets_[index].fetch_add(count, std::memory_order_relaxed); 13 | } 14 | 15 | uint64_t HistogramImpl::get_count() const { 16 | return count_.load(std::memory_order_relaxed); 17 | } 18 | 19 | uint64_t HistogramImpl::get_sum() const { 20 | return sum_.load(std::memory_order_relaxed); 21 | } 22 | 23 | double HistogramImpl::compute_percentile(double percentile) const { 24 | std::lock_guard lock(mutex_); 25 | uint64_t total = get_count(); 26 | if (total == 0) return 0.0; 27 | 28 | double threshold = total * (percentile / 100.0); 29 | uint64_t accumulated = 0; 30 | size_t bucket_idx = 0; 31 | 32 | for (; bucket_idx < bucketCount(); ++bucket_idx) { 33 | uint64_t bucket_count = buckets_[bucket_idx].load(std::memory_order_relaxed); 34 | if (accumulated + bucket_count >= threshold) break; 35 | accumulated += bucket_count; 36 | } 37 | 38 | uint64_t lower = 0, upper = 0; 39 | if (bucket_idx == 0) { 40 | upper = bucketMapper.FirstValue(); 41 | } else if (bucket_idx < bucketCount()) { 42 | lower = bucketMapper.BucketLimit(bucket_idx - 1) + 1; 43 | upper = bucketMapper.BucketLimit(bucket_idx); 44 | } else { 45 | lower = bucketMapper.LastValue() + 1; 46 | upper = max_.load(std::memory_order_relaxed); 47 | } 48 | 49 | uint64_t actual_max = max_.load(std::memory_order_relaxed); 50 | if (upper > actual_max) { 51 | upper = actual_max; 52 | } 53 | 54 | uint64_t bucket_count = buckets_[bucket_idx].load(std::memory_order_relaxed); 55 | if (bucket_count == 0) return upper; 56 | 57 | double fraction = (threshold - accumulated) / bucket_count; 58 | return lower + fraction * (upper - lower); 59 | } 60 | 61 | uint64_t HistogramImpl::get_max() const { 62 | return max_.load(std::memory_order_relaxed); 63 | } 64 | 65 | void HistogramImpl::merge(const HistogramImpl &other) { 66 | std::lock_guard lock(mutex_); 67 | count_.fetch_add(other.count_.load(std::memory_order_relaxed), std::memory_order_relaxed); 68 | sum_.fetch_add(other.sum_.load(std::memory_order_relaxed), std::memory_order_relaxed); 69 | update_max(max_, other.max_.load(std::memory_order_relaxed)); 70 | update_min(min_, other.min_.load(std::memory_order_relaxed)); 71 | 72 | for (size_t i = 0; i < bucketCount(); ++i) { 73 | buckets_[i].fetch_add(other.buckets_[i].load(std::memory_order_relaxed), std::memory_order_relaxed); 74 | } 75 | } 76 | 77 | void HistogramImpl::reset() { 78 | std::lock_guard lock(mutex_); 79 | count_.store(0, std::memory_order_relaxed); 80 | sum_.store(0, std::memory_order_relaxed); 81 | max_.store(0, std::memory_order_relaxed); 82 | min_.store(0, std::memory_order_relaxed); 83 | 84 | for (size_t i = 0; i < bucketCount(); ++i) { 85 | buckets_[i].store(0, std::memory_order_relaxed); 86 | } 87 | } 88 | 89 | void HistogramImpl::update_max(std::atomic& max, uint64_t value) { 90 | uint64_t current = max.load(std::memory_order_relaxed); 91 | while (value > current && !max.compare_exchange_weak(current, value, std::memory_order_relaxed)); 92 | } 93 | 94 | void HistogramImpl::update_min(std::atomic& min, uint64_t value) { 95 | uint64_t current = min.load(std::memory_order_relaxed); 96 | while (value < current && !min.compare_exchange_weak(current, value, std::memory_order_relaxed)); 97 | } 98 | 99 | void Statistics::record_time(Histogram hist, uint64_t duration, uint32_t count) { 100 | if (count > 1) { 101 | // for batch events, we average the duration 102 | duration /= count; 103 | } 104 | per_core_stats_.get().histograms_[hist].add(duration, count); 105 | } 106 | 107 | void Statistics::record_count(Ticker ticker, uint64_t count) { 108 | per_core_stats_.get().tickers_[ticker].fetch_add(count, std::memory_order_relaxed); 109 | } 110 | 111 | std::string Statistics::generate_report_and_reset() { 112 | std::array aggTickers = {}; 113 | std::array aggHist; 114 | 115 | per_core_stats_.for_each([&](StatisticsData &data) { 116 | // Merge tickers. 117 | for (uint32_t i = 0; i < TICKERS_COUNT; ++i) { 118 | uint64_t value = data.tickers_[i].load(std::memory_order_relaxed); 119 | aggTickers[i] += value; 120 | data.tickers_[i].store(0, std::memory_order_relaxed); 121 | } 122 | 123 | // Merge histograms. 124 | for (uint32_t h = 0; h < HISTOGRAMS_COUNT; ++h) { 125 | aggHist[h].merge(data.histograms_[h]); 126 | data.histograms_[h].reset(); 127 | } 128 | }); 129 | 130 | std::ostringstream oss; 131 | oss << std::setprecision(3) << std::fixed; 132 | for (uint32_t i = 0; i < TICKERS_COUNT; ++i) { 133 | oss << ticker_names.at(i) << " COUNT : " << aggTickers[i] << std::endl; 134 | } 135 | 136 | for (uint32_t h = 0; h < HISTOGRAMS_COUNT; ++h) { 137 | oss << histogram_names.at(h) << " P50 : " << aggHist[h].compute_percentile(50.0) 138 | << " P95 : " << aggHist[h].compute_percentile(95.0) 139 | << " P99 : " << aggHist[h].compute_percentile(99.0) 140 | << " P100 : " << aggHist[h].get_max() 141 | << " COUNT : " << aggHist[h].get_count() 142 | << " SUM : " << aggHist[h].get_sum() << std::endl; 143 | } 144 | 145 | return oss.str(); 146 | } 147 | 148 | Statistics g_statistics; -------------------------------------------------------------------------------- /tondb-scanner/src/Statistics.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "td/utils/ThreadLocalStorage.h" 9 | 10 | constexpr size_t bucketCount() { 11 | size_t count = 2; // 1 and 2 12 | double bucket_val = 2.0; 13 | constexpr double max = static_cast(std::numeric_limits::max()); 14 | while ((bucket_val = 1.5 * bucket_val) <= max) { 15 | ++count; 16 | } 17 | return count; 18 | }; 19 | 20 | consteval auto bucketValues() { 21 | std::array bucketValues; 22 | bucketValues[0] = 1; 23 | bucketValues[1] = 2; 24 | size_t i = 1; 25 | double bucket_val = static_cast(bucketValues[i++]); 26 | constexpr double max = static_cast(std::numeric_limits::max()); 27 | while ((bucket_val = 1.5 * bucket_val) <= max) { 28 | bucketValues[i++] = static_cast(bucket_val); 29 | } 30 | if (bucketCount() != i) { 31 | throw "bucketValues() does not match bucketCount()"; 32 | } 33 | return bucketValues; 34 | } 35 | 36 | class HistogramBucketMapper 37 | { 38 | public: 39 | consteval HistogramBucketMapper() : bucketValues_(bucketValues()) {} 40 | 41 | size_t IndexForValue(const uint64_t value) const { 42 | auto beg = bucketValues_.begin(); 43 | auto end = bucketValues_.end(); 44 | if (value >= LastValue()) 45 | return end - beg - 1; // bucketValues_.size() - 1 46 | else 47 | return std::lower_bound(beg, end, value) - beg; 48 | } 49 | 50 | constexpr uint64_t LastValue() const { return bucketValues_.back(); } 51 | constexpr uint64_t FirstValue() const { return bucketValues_.front(); } 52 | constexpr uint64_t BucketCount() const { return bucketValues_.size(); } 53 | 54 | uint64_t BucketLimit(const size_t bucketNumber) const { 55 | assert(bucketNumber < BucketCount()); 56 | return bucketValues_[bucketNumber]; 57 | } 58 | 59 | private: 60 | std::invoke_result_t bucketValues_; 61 | }; 62 | 63 | constexpr HistogramBucketMapper bucketMapper; 64 | 65 | class HistogramImpl { 66 | public: 67 | HistogramImpl() = default; 68 | void add(uint64_t duration, size_t count = 1); 69 | uint64_t get_count() const; 70 | uint64_t get_sum() const; 71 | double compute_percentile(double percentile) const; 72 | uint64_t get_max() const; 73 | void merge(const HistogramImpl &other); 74 | void reset(); 75 | 76 | private: 77 | std::atomic count_{0}; 78 | std::atomic sum_{0}; 79 | std::atomic min_{0}; 80 | std::atomic max_{0}; 81 | std::vector bucket_limits_; 82 | std::atomic buckets_[bucketCount()]; 83 | mutable std::mutex mutex_; 84 | 85 | static void update_max(std::atomic &max, uint64_t value); 86 | static void update_min(std::atomic &min, uint64_t value); 87 | }; 88 | 89 | enum Ticker : uint32_t { 90 | SEQNO_FETCH_ERROR = 0, 91 | INSERT_CONFLICT, 92 | TICKERS_COUNT 93 | }; 94 | 95 | enum Histogram : uint32_t { 96 | PROCESS_SEQNO = 0, 97 | 98 | CATCH_UP_WITH_PRIMARY, 99 | FETCH_SEQNO, 100 | 101 | PARSE_SEQNO, 102 | PARSE_BLOCK, 103 | PARSE_TRANSACTION, 104 | PARSE_ACCOUNT_STATE, 105 | PARSE_CONFIG, 106 | 107 | TRACE_ASSEMBLER_GC_STATE, 108 | TRACE_ASSEMBLER_SAVE_STATE, 109 | TRACE_ASSEMBLER_PROCESS_BLOCK, 110 | 111 | DETECT_INTERFACES_SEQNO, 112 | DETECT_ACTIONS_SEQNO, 113 | 114 | INSERT_SEQNO, 115 | INSERT_BATCH_CONNECT, 116 | INSERT_BATCH_EXEC_DATA, 117 | INSERT_BATCH_EXEC_STATES, 118 | INSERT_BATCH_COMMIT, 119 | 120 | HISTOGRAMS_COUNT 121 | }; 122 | 123 | const std::unordered_map ticker_names = { 124 | {SEQNO_FETCH_ERROR, "indexer.seqno.fetch.error"}, 125 | {INSERT_CONFLICT, "indexer.insert.conflict"}, 126 | }; 127 | const std::unordered_map histogram_names = { 128 | {PROCESS_SEQNO, "indexer.index.seqno.millis"}, 129 | {CATCH_UP_WITH_PRIMARY, "indexer.catch_up_with_primary.millis"}, 130 | {FETCH_SEQNO, "indexer.fetch.seqno.millis"}, 131 | {PARSE_SEQNO, "indexer.parse.seqno.millis"}, 132 | {PARSE_BLOCK, "indexer.parse.block.millis"}, 133 | {PARSE_TRANSACTION, "indexer.parse.transaction.micros"}, 134 | {PARSE_ACCOUNT_STATE, "indexer.parse.account_state.micros"}, 135 | {PARSE_CONFIG, "indexer.parse.config.micros"}, 136 | {TRACE_ASSEMBLER_GC_STATE, "indexer.traceassembler.gc_state.micros"}, 137 | {TRACE_ASSEMBLER_SAVE_STATE, "indexer.traceassembler.save_state.micros"}, 138 | {TRACE_ASSEMBLER_PROCESS_BLOCK, "indexer.traceassembler.process_block.micros"}, 139 | {DETECT_INTERFACES_SEQNO, "indexer.interfaces.seqno.millis"}, 140 | {DETECT_ACTIONS_SEQNO, "indexer.actions.seqno.micros"}, 141 | {INSERT_SEQNO, "indexer.insert.seqno.millis"}, 142 | {INSERT_BATCH_CONNECT, "indexer.insert.batch.connect.millis"}, 143 | {INSERT_BATCH_EXEC_DATA, "indexer.insert.batch.exec_data.millis"}, 144 | {INSERT_BATCH_EXEC_STATES, "indexer.insert.batch.exec_states.millis"}, 145 | {INSERT_BATCH_COMMIT, "indexer.insert.batch.commit.millis"}, 146 | }; 147 | 148 | class Statistics { 149 | public: 150 | void record_time(Histogram hist, uint64_t duration, uint32_t count = 1); 151 | void record_count(Ticker ticker, uint64_t count = 1); 152 | std::string generate_report_and_reset(); 153 | 154 | private: 155 | struct StatisticsData { 156 | std::atomic_uint_fast64_t tickers_[TICKERS_COUNT] = {{0}}; 157 | HistogramImpl histograms_[HISTOGRAMS_COUNT]; 158 | }; 159 | 160 | td::ThreadLocalStorage per_core_stats_; 161 | }; 162 | 163 | extern Statistics g_statistics; 164 | -------------------------------------------------------------------------------- /tondb-scanner/src/TraceAssembler.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "msgpack-utils.h" 3 | #include "IndexData.h" 4 | 5 | 6 | struct Bits256Hasher { 7 | std::size_t operator()(const td::Bits256& k) const { 8 | std::size_t seed = 0; 9 | for(const auto& el : k.as_array()) { 10 | seed ^= std::hash{}(el) + 0x9e3779b9 + (seed << 6) + (seed >> 2); 11 | } 12 | return seed; 13 | } 14 | }; 15 | 16 | struct TraceEdgeImpl { 17 | using Type = schema::TraceEdge::Type; 18 | 19 | td::Bits256 trace_id; 20 | td::Bits256 msg_hash; 21 | uint64_t msg_lt; 22 | std::optional left_tx; 23 | std::optional right_tx; 24 | 25 | Type type{Type::ord}; 26 | bool incomplete{false}; 27 | bool broken{false}; 28 | 29 | // methods 30 | schema::TraceEdge to_schema() const; 31 | static TraceEdgeImpl from_schema(const schema::TraceEdge& edge); 32 | 33 | MSGPACK_DEFINE(trace_id, msg_hash, msg_lt, left_tx, right_tx, type, incomplete, broken); 34 | }; 35 | MSGPACK_ADD_ENUM(TraceEdgeImpl::Type); 36 | 37 | struct TraceImpl; 38 | using TraceImplPtr = std::shared_ptr; 39 | struct TraceImpl { 40 | using State = schema::Trace::State; 41 | 42 | td::Bits256 trace_id; 43 | 44 | // info 45 | std::optional external_hash; 46 | ton::BlockSeqno mc_seqno_start; 47 | ton::BlockSeqno mc_seqno_end; 48 | 49 | uint64_t start_lt; 50 | uint32_t start_utime; 51 | 52 | uint64_t end_lt{0}; 53 | uint32_t end_utime{0}; 54 | 55 | State state{State::pending}; 56 | 57 | size_t pending_edges{0}; 58 | size_t edges{0}; 59 | size_t nodes{0}; 60 | 61 | // methods 62 | TraceImpl() {} 63 | TraceImpl(ton::BlockSeqno seqno, const schema::Transaction &tx) : 64 | trace_id(tx.hash), external_hash((tx.in_msg.has_value() ? std::optional(tx.in_msg.value().hash) : std::nullopt)), 65 | mc_seqno_start(seqno), mc_seqno_end(seqno), start_lt(tx.lt), start_utime(tx.now), end_lt(tx.lt), end_utime(tx.now), 66 | state(State::pending), nodes(1) {} 67 | 68 | schema::Trace to_schema() const; 69 | static TraceImplPtr from_schema(const schema::Trace& trace); 70 | 71 | MSGPACK_DEFINE(trace_id, external_hash, mc_seqno_start, mc_seqno_end, start_lt, start_utime, end_lt, end_utime, state, pending_edges, edges, nodes); 72 | }; 73 | MSGPACK_ADD_ENUM(TraceImpl::State); 74 | 75 | 76 | class TraceAssembler: public td::actor::Actor { 77 | struct Task { 78 | ton::BlockSeqno seqno_; 79 | ParsedBlockPtr block_; 80 | td::Promise promise_; 81 | }; 82 | 83 | std::string db_path_; 84 | size_t gc_distance_; 85 | ton::BlockSeqno expected_seqno_{0}; 86 | std::map queue_; 87 | size_t broken_count_{0}; 88 | 89 | std::unordered_map pending_traces_; 90 | std::unordered_map pending_edges_; 91 | public: 92 | TraceAssembler(std::string db_path, size_t gc_distance); 93 | 94 | void assemble(ton::BlockSeqno mc_seqno, ParsedBlockPtr mc_block_, td::Promise promise); 95 | 96 | td::Result restore_state(ton::BlockSeqno expected_seqno); 97 | void set_expected_seqno(ton::BlockSeqno expected_seqno); 98 | void start_up() override; 99 | void alarm() override; 100 | private: 101 | void process_queue(); 102 | void process_block(ton::BlockSeqno seqno, ParsedBlockPtr block); 103 | void process_transaction(ton::BlockSeqno seqno, schema::Transaction& tx, std::vector& edges_found_, 104 | std::unordered_set& updated_traces_, std::unordered_set& updated_edges_); 105 | }; 106 | -------------------------------------------------------------------------------- /tondb-scanner/src/convert-utils.cpp: -------------------------------------------------------------------------------- 1 | #include "td/actor/actor.h" 2 | #include "vm/cells/Cell.h" 3 | #include "vm/cells/CellSlice.h" 4 | #include "vm/stack.hpp" 5 | #include "common/refcnt.hpp" 6 | #include "smc-envelope/SmartContract.h" 7 | #include "crypto/block/block-auto.h" 8 | #include "crypto/block/block-parse.h" 9 | #include "td/utils/base64.h" 10 | #include "convert-utils.h" 11 | 12 | 13 | td::Result convert::to_raw_address(td::Ref cs) { 14 | auto tag = block::gen::MsgAddress().get_tag(*cs); 15 | switch (tag) { 16 | case block::gen::MsgAddress::cons1: 17 | switch (block::gen::MsgAddressInt().get_tag(*cs)) { 18 | case block::gen::MsgAddressInt::addr_var: 19 | return "addr_var"; 20 | case block::gen::MsgAddressInt::addr_std: { 21 | block::gen::MsgAddressInt::Record_addr_std addr; 22 | if (!tlb::csr_unpack(cs, addr)) { 23 | return td::Status::Error("Failed to unpack MsgAddressInt"); 24 | } 25 | if (addr.anycast.not_null()) { 26 | if (addr.anycast->bit_at(0) == 1) { 27 | auto anycast_slice = vm::CellSlice(*addr.anycast); 28 | anycast_slice.advance(1); // skip maybe bit 29 | block::gen::Anycast::Record anycast; 30 | if (!tlb::unpack_exact(anycast_slice, anycast)) { 31 | return td::Status::Error("Failed to unpack Anycast"); 32 | } 33 | addr.address.bits().copy_from(anycast.rewrite_pfx->cbits(), anycast.depth); 34 | } 35 | } 36 | return std::to_string(addr.workchain_id) + ":" + addr.address.to_hex(); 37 | } 38 | default: 39 | return td::Status::Error("Failed to unpack MsgAddressInt"); 40 | } 41 | case block::gen::MsgAddress::cons2: 42 | switch (block::gen::MsgAddressExt().get_tag(*cs)) { 43 | case block::gen::MsgAddressExt::addr_none: 44 | return "addr_none"; 45 | case block::gen::MsgAddressExt::addr_extern: 46 | return "addr_extern"; 47 | default: 48 | return td::Status::Error("Failed to unpack MsgAddressExt"); 49 | } 50 | default: 51 | return td::Status::Error("Failed to unpack MsgAddress"); 52 | } 53 | } 54 | 55 | td::Result convert::to_std_address(td::Ref cs) { 56 | auto tag = block::gen::MsgAddress().get_tag(*cs); 57 | switch (tag) { 58 | case block::gen::MsgAddress::cons1: 59 | switch (block::gen::MsgAddressInt().get_tag(*cs)) { 60 | case block::gen::MsgAddressInt::addr_var: 61 | return td::Status::Error("addr_var is not std address"); 62 | case block::gen::MsgAddressInt::addr_std: { 63 | block::gen::MsgAddressInt::Record_addr_std addr; 64 | if (!tlb::csr_unpack(cs, addr)) { 65 | return td::Status::Error("Failed to unpack addr_std"); 66 | } 67 | if (addr.anycast.not_null()) { 68 | if (addr.anycast->bit_at(0) == 1) { 69 | auto anycast_slice = vm::CellSlice(*addr.anycast); 70 | anycast_slice.advance(1); // skip maybe bit 71 | block::gen::Anycast::Record anycast; 72 | if (!tlb::unpack_exact(anycast_slice, anycast)) { 73 | return td::Status::Error("Failed to unpack Anycast"); 74 | } 75 | addr.address.bits().copy_from(anycast.rewrite_pfx->cbits(), anycast.depth); 76 | } 77 | } 78 | return block::StdAddress(addr.workchain_id, addr.address); 79 | } 80 | default: 81 | return td::Status::Error("Failed to unpack MsgAddressInt"); 82 | } 83 | case block::gen::MsgAddress::cons2: 84 | return td::Status::Error("MsgAddressExt is not std address"); 85 | default: 86 | return td::Status::Error("Failed to unpack MsgAddress"); 87 | } 88 | } 89 | 90 | std::string convert::to_raw_address(block::StdAddress address) { 91 | return std::to_string(address.workchain) + ":" + address.addr.to_hex(); 92 | } 93 | 94 | td::Result> convert::to_bytes(td::Ref cell) { 95 | if (cell.is_null()) { 96 | return std::nullopt; 97 | } 98 | TRY_RESULT(boc, vm::std_boc_serialize(cell, vm::BagOfCells::Mode::WithCRC32C)); 99 | return td::base64_encode(boc.as_slice().str()); 100 | } 101 | -------------------------------------------------------------------------------- /tondb-scanner/src/convert-utils.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | 4 | namespace convert { 5 | td::Result to_raw_address(td::Ref cs); 6 | 7 | std::string to_raw_address(block::StdAddress address); 8 | 9 | td::Result to_std_address(td::Ref cs); 10 | 11 | td::Result> to_bytes(td::Ref cell); 12 | } 13 | -------------------------------------------------------------------------------- /tondb-scanner/src/msgpack-utils.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include "crypto/block/block-auto.h" 3 | #include "crypto/block/block-parse.h" 4 | 5 | namespace msgpack { 6 | MSGPACK_API_VERSION_NAMESPACE(MSGPACK_DEFAULT_API_NS) { 7 | namespace adaptor { 8 | 9 | template<> 10 | struct pack { 11 | template 12 | msgpack::packer& operator()(msgpack::packer& o, const block::StdAddress& v) const { 13 | std::string addr = std::to_string(v.workchain) + ":" + v.addr.to_hex(); 14 | o.pack(addr); 15 | return o; 16 | } 17 | }; 18 | 19 | template <> 20 | struct convert { 21 | msgpack::object const& operator()(msgpack::object const& o, block::StdAddress& v) const { 22 | if (o.type != msgpack::type::STR) throw msgpack::type_error(); 23 | std::string addr = o.as(); 24 | if (!v.parse_addr(addr)) throw std::runtime_error("Failed to deserialize block::StdAddress"); 25 | return o; 26 | } 27 | }; 28 | 29 | template<> 30 | struct pack { 31 | template 32 | msgpack::packer& operator()(msgpack::packer& o, const td::Bits256& v) const { 33 | o.pack_bin(32); 34 | o.pack_bin_body((const char*)v.data(), 32); 35 | return o; 36 | } 37 | }; 38 | 39 | template <> 40 | struct convert { 41 | msgpack::object const& operator()(msgpack::object const& o, td::Bits256& v) const { 42 | if (o.type == msgpack::type::BIN || o.via.bin.size == 32) { 43 | std::memcpy(v.data(), o.via.bin.ptr, 32); 44 | return o; 45 | } 46 | // Previously td::Bits256 was serialized as a hex string. Keep this for compatibility. 47 | if (o.type == msgpack::type::STR) { 48 | std::string hex = o.as(); 49 | if (v.from_hex(hex) < 0) throw std::runtime_error("Failed to deserialize td::Bits256"); 50 | return o; 51 | } 52 | throw msgpack::type_error(); 53 | } 54 | }; 55 | 56 | template<> 57 | struct pack { 58 | template 59 | msgpack::packer& operator()(msgpack::packer& o, const td::RefInt256& v) const { 60 | o.pack(v->to_dec_string()); 61 | return o; 62 | } 63 | }; 64 | 65 | template <> 66 | struct convert { 67 | msgpack::object const& operator()(msgpack::object const& o, td::RefInt256& v) const { 68 | throw std::runtime_error("Deserializion of td::RefInt256 is not implemented"); 69 | return o; 70 | } 71 | }; 72 | 73 | } // namespace adaptor 74 | } // MSGPACK_API_VERSION_NAMESPACE(MSGPACK_DEFAULT_API_NS) 75 | } // namespace msgpack 76 | -------------------------------------------------------------------------------- /tondb-scanner/src/parse_token_data.cpp: -------------------------------------------------------------------------------- 1 | #include "InsertManager.h" 2 | #include "tokens-tlb.h" 3 | #include "common/checksum.h" 4 | 5 | 6 | td::Result parse_snake_data(td::Ref data) { 7 | size_t bsize = 1024 * 8; 8 | unsigned char buffer[bsize]; 9 | td::BitPtr bw{buffer}; 10 | 11 | while (true) { 12 | if (bw.offs + data->size() > bsize * 8) { 13 | break; // buffer overflow 14 | } 15 | bw.concat(data->prefetch_bits(data->size())); 16 | 17 | auto cell = data->prefetch_ref(); 18 | if (cell.is_null()) { 19 | break; 20 | } 21 | data = vm::load_cell_slice_ref(cell); 22 | } 23 | 24 | if (!bw.byte_aligned()) { 25 | return td::Status::Error("Not byte aligned"); 26 | } 27 | 28 | return std::string(buffer, bw.get_byte_ptr()); 29 | } 30 | 31 | td::Result parse_chunks_data(td::Ref data) { 32 | try { 33 | vm::Dictionary dict(data, 32); 34 | 35 | size_t bsize = 1024 * 8; 36 | unsigned char buffer[bsize]; 37 | td::BitPtr bw{buffer}; 38 | 39 | uint c = 0; 40 | while (dict.uint_key_exists(c)) { 41 | auto value = dict.lookup_ref(td::BitArray<32>(c)); 42 | if (value.not_null()) { 43 | auto data = vm::load_cell_slice_ref(value); 44 | if (bw.offs + data->size() > bsize * 8) { 45 | break; // buffer overflow 46 | } 47 | 48 | bw.concat(data->prefetch_bits(data->size())); 49 | } 50 | c++; 51 | } 52 | 53 | if (!bw.byte_aligned()) { 54 | return td::Status::Error("Not byte aligned"); 55 | } 56 | 57 | return std::string(buffer, bw.get_byte_ptr()); 58 | } catch (vm::VmError& err) { 59 | return td::Status::Error(PSLICE() << "Exception while parsing chunks data: " << err.get_msg()); 60 | } 61 | } 62 | 63 | td::Result parse_content_data(td::Ref cs) { 64 | switch (tokens::gen::t_ContentData.check_tag(*cs)) { 65 | case tokens::gen::ContentData::snake: { 66 | tokens::gen::ContentData::Record_snake snake_record; 67 | if (!tlb::csr_unpack(cs, snake_record)) { 68 | return td::Status::Error("Failed to unpack snake token data"); 69 | } 70 | return parse_snake_data(snake_record.data); 71 | } 72 | case tokens::gen::ContentData::chunks: { 73 | tokens::gen::ContentData::Record_chunks chunks_record; 74 | if (!tlb::csr_unpack(cs, chunks_record)) { 75 | return td::Status::Error("Failed to unpack chunks token data"); 76 | } 77 | return parse_chunks_data(chunks_record.data); 78 | } 79 | default: 80 | return td::Status::Error("Unknown content data"); 81 | } 82 | } 83 | 84 | td::Result> parse_token_data(td::Ref cell) { 85 | static std::string attributes[] = {"uri", "name", "description", "image", "image_data", "symbol", "decimals", "amount_style", "render_type"}; 86 | 87 | try { 88 | auto cs = vm::load_cell_slice_ref(cell); 89 | switch (tokens::gen::t_FullContent.check_tag(*cs)) { 90 | case tokens::gen::FullContent::offchain: { 91 | tokens::gen::FullContent::Record_offchain offchain_record; 92 | if (!tlb::csr_unpack(cs, offchain_record)) { 93 | return td::Status::Error("Failed to unpack offchain token data"); 94 | } 95 | auto uri_r = parse_snake_data(offchain_record.uri); 96 | if (uri_r.is_error()) { 97 | return uri_r.move_as_error(); 98 | } 99 | auto uri = uri_r.move_as_ok(); 100 | if (!td::check_utf8(uri)) { 101 | return td::Status::Error("Invalid uri"); 102 | } 103 | return std::map{{"uri", std::move(uri)}}; 104 | } 105 | case tokens::gen::FullContent::onchain: { 106 | tokens::gen::FullContent::Record_onchain onchain_record; 107 | if (!tlb::csr_unpack(cs, onchain_record)) { 108 | return td::Status::Error("Failed to unpack onchain token data"); 109 | } 110 | 111 | vm::Dictionary dict(onchain_record.data, 256); 112 | std::map res; 113 | 114 | for (auto attr : attributes) { 115 | auto value_cs = dict.lookup(td::sha256_bits256(attr)); 116 | if (value_cs.not_null()) { 117 | // TEP-64 standard requires that all attributes are stored in a dictionary with a single reference as a value: 118 | // onchain#00 data:(HashmapE 256 ^ContentData) = FullContent; 119 | // however, some contracts store attributes as a direct value: 120 | // onchain#00 data:(HashmapE 256 ContentData) = FullContent; 121 | // so we need to handle both cases. 122 | if (value_cs->size() == 0 && value_cs->size_refs() == 1) { 123 | value_cs = vm::load_cell_slice_ref(value_cs->prefetch_ref()); 124 | } 125 | 126 | auto attr_data_r = parse_content_data(value_cs); 127 | if (attr_data_r.is_error()) { 128 | LOG(ERROR) << "Failed to parse attribute " << attr << ": " << attr_data_r.move_as_error().message(); 129 | continue; 130 | } 131 | auto attr_data = attr_data_r.move_as_ok(); 132 | if (attr == "image_data") { 133 | res[attr] = td::base64_encode(attr_data); 134 | } else { 135 | if (!td::check_utf8(attr_data)) { 136 | LOG(ERROR) << "Invalid data (not utf8) in attribute " << attr; 137 | continue; 138 | } 139 | res[attr] = std::move(attr_data); 140 | } 141 | } 142 | } 143 | return res; 144 | } 145 | default: 146 | return td::Status::Error("Unknown token data type"); 147 | } 148 | } catch (vm::VmError& err) { 149 | return td::Status::Error(PSLICE() << "Failed to parse token data: " << err.get_msg()); 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /tondb-scanner/src/parse_token_data.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "td/utils/Status.h" 4 | #include "vm/cells/Cell.h" 5 | #include "common/refcnt.hpp" 6 | 7 | 8 | td::Result> parse_token_data(td::Ref cell); 9 | -------------------------------------------------------------------------------- /tondb-scanner/src/queue_state.cpp: -------------------------------------------------------------------------------- 1 | #include "queue_state.h" 2 | 3 | 4 | QueueState operator+(QueueState l, const QueueState &r) { 5 | return QueueState{l.mc_blocks_ + r.mc_blocks_, l.blocks_ + r.blocks_, l.txs_ + r.txs_, l.msgs_ + r.msgs_, l.traces_ + r.traces_}; 6 | } 7 | 8 | QueueState operator-(QueueState l, const QueueState &r) { 9 | return QueueState{l.mc_blocks_ - r.mc_blocks_, l.blocks_ - r.blocks_, l.txs_ - r.txs_, l.msgs_ - r.msgs_, l.traces_ - r.traces_}; 10 | } 11 | 12 | QueueState& QueueState::operator+=(const QueueState& r) { 13 | mc_blocks_ += r.mc_blocks_; 14 | blocks_ += r.blocks_; 15 | txs_ += r.txs_; 16 | msgs_ += r.msgs_; 17 | traces_ += r.traces_; 18 | return *this; 19 | } 20 | 21 | QueueState& QueueState::operator-=(const QueueState& r) { 22 | mc_blocks_ -= r.mc_blocks_; 23 | blocks_ -= r.blocks_; 24 | txs_ -= r.txs_; 25 | msgs_ -= r.msgs_; 26 | traces_ -= r.traces_; 27 | return *this; 28 | } 29 | -------------------------------------------------------------------------------- /tondb-scanner/src/queue_state.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | struct QueueState { 5 | std::int32_t mc_blocks_{0}; 6 | std::int32_t blocks_{0}; 7 | std::int32_t txs_{0}; 8 | std::int32_t msgs_{0}; 9 | std::int32_t traces_{0}; 10 | 11 | QueueState& operator+=(const QueueState& r); 12 | QueueState& operator-=(const QueueState& r); 13 | 14 | friend QueueState operator+(QueueState l, const QueueState& r); 15 | friend QueueState operator-(QueueState l, const QueueState& r); 16 | 17 | friend inline bool operator==(const QueueState& l, const QueueState& r) { 18 | return (l.mc_blocks_ == r.mc_blocks_) && (l.blocks_ == r.blocks_) && (l.txs_ == r.txs_) && (l.msgs_ == r.msgs_); 19 | } 20 | friend inline bool operator<(const QueueState& l, const QueueState& r) { 21 | return (l.mc_blocks_ <= r.mc_blocks_) && (l.blocks_ <= r.blocks_) && (l.txs_ <= r.txs_) && (l.msgs_ <= r.msgs_); 22 | } 23 | }; 24 | -------------------------------------------------------------------------------- /tondb-scanner/src/smc-interfaces/InterfacesDetector.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include "td/actor/MultiPromise.h" 5 | #include 6 | #include "Tokens.h" 7 | #include "NftSale.h" 8 | 9 | template 10 | class InterfacesDetector: public td::actor::Actor { 11 | public: 12 | using DetectedInterface = std::variant; 13 | 14 | InterfacesDetector(block::StdAddress address, 15 | td::Ref code_cell, 16 | td::Ref data_cell, 17 | AllShardStates shard_states, 18 | std::shared_ptr config, 19 | td::Promise> promise) : 20 | address_(std::move(address)), code_cell_(std::move(code_cell)), data_cell_(std::move(data_cell)), 21 | shard_states_(std::move(shard_states)), config_(std::move(config)), promise_(std::move(promise)) { 22 | found_interfaces_ = std::make_shared>(); 23 | } 24 | 25 | void start_up() override { 26 | auto P = td::PromiseCreator::lambda([&, SelfId=actor_id(this)](td::Result res) mutable { 27 | td::actor::send_closure(SelfId, &InterfacesDetector::finish, std::move(res)); 28 | }); 29 | 30 | td::MultiPromise mp; 31 | auto ig = mp.init_guard(); 32 | ig.add_promise(std::move(P)); 33 | 34 | start_detection(ig); 35 | } 36 | private: 37 | block::StdAddress address_; 38 | td::Ref code_cell_; 39 | td::Ref data_cell_; 40 | AllShardStates shard_states_; 41 | std::shared_ptr config_; 42 | 43 | std::shared_ptr> found_interfaces_; 44 | td::Promise> promise_; 45 | 46 | template 47 | void start_detection(td::MultiPromise::InitGuard& ig) { 48 | detect_interface(ig.get_promise()); 49 | 50 | if constexpr (sizeof...(RemainingDetectors) > 0) { 51 | start_detection(ig); 52 | } 53 | } 54 | 55 | template 56 | void detect_interface(td::Promise promise) { 57 | auto P = td::PromiseCreator::lambda([&, SelfId = actor_id(this), promise = std::move(promise)](td::Result data) mutable { 58 | if (data.is_ok()) { 59 | LOG(DEBUG) << "Detected interface " << typeid(typename Detector::Result).name() << " for " << address_; 60 | send_lambda(SelfId, [this, data = data.move_as_ok(), promise = std::move(promise)]() mutable { 61 | found_interfaces_->push_back(data); 62 | promise.set_value(td::Unit()); 63 | }); 64 | } else { 65 | promise.set_value(td::Unit()); 66 | } 67 | }); 68 | td::actor::create_actor(td::Slice(typeid(Detector).name()), address_, code_cell_, data_cell_, shard_states_, config_, std::move(P)).release(); 69 | } 70 | 71 | void finish(td::Result res) { 72 | if (res.is_error()) { 73 | promise_.set_error(res.move_as_error_prefix("Failed to detect interfaces: ")); 74 | } else { 75 | promise_.set_value(std::move(*found_interfaces_)); 76 | } 77 | stop(); 78 | } 79 | }; 80 | 81 | -------------------------------------------------------------------------------- /tondb-scanner/src/smc-interfaces/NftSale.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | 6 | using AllShardStates = std::vector>; 7 | 8 | class GetGemsNftFixPriceSale: public td::actor::Actor { 9 | public: 10 | struct Result { 11 | block::StdAddress address; 12 | bool is_complete; 13 | uint32_t created_at; 14 | block::StdAddress marketplace_address; 15 | block::StdAddress nft_address; 16 | std::optional nft_owner_address; 17 | td::RefInt256 full_price; 18 | block::StdAddress marketplace_fee_address; 19 | td::RefInt256 marketplace_fee; 20 | block::StdAddress royalty_address; 21 | td::RefInt256 royalty_amount; 22 | }; 23 | 24 | GetGemsNftFixPriceSale(block::StdAddress address, 25 | td::Ref code_cell, 26 | td::Ref data_cell, 27 | AllShardStates shard_states, 28 | std::shared_ptr config, 29 | td::Promise promise); 30 | 31 | void start_up() override; 32 | 33 | private: 34 | block::StdAddress address_; 35 | td::Ref code_cell_; 36 | td::Ref data_cell_; 37 | AllShardStates shard_states_; 38 | std::shared_ptr config_; 39 | td::Promise promise_; 40 | }; 41 | 42 | class GetGemsNftAuction: public td::actor::Actor { 43 | public: 44 | struct Result { 45 | block::StdAddress address; 46 | bool end; 47 | uint32_t end_time; 48 | block::StdAddress mp_addr; 49 | block::StdAddress nft_addr; 50 | std::optional nft_owner; 51 | td::RefInt256 last_bid; 52 | std::optional last_member; 53 | uint32_t min_step; 54 | block::StdAddress mp_fee_addr; 55 | uint32_t mp_fee_factor, mp_fee_base; 56 | block::StdAddress royalty_fee_addr; 57 | uint32_t royalty_fee_factor, royalty_fee_base; 58 | td::RefInt256 max_bid; 59 | td::RefInt256 min_bid; 60 | uint32_t created_at; 61 | uint32_t last_bid_at; 62 | bool is_canceled; 63 | }; 64 | 65 | GetGemsNftAuction(block::StdAddress address, 66 | td::Ref code_cell, 67 | td::Ref data_cell, 68 | AllShardStates shard_states, 69 | std::shared_ptr config, 70 | td::Promise promise); 71 | 72 | void start_up() override; 73 | 74 | private: 75 | block::StdAddress address_; 76 | td::Ref code_cell_; 77 | td::Ref data_cell_; 78 | AllShardStates shard_states_; 79 | std::shared_ptr config_; 80 | td::Promise promise_; 81 | }; 82 | -------------------------------------------------------------------------------- /tondb-scanner/src/smc-interfaces/Tokens.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | 6 | using AllShardStates = std::vector>; 7 | 8 | class JettonWalletDetectorR: public td::actor::Actor { 9 | public: 10 | struct Result { 11 | td::RefInt256 balance; 12 | block::StdAddress address; 13 | block::StdAddress owner; 14 | block::StdAddress jetton; 15 | std::optional mintless_is_claimed; 16 | }; 17 | 18 | JettonWalletDetectorR(block::StdAddress address, 19 | td::Ref code_cell, 20 | td::Ref data_cell, 21 | AllShardStates shard_states, 22 | std::shared_ptr config, 23 | td::Promise promise); 24 | 25 | void start_up() override; 26 | 27 | private: 28 | void verify_with_master(td::Ref master_code, td::Ref master_data, Result jetton_wallet_data); 29 | 30 | block::StdAddress address_; 31 | td::Ref code_cell_; 32 | td::Ref data_cell_; 33 | AllShardStates shard_states_; 34 | std::shared_ptr config_; 35 | td::Promise promise_; 36 | }; 37 | 38 | class JettonMasterDetectorR: public td::actor::Actor { 39 | public: 40 | struct Result { 41 | block::StdAddress address; 42 | td::RefInt256 total_supply; 43 | bool mintable; 44 | std::optional admin_address; 45 | std::optional> jetton_content; 46 | vm::CellHash jetton_wallet_code_hash; 47 | }; 48 | 49 | JettonMasterDetectorR(block::StdAddress address, 50 | td::Ref code_cell, 51 | td::Ref data_cell, 52 | AllShardStates shard_states, 53 | std::shared_ptr config, 54 | td::Promise promise); 55 | 56 | void start_up() override; 57 | 58 | private: 59 | block::StdAddress address_; 60 | td::Ref code_cell_; 61 | td::Ref data_cell_; 62 | AllShardStates shard_states_; 63 | std::shared_ptr config_; 64 | td::Promise promise_; 65 | }; 66 | 67 | class NftItemDetectorR: public td::actor::Actor { 68 | public: 69 | struct Result { 70 | struct DNSEntry { 71 | std::string domain; 72 | std::optional wallet; 73 | std::optional next_resolver; 74 | std::optional site_adnl; 75 | std::optional storage_bag_id; 76 | }; 77 | block::StdAddress address; 78 | bool init; 79 | td::RefInt256 index; 80 | std::optional collection_address; 81 | std::optional owner_address; 82 | std::optional> content; 83 | std::optional dns_entry; 84 | }; 85 | 86 | static bool is_testnet; 87 | 88 | NftItemDetectorR(block::StdAddress address, 89 | td::Ref code_cell, 90 | td::Ref data_cell, 91 | AllShardStates shard_states, 92 | std::shared_ptr config, 93 | td::Promise promise); 94 | 95 | void start_up() override; 96 | 97 | private: 98 | void got_collection(Result item_data, td::Ref ind_content, td::Ref collection_code, td::Ref collection_data); 99 | td::Status verify_with_collection(block::StdAddress collection_address, td::Ref collection_code, td::Ref collection_data, td::RefInt256 index); 100 | td::Result> get_content(td::RefInt256 index, td::Ref ind_content, block::StdAddress collection_address, 101 | td::Ref collection_code, td::Ref collection_data); 102 | void process_domain_and_dns_data(const block::StdAddress& root_address, const std::function()>& get_domain_function, Result& item_data); 103 | td::Result get_dns_entry_data(); 104 | td::Result get_ton_domain(); 105 | td::Result get_t_me_domain(); 106 | static block::StdAddress get_dot_ton_dns_root_addr(); 107 | static std::optional dot_t_dot_me_dns_root_addr(); 108 | 109 | block::StdAddress address_; 110 | td::Ref code_cell_; 111 | td::Ref data_cell_; 112 | AllShardStates shard_states_; 113 | std::shared_ptr config_; 114 | td::Promise promise_; 115 | 116 | td::Ref ind_content_; 117 | }; 118 | 119 | class NftCollectionDetectorR: public td::actor::Actor { 120 | public: 121 | struct Result { 122 | block::StdAddress address; 123 | td::RefInt256 next_item_index; 124 | std::optional owner_address; 125 | std::optional> collection_content; 126 | }; 127 | 128 | NftCollectionDetectorR(block::StdAddress address, 129 | td::Ref code_cell, 130 | td::Ref data_cell, 131 | AllShardStates shard_states, 132 | std::shared_ptr config, 133 | td::Promise promise); 134 | 135 | void start_up() override; 136 | 137 | private: 138 | block::StdAddress address_; 139 | td::Ref code_cell_; 140 | td::Ref data_cell_; 141 | AllShardStates shard_states_; 142 | std::shared_ptr config_; 143 | td::Promise promise_; 144 | }; 145 | -------------------------------------------------------------------------------- /tondb-scanner/src/smc-interfaces/execute-smc.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "execute-smc.h" 3 | 4 | 5 | td::Result> execute_smc_method(const block::StdAddress& address, td::Ref code, td::Ref data, 6 | std::shared_ptr config, const std::string& method_id, 7 | std::vector input) { 8 | ton::SmartContract smc({code, data}); 9 | ton::SmartContract::Args args; 10 | args.set_libraries(vm::Dictionary(config->get_libraries_root(), 256)); 11 | args.set_config(config); 12 | args.set_now(td::Time::now()); 13 | args.set_address(std::move(address)); 14 | args.set_stack(std::move(input)); 15 | 16 | args.set_method_id(method_id); 17 | auto res = smc.run_get_method(args); 18 | 19 | if (!res.success) { 20 | return td::Status::Error(method_id + " failed"); 21 | } 22 | 23 | auto stack = res.stack->extract_contents(); 24 | 25 | return stack; 26 | } -------------------------------------------------------------------------------- /tondb-scanner/src/smc-interfaces/execute-smc.h: -------------------------------------------------------------------------------- 1 | #include "smc-envelope/SmartContract.h" 2 | 3 | td::Result> execute_smc_method(const block::StdAddress& address, 4 | td::Ref code, td::Ref data, 5 | std::shared_ptr config, const std::string& method_id, 6 | std::vector input); 7 | 8 | template 9 | td::Result> execute_smc_method(const block::StdAddress& address, td::Ref code, td::Ref data, 10 | std::shared_ptr config, const std::string& method_id, 11 | std::vector input, const std::array& expected_types) { 12 | TRY_RESULT(stack, execute_smc_method(address, code, data, config, method_id, std::move(input))); 13 | 14 | if (stack.size() != expected_types.size()) { 15 | return td::Status::Error(method_id + " unexpected result stack depth"); 16 | } 17 | for (size_t i = 0; i < expected_types.size(); i++) { 18 | if (stack[i].type() != expected_types[i]) { 19 | return td::Status::Error(method_id + " unexpected stack entry type"); 20 | } 21 | } 22 | 23 | return stack; 24 | } 25 | -------------------------------------------------------------------------------- /tondb-scanner/src/tlb/tokens.tlb: -------------------------------------------------------------------------------- 1 | nothing$0 {X:Type} = Maybe X; 2 | just$1 {X:Type} value:X = Maybe X; 3 | left$0 {X:Type} {Y:Type} value:X = Either X Y; 4 | right$1 {X:Type} {Y:Type} value:Y = Either X Y; 5 | var_uint$_ {n:#} len:(#< n) value:(uint (len * 8)) 6 | = VarUInteger n; 7 | 8 | addr_none$00 = MsgAddressExt; 9 | addr_extern$01 len:(## 9) external_address:(bits len) 10 | = MsgAddressExt; 11 | anycast_info$_ depth:(#<= 30) { depth >= 1 } 12 | rewrite_pfx:(bits depth) = Anycast; 13 | addr_std$10 anycast:(Maybe Anycast) 14 | workchain_id:int8 address:bits256 = MsgAddressInt; 15 | addr_var$11 anycast:(Maybe Anycast) addr_len:(## 9) 16 | workchain_id:int32 address:(bits addr_len) = MsgAddressInt; 17 | _ _:MsgAddressInt = MsgAddress; 18 | _ _:MsgAddressExt = MsgAddress; 19 | 20 | transfer_jetton#0f8a7ea5 query_id:uint64 amount:(VarUInteger 16) destination:MsgAddress 21 | response_destination:MsgAddress custom_payload:(Maybe ^Cell) 22 | forward_ton_amount:(VarUInteger 16) forward_payload:(Either Cell ^Cell) 23 | = InternalMsgBody; 24 | 25 | transfer_notification#7362d09c query_id:uint64 amount:(VarUInteger 16) 26 | sender:MsgAddress forward_payload:(Either Cell ^Cell) 27 | = InternalMsgBody; 28 | 29 | excesses#d53276db query_id:uint64 = InternalMsgBody; 30 | 31 | burn#595f07bc query_id:uint64 amount:(VarUInteger 16) 32 | response_destination:MsgAddress custom_payload:(Maybe ^Cell) 33 | = InternalMsgBody; 34 | 35 | transfer_nft#5fcc3d14 query_id:uint64 new_owner:MsgAddress response_destination:MsgAddress 36 | custom_payload:(Maybe ^Cell) forward_amount:(VarUInteger 16) 37 | forward_payload:(Either Cell ^Cell) = InternalMsgBody; 38 | 39 | // ----- Unspecified by standard, but suggested format of internal message 40 | 41 | internal_transfer query_id:uint64 amount:(VarUInteger 16) from:MsgAddress 42 | response_address:MsgAddress 43 | forward_ton_amount:(VarUInteger 16) 44 | forward_payload:(Either Cell ^Cell) 45 | = InternalMsgBody; 46 | burn_notification query_id:uint64 amount:(VarUInteger 16) 47 | sender:MsgAddress response_destination:MsgAddress 48 | = InternalMsgBody; 49 | 50 | 51 | 52 | bit$_ (## 1) = Bit; 53 | /* 54 | * 55 | * FROM hashmap.tlb 56 | * 57 | */ 58 | // ordinary Hashmap / HashmapE, with fixed length keys 59 | // 60 | hm_edge#_ {n:#} {X:Type} {l:#} {m:#} label:(HmLabel ~l n) 61 | {n = (~m) + l} node:(HashmapNode m X) = Hashmap n X; 62 | 63 | hmn_leaf#_ {X:Type} value:X = HashmapNode 0 X; 64 | hmn_fork#_ {n:#} {X:Type} left:^(Hashmap n X) 65 | right:^(Hashmap n X) = HashmapNode (n + 1) X; 66 | 67 | hml_short$0 {m:#} {n:#} len:(Unary ~n) {n <= m} s:(n * Bit) = HmLabel ~n m; 68 | hml_long$10 {m:#} n:(#<= m) s:(n * Bit) = HmLabel ~n m; 69 | hml_same$11 {m:#} v:Bit n:(#<= m) = HmLabel ~n m; 70 | 71 | unary_zero$0 = Unary ~0; 72 | unary_succ$1 {n:#} x:(Unary ~n) = Unary ~(n + 1); 73 | 74 | hme_empty$0 {n:#} {X:Type} = HashmapE n X; 75 | hme_root$1 {n:#} {X:Type} root:^(Hashmap n X) = HashmapE n X; 76 | 77 | chunked_data#_ data:(HashmapE 32 ^Cell) = ChunkedData; // chunked_data#_ data:(HashMapE 32 ^(SnakeData ~0)) = ChunkedData; 78 | text#_ data:Cell = Text; // text#_ {n:#} data:(SnakeData ~n) = Text; 79 | snake#00 data:Cell = ContentData; // snake#00 data:(SnakeData ~n) = ContentData; 80 | chunks#01 data:ChunkedData = ContentData; 81 | onchain#00 data:(HashmapE 256 ^ContentData) = FullContent; 82 | offchain#01 uri:Text = FullContent; 83 | 84 | proto_http#4854 = Protocol; 85 | proto_list_nil$0 = ProtoList; 86 | 87 | proto_list_next$1 head:Protocol tail:ProtoList = ProtoList; 88 | 89 | cap_is_wallet#2177 = SmcCapability; 90 | 91 | cap_list_nil$0 = SmcCapList; 92 | cap_list_next$1 head:SmcCapability tail:SmcCapList = SmcCapList; 93 | 94 | dns_smc_address#9fd3 smc_addr:MsgAddressInt flags:(## 8) { flags <= 1 } 95 | cap_list:flags . 0?SmcCapList = DNSRecord; 96 | dns_next_resolver#ba93 resolver:MsgAddressInt = DNSRecord; 97 | dns_adnl_address#ad01 adnl_addr:bits256 flags:(## 8) { flags <= 1 } 98 | proto_list:flags . 0?ProtoList = DNSRecord; 99 | dns_storage_address#7473 bag_id:bits256 = DNSRecord; 100 | 101 | _ (HashmapE 256 ^DNSRecord) = DNS_RecordSet; 102 | --------------------------------------------------------------------------------