├── .gitignore ├── .gitlab-ci.yml ├── .gitmodules ├── Cargo.toml ├── LICENSE ├── Makefile ├── README.md ├── documents ├── arch_design.md ├── code_about.md ├── firecracker_kernel_notes.md ├── gitlab-runner.config.toml ├── roadmap.md ├── system_admin.md └── user_guide.md ├── src ├── client │ ├── Cargo.toml │ └── src │ │ ├── cfg_file.rs │ │ ├── cmd_line.rs │ │ ├── main.rs │ │ └── ops │ │ ├── config.rs │ │ ├── env │ │ ├── add.rs │ │ ├── del.rs │ │ ├── get │ │ │ ├── mod.rs │ │ │ ├── scp.rs │ │ │ └── ttrexec.rs │ │ ├── list.rs │ │ ├── listall.rs │ │ ├── mod.rs │ │ ├── push │ │ │ ├── mod.rs │ │ │ ├── scp.rs │ │ │ └── ttrexec.rs │ │ ├── run │ │ │ ├── mod.rs │ │ │ ├── ssh.rs │ │ │ └── ttrexec.rs │ │ ├── show.rs │ │ ├── start.rs │ │ ├── stop.rs │ │ └── update.rs │ │ ├── mod.rs │ │ └── status.rs ├── core │ ├── Cargo.toml │ └── src │ │ ├── def.rs │ │ ├── lib.rs │ │ ├── linux │ │ ├── mod.rs │ │ ├── nat │ │ │ └── mod.rs │ │ └── vm │ │ │ ├── cgroup.rs │ │ │ ├── engine │ │ │ ├── firecracker │ │ │ │ ├── mod.rs │ │ │ │ ├── suitable_env.rs │ │ │ │ └── unsuitable_env.rs │ │ │ ├── mod.rs │ │ │ └── qemu.rs │ │ │ ├── mod.rs │ │ │ └── util.rs │ │ ├── mocker │ │ ├── mod.rs │ │ ├── nat.rs │ │ └── vm.rs │ │ └── test.rs ├── core_def │ ├── Cargo.toml │ └── src │ │ └── lib.rs ├── proxy │ ├── Cargo.toml │ ├── src │ │ ├── bin │ │ │ └── ttproxy.rs │ │ ├── cfg.rs │ │ ├── def.rs │ │ ├── hdr │ │ │ ├── add_env.rs │ │ │ ├── mod.rs │ │ │ └── sync.rs │ │ ├── http.rs │ │ ├── lib.rs │ │ ├── uau.addr │ │ └── util.rs │ └── tests │ │ ├── env │ │ └── mod.rs │ │ ├── integration.rs │ │ ├── knead │ │ └── mod.rs │ │ └── standalone │ │ └── mod.rs ├── rexec │ ├── Cargo.toml │ ├── README.md │ ├── src │ │ ├── bin │ │ │ ├── cli.rs │ │ │ └── daemon.rs │ │ ├── client.rs │ │ ├── common.rs │ │ ├── lib.rs │ │ ├── sendfile.rs │ │ └── server.rs │ └── tests │ │ └── integration.rs ├── server │ ├── Cargo.toml │ ├── src │ │ ├── bin │ │ │ └── ttserver.rs │ │ ├── cfg.rs │ │ ├── def.rs │ │ ├── hdr │ │ │ ├── mod.rs │ │ │ └── server.rs │ │ ├── init.rs │ │ ├── lib.rs │ │ └── util.rs │ └── tests │ │ ├── env │ │ └── mod.rs │ │ ├── integration.rs │ │ ├── knead │ │ └── mod.rs │ │ └── standalone │ │ └── mod.rs ├── server_def │ ├── .gitignore │ ├── Cargo.toml │ └── src │ │ ├── included_ops_map.rs │ │ └── lib.rs └── utils │ ├── Cargo.toml │ └── src │ └── lib.rs └── tools ├── firecracker_update_image.sh ├── fmt.sh ├── githooks ├── pre-commit └── pre-commit.sample ├── images ├── README.md ├── freebsd_vm │ ├── rc.local │ └── rc.local_hostfwd ├── id_rsa ├── id_rsa.pub └── linux_vm │ ├── README.md │ ├── firecracker_kernel_config │ ├── centos:linux-3.10.0-1062.el7.x86_64 │ ├── centos:linux-3.10.0-1127.el7.x86_64 │ ├── centos:linux-3.10.0-123.el7.x86_64 │ ├── centos:linux-3.10.0-229.el7.x86_64 │ ├── centos:linux-3.10.0-327.el7.x86_64 │ ├── centos:linux-3.10.0-514.el7.x86_64 │ ├── centos:linux-3.10.0-693.el7.x86_64 │ ├── centos:linux-3.10.0-862.el7.x86_64 │ ├── centos:linux-3.10.0-957.el7.x86_64 │ ├── centos:linux-4.14.0-115.el7.0.1.x86_64 │ ├── centos:linux-4.18.0-147.0.3.el8.x86_64 │ ├── centos:linux-4.18.0-147.3.1.el8.x86_64 │ ├── centos:linux-4.18.0-147.5.1.el8.x86_64 │ ├── centos:linux-4.18.0-147.8.1.el8.x86_64 │ ├── centos:linux-4.18.0-147.el8.x86_64 │ ├── centos:linux-4.18.0-193.1.2.el8.x86_64 │ ├── centos:linux-4.18.0-193.14.2.el8.x86_64 │ ├── centos:linux-4.18.0-193.19.1.el8.x86_64 │ ├── centos:linux-4.18.0-193.6.3.el8.x86_64 │ ├── centos:linux-4.18.0-193.el8.x86_64 │ ├── centos:linux-4.18.0-80.1.2.el8.x86_64 │ ├── centos:linux-4.18.0-80.11.1.el8.x86_64 │ ├── centos:linux-4.18.0-80.11.2.el8.x86_64 │ ├── centos:linux-4.18.0-80.4.2.el8.x86_64 │ ├── centos:linux-4.18.0-80.7.1.el8.x86_64 │ ├── centos:linux-4.18.0-80.7.2.el8.x86_64 │ ├── centos:linux-4.18.0-80.el8.x86_64 │ └── mainline-4.19.148-minimal-firecracker-dev │ ├── rc.local │ └── rc.local_hostfwd ├── install.sh ├── qemu_update_image.sh └── trash ├── baseimage_update.sh ├── muslenv.sh ├── net.start.gentoo ├── start.sh └── vfio_bind.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | target/ 4 | 5 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 6 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 7 | Cargo.lock 8 | 9 | # These are backup files generated by rustfmt 10 | **/*.rs.bk 11 | 12 | 13 | #Added by cargo 14 | 15 | /target 16 | 17 | 18 | #Added by cargo 19 | # 20 | #already existing elements were commented out 21 | 22 | #/target 23 | #Cargo.lock 24 | 25 | nohup.out 26 | tt.*.*z 27 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | stages: 2 | - build 3 | - test 4 | - deploy 5 | 6 | job_build: 7 | stage: build 8 | script: 9 | - make lint 10 | tags: 11 | - rust 12 | 13 | job_test: 14 | stage: test 15 | script: 16 | - make test 17 | - export PATH=$PATH:~/.cargo/bin 18 | - cargo kcov --all 19 | - COVERAGE=$(grep -Po '(?<="covered":")\d+(?:\.\d+)?(?=")' target/cov/index.js) 20 | # Tips: use '^Coverage:\s+(\d+(?:\.\d+)?)' on gitlab to capture the result 21 | - echo "Coverage:" ${COVERAGE} 22 | tags: 23 | - rust 24 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "tools/firecracker"] 2 | path = tools/firecracker 3 | url = https://gitee.com/kt10/firecracker 4 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "3" 3 | members = [ 4 | "src/core_def", 5 | "src/core", 6 | "src/server_def", 7 | "src/server", 8 | "src/proxy", 9 | "src/client", 10 | "src/rexec", 11 | "src/utils", 12 | ] 13 | 14 | [profile.dev] 15 | overflow-checks = true 16 | panic = "unwind" 17 | 18 | [profile.release] 19 | lto = true 20 | incremental = false 21 | overflow-checks = false 22 | panic = "unwind" 23 | 24 | [profile.bench] 25 | codegen-units = 1 26 | overflow-checks = false 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 范辉 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | CARGO = ~/.cargo/bin/cargo 3 | BUILD_DIR = /tmp/.__tt_build_dir__ 4 | PACK_NAME = tt 5 | PACK_DIR = $(BUILD_DIR)/$(PACK_NAME) 6 | TARGET = $(shell rustup toolchain list | grep default | sed 's/^[^-]\+-//' | sed 's/ \+(default).*//') 7 | 8 | all: pack 9 | 10 | build: 11 | $(CARGO) build --bins 12 | 13 | release: 14 | $(CARGO) build --bins --release 15 | 16 | pack: release 17 | -@ rm -rf $(PACK_DIR) 18 | mkdir -p $(PACK_DIR) 19 | cd target/release && cp tt ttserver ttproxy $(PACK_DIR)/ 20 | cp tools/install.sh $(PACK_DIR)/ 21 | \ 22 | git submodule update --init --recursive 23 | cd tools/firecracker \ 24 | && cargo build --release --target=$(TARGET) --target-dir=$(BUILD_DIR) \ 25 | && cp $(BUILD_DIR)/$(TARGET)/release/firecracker $(PACK_DIR)/ 26 | \ 27 | chmod -R +x $(PACK_DIR) 28 | tar -C $(BUILD_DIR) -zcf $(PACK_NAME).tar.gz $(PACK_NAME) 29 | 30 | install: 31 | $(CARGO) install -f --bins --path src/rexec --root=/usr/local/ 32 | $(CARGO) install -f --bins --path src/client --root=/usr/local/ 33 | $(CARGO) install -f --bins --path src/server --root=/usr/local/ # will fail on MacOS 34 | $(CARGO) install -f --bins --path src/proxy --root=/usr/local/ # will fail on MacOS 35 | 36 | lint: githook 37 | $(CARGO) clippy 38 | cd src/core && $(CARGO) clippy --features="testmock" 39 | cd src/core && $(CARGO) clippy --no-default-features 40 | cd src/core && $(CARGO) clippy --no-default-features --features="zfs" 41 | cd src/core && $(CARGO) clippy --no-default-features --features="cow" 42 | cd src/core && $(CARGO) clippy --no-default-features --features="nft" 43 | cd src/core && $(CARGO) clippy --no-default-features --features="cow nft" 44 | cd src/server && $(CARGO) clippy --features="testmock" 45 | cd src/server && $(CARGO) clippy --no-default-features 46 | cd src/server && $(CARGO) clippy --no-default-features --features="zfs" 47 | cd src/server && $(CARGO) clippy --no-default-features --features="cow" 48 | cd src/server && $(CARGO) clippy --no-default-features --features="nft" 49 | cd src/server && $(CARGO) clippy --no-default-features --features="cow nft" 50 | cd src/proxy && $(CARGO) clippy --features="testmock" 51 | 52 | test: test_debug 53 | 54 | test_debug: stop 55 | $(CARGO) test -- --test-threads=1 --nocapture 56 | -@ pkill -9 integration 57 | cd src/server && $(CARGO) test --features="testmock" -- --test-threads=1 --nocapture 58 | -@ pkill -9 integration 59 | cd src/server && $(CARGO) test --features="testmock, cow" -- --test-threads=1 --nocapture 60 | -@ pkill -9 integration 61 | cd src/server && $(CARGO) test --no-default-features --features="testmock" -- --test-threads=1 --nocapture 62 | -@ pkill -9 integration 63 | cd src/server && $(CARGO) test --no-default-features --features="testmock, cow" -- --test-threads=1 --nocapture 64 | -@ pkill -9 integration 65 | cd src/proxy && $(CARGO) test --features="testmock" -- --test-threads=1 --nocapture 66 | -@ pkill -9 integration 67 | 68 | test_release: stop 69 | $(CARGO) test --release -- --test-threads=1 --nocapture 70 | -@ pkill -9 integration 71 | cd src/server && $(CARGO) test --release --features="testmock" -- --test-threads=1 --nocapture 72 | -@ pkill -9 integration 73 | cd src/server && $(CARGO) test --release --features="testmock, cow" -- --test-threads=1 --nocapture 74 | -@ pkill -9 integration 75 | cd src/server && $(CARGO) test --release --no-default-features --features="testmock" -- --test-threads=1 --nocapture 76 | -@ pkill -9 integration 77 | cd src/server && $(CARGO) test --release --no-default-features --features="testmock, cow" -- --test-threads=1 --nocapture 78 | -@ pkill -9 integration 79 | cd src/proxy && $(CARGO) test --release --features="testmock" -- --test-threads=1 --nocapture 80 | -@ pkill -9 integration 81 | 82 | fmt: 83 | @ ./tools/fmt.sh 84 | 85 | doc: 86 | $(CARGO) doc --open -p tt 87 | $(CARGO) doc --open -p ttrexec 88 | $(CARGO) doc --open -p ttproxy 89 | $(CARGO) doc --open -p ttserver # will fail on MacOS 90 | $(CARGO) doc --open -p ttcore # will fail on MacOS 91 | 92 | githook: 93 | @mkdir -p ./.git/hooks # play with online gitlab-ci 94 | # @cp ./tools/githooks/pre-commit ./.git/hooks/ 95 | 96 | stop: 97 | -@ pkill -9 ttproxy 98 | -@ pkill -9 ttserver 99 | -@ pkill -9 ttrexec-daemon 100 | 101 | clean: 102 | @ git clean -fdx 103 | @ $(CARGO) clean 104 | @ find . -type f -name "Cargo.lock" | xargs rm -f 105 | 106 | cleanall: clean 107 | @ rm -rf client core core_def server server_def proxy rexec 108 | @ find . -type d -name "target" | xargs rm -rf 109 | -------------------------------------------------------------------------------- /documents/arch_design.md: -------------------------------------------------------------------------------- 1 | # System Design 2 | 3 | ## 架构说明 4 | 5 | ### 上层逻辑 6 | 7 | TT 中的基本管理单位为 “环境(ENV)”, 如下以时序图的方式展现一个 “环境” 的创建过程. 8 | 9 | ```mermaid 10 | sequenceDiagram 11 | autonumber 12 | 13 | participant C as Client 14 | participant P as Proxy 15 | participant S as Server[s] 16 | participant R as Core 17 | 18 | C->>P: 创建 VM 的请求 19 | par 分发请求 20 | P-->>S: 计算可用资源, 并行分发 21 | end 22 | 23 | S->>R: 调用 Core 创建 VM 24 | Note right of S: 配置网络及生命周期 25 | R->>S: return 26 | 27 | par 返回结果 28 | S->>P: 异步返回 29 | end 30 | 31 | P->>C: 聚合各 Server 的结果 32 | 33 | loop 资源管理 34 | R-->>R: 定时清理过期的 VM 35 | end 36 | ``` 37 | 38 | ### Core 内部实现 39 | 40 | #### On Linux 41 | 42 | ```mermaid 43 | sequenceDiagram 44 | autonumber 45 | 46 | participant C as Core 47 | participant K as Kernel 48 | participant Q as Qemu/FireCracker 49 | participant N as Nftables 50 | 51 | C->>K: 创建 PID NS 与 CGROUP 52 | C->>Q: 增量(COW)创建运行时镜像 53 | C->>N: 使用 Nftables 的哈希表结构管理 NAT 规则 54 | ``` 55 | 56 | ## Why NOT 57 | 58 | ### Why NOT K8S 59 | 60 | K8S 主要用于调度容器, 不适于对隔离性要求较高的场景. 61 | 62 | ### Why NOT OpenStack 63 | 64 | OpenStack 太过复杂, 需要专门的团队维护, 成本太高. 65 | 66 | ### Why NOT Ansible 67 | 68 | Ansible 只是一个批量管理工具, 不具备虚拟方案的管理与调度功能. 69 | 70 | ### Why NOT Libvirt 71 | 72 | Libvirt 在安装系统及远程管理方面非常便捷, 但不具备自动化调度的能力. 当前 TT 系统使用 Libvirt 做为基础系统镜像的安装工具. 73 | -------------------------------------------------------------------------------- /documents/code_about.md: -------------------------------------------------------------------------------- 1 | # tt 2 | 3 | tt 主要代码使用 Rust 编写. 4 | 5 | Rust 是一门风格**紧凑**、运行**高效**的现代开发语言, 以**最少的代码量**实现**最高的性能**, 打破了几十年来的旷世难题: 静态语言与动态语言两者的优势不能兼得. 6 | 7 | ## 项目结构 8 | 9 | ``` 10 | $ (git)-[master]-% tree -F -I 'target' -L 1 11 | . 12 | ├── Cargo.toml # 项目配置文件 13 | ├── README.md # 项目主文档 14 | ├── documents/ # 项目详细文档 15 | ├── src/core/ # 服务端的核心逻辑实现 16 | ├── src/core_def/ # 从 core 模块中提取出的通用定义, 供 server 模块使用 17 | ├── src/server/ # 后端 Server 的代码实现, 可独立运行, 也可挂靠在 Proxy 之后 18 | ├── src/server_def/ # 从 server 模块中提取出的通用定义, 供 proxy 模块使用 19 | ├── src/proxy/ # 分布式架构后端, 负责统筹调度多个 Server 的资源 20 | ├── src/rexec/ # 一个轻量级的"远程命令执行和文件转输"方案 21 | ├── src/client/ # 客户端 tt 命令的代码实现 22 | ├── tools/ # 外围脚本工具 23 | └── ... 24 | ``` 25 | 26 | ## 代码规模 27 | 28 | ``` 29 | $ (git)-[master]-% find . -type f \ 30 | | grep -Ev 'target|tools/(.*kernel_config|firecracker)|\.(git|lock)' \ 31 | | xargs wc -l \ 32 | | grep -Ev '^ +[0-9]{1,2} ' 33 | 34 | 108 ./Makefile 35 | 194 ./README.md 36 | 482 ./documents/user_guide.md 37 | 169 ./src/rexec/tests/integration.rs 38 | 117 ./src/rexec/src/bin/cli.rs 39 | 149 ./src/rexec/src/client.rs 40 | 272 ./src/rexec/src/server.rs 41 | 164 ./src/rexec/src/common.rs 42 | 1220 ./src/core/src/def.rs 43 | 293 ./src/core/src/linux/nat/mod.rs 44 | 205 ./src/core/src/linux/mod.rs 45 | 150 ./src/core/src/linux/vm/cgroup.rs 46 | 357 ./src/core/src/linux/vm/engine/qemu.rs 47 | 245 ./src/core/src/linux/vm/engine/firecracker/suitable_env.rs 48 | 312 ./src/proxy/tests/knead/mod.rs 49 | 220 ./src/proxy/tests/standalone/mod.rs 50 | 179 ./src/proxy/tests/env/mod.rs 51 | 151 ./src/proxy/src/util.rs 52 | 102 ./src/proxy/src/def.rs 53 | 145 ./src/proxy/src/hdr/add_env.rs 54 | 424 ./src/proxy/src/hdr/mod.rs 55 | 265 ./src/proxy/src/lib.rs 56 | 198 ./src/core_def/src/lib.rs 57 | 267 ./src/server/tests/knead/mod.rs 58 | 193 ./src/server/tests/standalone/mod.rs 59 | 107 ./src/server/tests/env/mod.rs 60 | 100 ./src/server/src/bin/ttserver.rs 61 | 329 ./src/server/src/hdr/mod.rs 62 | 120 ./src/server/src/lib.rs 63 | 231 ./src/server_def/src/lib.rs 64 | 555 ./src/client/src/cmd_line.rs 65 | 157 ./src/client/src/ops/env/update.rs 66 | 253 ./src/client/src/ops/env/run/mod.rs 67 | 114 ./src/client/src/ops/env/run/ssh.rs 68 | 131 ./src/client/src/ops/mod.rs 69 | 149 ./src/client/src/cfg_file.rs 70 | 12034 total 71 | ``` 72 | -------------------------------------------------------------------------------- /documents/firecracker_kernel_notes.md: -------------------------------------------------------------------------------- 1 | # Kernel 2 | 3 | ## Configs MUST be Built In 4 | 5 | #### file system 6 | 7 | - CONFIG_EXT4_FS 8 | 9 | #### virtio basic 10 | 11 | - CONFIG_VIRTIO_PCI 12 | - CONFIG_VIRTIO_INPUT 13 | - CONFIG_VIRTIO_MMIO 14 | - CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES 15 | 16 | #### virtio block device 17 | 18 | - CONFIG_VIRTIO_BLK 19 | 20 | #### virtio network device 21 | 22 | - CONFIG_VSOCKETS 23 | - CONFIG_VIRTIO_VSOCKETS 24 | - CONFIG_VIRTIO_NET 25 | 26 | #### make script[s] exectuable 27 | 28 | - BINFMT_MISC 29 | 30 | ## Configs MUST be disabled 31 | 32 | #### accept stripped modular[s] 33 | 34 | - CONFIG_MODULE_SIG 35 | -------------------------------------------------------------------------------- /documents/gitlab-runner.config.toml: -------------------------------------------------------------------------------- 1 | concurrent = 1 2 | check_interval = 0 3 | 4 | [session_server] 5 | session_timeout = 1800 6 | 7 | [[runners]] 8 | name = "YourHostName" 9 | url = "https://gitlab.com/" 10 | token = "YourGitLabToken" 11 | executor = "shell" 12 | cache_dir = "/home/gitlab-runner/build-cache" 13 | -------------------------------------------------------------------------------- /documents/roadmap.md: -------------------------------------------------------------------------------- 1 | # 开发路线 (RoadMap) 2 | 3 | ## v0.1.x 4 | 5 | - [Y] 单点架构, 基本功能可用 6 | 7 | ## v0.2.x 8 | 9 | - [Y] 分布式架构, 核心功能可用 10 | 11 | ## v0.3.x 12 | 13 | - [Y] v0.3.1: 优化资源调度算法 14 | - [Y] v0.3.1: 支持服务重启后恢复已有的 ENV 15 | - [-] 优化系统布署流程, 提供配置的自动化工具集 16 | - [-] 优化基础镜像制作流程, 提供配置的自动化工具集 17 | - [-] 提高核心模块的测试覆盖率 18 | - [-] 提供视频演示教程 19 | 20 | ## v0.4.x 21 | 22 | - [-] 优化命令行客户端的用户提示信息 23 | - 当前显示给用户的是开发端的信息 24 | - [-] 提供前端界面 25 | - 初步规划是基于 Yew/Seed 等 Rust 前端框架实现 26 | - [-] 完善英文文档 27 | 28 | ## v0.5.x 29 | 30 | - [-] 优化前端界面实现, 增强特性 31 | - 如: Web Terminal 等功能 32 | - [-] 提供一个实时体验的网址 33 | 34 | ## v0.6.x 35 | 36 | - [-] 支持将运行时镜像保存为基础镜像(模板镜像) 37 | - [-] 支持对已创建的 ENV 打快照(多级快照如何高效管理?) 38 | 39 | ## v0.7.x 40 | 41 | - [-] 待定... 42 | -------------------------------------------------------------------------------- /documents/system_admin.md: -------------------------------------------------------------------------------- 1 | # TT 系统管理员指南 2 | 3 | Guide for system admin. 4 | 5 | ## 环境准备 6 | 7 | > If use zfs, should do this: `zfs create -V 1M zroot/tt/__sample__`. 8 | 9 | 镜像相关: 10 | - [Linux, MUST] `systemctl disable NetworkManager` 11 | - [Linux, Optional] `systemctl disable firewalld` 12 | - [Linux, Optional] `sed -i 's/SELINUX=.*/SELINUX=disabled/' /etc/selinux/config` 13 | - 保证本项目定制的 "[rc.local](../tools/images/linux_vm/rc.local)" 文件开机启动 14 | - [ FireCracker ] 内核模块只能 `strip --strip-debug`, strip 过度会导致无法载入 15 | - [ FireCracker ] 关闭内核的模块签名校验功能, 因为签名信息存在于已被 strip 的 debug 信息中 16 | - [ FireCracker ] `firecracker` 二进制文件要使用 `musl-libc` 编译 17 | - 由于 `Rust` 动态链接 `glibc` , 故运行时存在更多的不确定性因素, 参见: [issue #2044](https://github.com/firecracker-microvm/firecracker/issues/2044) 18 | 19 | #### On Linux 20 | 21 | > 使用 Qemu\FireCracker 做为 VM 引擎, 使用 Nftables 做 NAT 端口转发. 22 | > 23 | > - `firecracker` 程序路径必须是 `/usr/sbin/firecracker` 24 | 25 | 环境配置: 26 | 27 | - `modprobe tun vhost_net` 28 | 29 | 组件安装 30 | 31 | - `qemu` 32 | - `nftables` 33 | - `ZOL: zfs on linux` 34 | 35 | ## ttserver 配置 36 | 37 | ```shell 38 | USAGE: 39 | ttserver [FLAGS] [OPTIONS] 40 | 41 | FLAGS: 42 | -h, --help Prints help information 43 | -V, --version Prints version information 44 | 45 | OPTIONS: 46 | --cpu-total 可以使用的 CPU 核心总数. 47 | --disk-total 可以使用的磁盘总量, 单位: MB. 48 | --image-path 镜像存放路径. 49 | --cfgdb-path ENV 配置信息存放路径. 50 | --log-path 日志存储路径. 51 | --mem-total 可以使用的内存总量, 单位: MB. 52 | --serv-addr 服务监听地址. 53 | --serv-port 服务监听端口. 54 | ``` 55 | 56 | ## ttproxy 配置 57 | 58 | ```shell 59 | USAGE: 60 | ttproxy [OPTIONS] 61 | 62 | FLAGS: 63 | -h, --help Prints help information 64 | -V, --version Prints version information 65 | 66 | OPTIONS: 67 | --proxy-addr ttproxy 地址, eg: 127.0.0.1:19527. 68 | --server-set ... ttserver 地址, eg: 127.0.0.1:9527,10.10.10.101:9527. 69 | ``` 70 | 71 | ## gitlab CI/CD 72 | 73 | shell 模式下, 需要将 `~gitlab-runner/.bash_logout` 删除: 74 | 75 | ```shell 76 | rm ~gitlab-runner/.bash_logout 77 | ``` 78 | -------------------------------------------------------------------------------- /src/client/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tt" 3 | version = "0.3.2" 4 | 5 | edition = "2024" 6 | authors = ["FanHui ", "FanHui "] 7 | description = "Lightweight private cloud solution for SME scenarios." 8 | license = "MIT OR Apache-2.0" 9 | keywords = ["firecracker", "qemu", "kvm", "openstack", "k8s", "cloud"] 10 | categories = ["command-line-utilities"] 11 | 12 | [dependencies] 13 | clap = "4.5.26" 14 | 15 | nix = { version = "0.29.0", features = ["fs"] } 16 | serde_json = "1.0.133" 17 | serde = { version = "1.0.216", features = ["derive"] } 18 | 19 | threadpool = "1.8.1" 20 | 21 | ttutils = { path = "../utils" } 22 | ttserver_def = { path = "../server_def" } 23 | ttrexec = { path = "../rexec", default-features = false, features = [ "client" ] } 24 | ruc = "8.1.2" 25 | -------------------------------------------------------------------------------- /src/client/src/cfg_file.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Cfg File 3 | //! 4 | 5 | use ruc::*; 6 | use serde::{Deserialize, Serialize}; 7 | use std::{env, fs, net::SocketAddr, path, sync::LazyLock}; 8 | 9 | /// 客户端配置路径 10 | pub static CFG_PATH: LazyLock = LazyLock::new(|| format!("{}/.tt",pnk!(env::var("HOME")))); 11 | /// 客户端配置文件 12 | pub static CFG_FILE: LazyLock = LazyLock::new(|| format!("{}/tt.json", CFG_PATH.as_str())); 13 | /// 客户端配置信息 14 | pub static CFG: LazyLock = LazyLock::new(|| pnk!(read_cfg())); 15 | 16 | /// 后续支持同时连接多个 server 17 | #[derive(Debug, Default, Deserialize, Serialize)] 18 | pub struct Cfg { 19 | /// 暂时只接受单个值 20 | pub server_list: Server, 21 | /// 本机别名 22 | pub client_id: String, 23 | } 24 | 25 | impl Cfg { 26 | /// TODO 27 | pub fn print_to_user(&self) { 28 | dbg!(self); 29 | } 30 | 31 | fn get_servaddr(&self) -> Result { 32 | format!("{}:{}", self.server_list.addr, self.server_list.port) 33 | .parse::() 34 | .c(d!()) 35 | } 36 | } 37 | 38 | /// 转换平面地址为 SocketAddr 39 | pub fn get_servaddr() -> Result { 40 | read_cfg().c(d!())?.get_servaddr().c(d!()) 41 | } 42 | 43 | /// 服务端连接信息 44 | #[derive(Debug, Default, Deserialize, Serialize)] 45 | pub struct Server { 46 | /// IP 地址 47 | pub addr: String, 48 | /// 服务端口 49 | pub port: u16, 50 | } 51 | 52 | impl Server { 53 | /// 创建新的服务端连接实例 54 | pub fn new(addr: &str, port: u16) -> Server { 55 | Server { 56 | addr: addr.to_owned(), 57 | port, 58 | } 59 | } 60 | } 61 | 62 | /// 读取客户端配置文件 63 | pub fn read_cfg() -> Result { 64 | fs::read(CFG_FILE.as_str()) 65 | .c(d!()) 66 | .and_then(|cfg| serde_json::from_slice(&cfg).c(d!())) 67 | } 68 | 69 | /// 写入客户端配置文件 70 | pub fn write_cfg(cfg: &Cfg) -> Result<()> { 71 | serde_json::to_string_pretty(cfg) 72 | .c(d!()) 73 | .and_then(|cfg| fs::write(CFG_FILE.as_str(), cfg).c(d!())) 74 | } 75 | 76 | /// 创建客户端配置路径和文件 77 | pub fn cfg_init() -> Result<()> { 78 | if !path::Path::new(CFG_FILE.as_str()).exists() { 79 | fs::create_dir_all(CFG_PATH.as_str()).c(d!())?; 80 | } 81 | 82 | // 如果存在并且格式解析通过, 则什么都不做; 83 | // 否则就写入一个空文件 84 | read_cfg() 85 | .c(d!()) 86 | .map(|_| ()) 87 | .or_else(|_| write_cfg(&Cfg::default()).c(d!())) 88 | .and_then(|_| set_sshenv()) 89 | } 90 | 91 | ////////////////////////////////////////////////////////////////// 92 | 93 | const RSA_PRIVATE: &str = "-----BEGIN RSA PRIVATE KEY----- 94 | MIIEowIBAAKCAQEA5Pwkml4qe8MGtsbA2mZunipshE/dXyNvLn3VTGb7Mpu7PMtE 95 | FcXTterfSJWx/aIJ6f8yxEyUpdTMaBM9UdiPaxPkdHq2ngU/yyLBeZyMjAWf2ZOU 96 | kTa6HIlvvcOAMVmR3LoBs51RSMW3gtiWv6uMcQ1kW7TweZ79kfdnk8GWek6L8Y5k 97 | hl1HAVUVRYLxNJOyoHUQhivatCTlGTE9nd37PBZDiT3ks552sGlIi5S2BbGNUeBf 98 | /tK/s5Ww4CqwZp479RmCqIQXBTLMNdJtPHFN1Iycpuu0dCNjSjABYVgWQIeCBjCH 99 | 4O8N1GtiuJx0MJXJzlrme485Zf52EZq0vU9yxQIDAQABAoIBAQDT/mPczoVCY0Ja 100 | ARQWnnKW1+vzawUlyWZrgm/w9f5l0iu8kusLxUTFzRa+2mgYyuWmz28usT+Fb8d2 101 | KynAFmBg39/HvrxG+9EdvaWlczvjfmmJQ8ptzl7rgIoFA3QxPB2AXmyo32KbnwDQ 102 | kLiv5qB1IdLh3FguIPXdJ1GrR7SKsWDVooOEWJvJfbNxt/ak5gxow9LIN/2Wv1PM 103 | xh/+C7Evx1auOkVuC9+2F7rJbb5M/B6B+YLqjOMtjIe1ME9VUqkSuf3bsYHpMVv5 104 | LLVylEpfdtYG6fi1t9SA+P6e4DyH13zQLSN3QDjUdeeyIbXqe1FruiYpQtLo3A4r 105 | 6+rKaFjZAoGBAPurbsDvtAbxfR0YFKNov3nhE4QE71rr+B1VLGinnHX3I60tpY07 106 | jhf+v1ntmCsXY64W8HFdn0rOpUXPhwsdyXi9lVBA9JFAICPyVu/o6mJXfJ8a1BMi 107 | E96veImNDwuK/0Ae5VK1jcdouZgoG9co6AWmgueHx3D92fFs/Oirh+1nAoGBAOjs 108 | yZpt1+gH1m3/skzUoKGgc+vk13LM42EtjTQy1mnWfks7Nl8lkDKrlyTqDJxeycTP 109 | Cr7+bkqQkmQFHtzuBiSq7yt4KXOdlWYrsx9KSbYMXFPdugxfiOFQsidNAOLpobjh 110 | QIJWmFYSQFWdVNv/YOQ8DCNbznt5VhlBF4UkY9bzAoGAd+862LdjE+wBs9vF+hnx 111 | JiQdKM0xRCMwGsp8X2OBLLaaSe1299dp4AWHK1QPMHn1BwHnlB8JypywJpS/xoxr 112 | dx7iCVzrME1fA8J5q9tT14nZ2fjvGC8lSPpWdzbB9L5I5kXTA5eB+YXu7JQwsFjO 113 | OeMgfzY11aMkOem2nSshnAECgYBs1WcFz1lYw4C/+P+4wokjvDMt/8ljjLSZzYzy 114 | 3OYuodh1En+/SW/tHRwMVYf68JdabFtbDss97/tW3MWk+VrJe00xhH3p1bHfAYA6 115 | mJ2EgJYLYcjyyxjMHsZ/co19eSjll+pqfEfFv9Vrq43hFZySSDRruRPrwbAnMLDq 116 | tywnXQKBgG7MosZPAKKGUUpBqw6ZbkHy/K+qSUtKvyUVIXVBl8hQGeeKltcOVesG 117 | UIkpfJfmyFlw52THX7jwc0RmHfIjjeit4vRW2Id2wUNRlgFW5OQu5PCncRhKpdJM 118 | wUVL9Lu+n7huI+GtsmATdhUYlzE/K8nNmhJuw3abrGPPUgrtx5OE 119 | -----END RSA PRIVATE KEY-----"; 120 | 121 | const SSH_SETTING: &str = "StrictHostKeyChecking no 122 | UserKnownHostsFile /dev/null"; 123 | 124 | static SSH_CFG_DIR: LazyLock = 125 | LazyLock::new(|| format!("{}/.ssh", pnk!(env::var("HOME")))); 126 | static SSH_CFG_FILE: LazyLock = 127 | LazyLock::new(|| format!("{}/config", SSH_CFG_DIR.as_str())); 128 | /// 连接 VM 所用的私钥在 client 上的路径 129 | pub static SSH_VM_KEY: LazyLock = 130 | LazyLock::new(|| format!("{}/tt_rsa", SSH_CFG_DIR.as_str())); 131 | 132 | /// 确保无交互的建立 SSH 连接 133 | fn set_sshenv() -> Result<()> { 134 | fs::create_dir_all(SSH_CFG_DIR.as_str()) 135 | .c(d!()) 136 | .and_then(|_| fs::write(SSH_CFG_FILE.as_str(), SSH_SETTING).c(d!())) 137 | .and_then(|_| { 138 | omit!(fs::remove_file(SSH_VM_KEY.as_str()).c(d!())); 139 | fs::write(SSH_VM_KEY.as_str(), RSA_PRIVATE).c(d!()) 140 | }) 141 | .and_then(|_| { 142 | crate::cmd_exec("chmod", &["0400", SSH_VM_KEY.as_str()]).c(d!()) 143 | }) 144 | } 145 | -------------------------------------------------------------------------------- /src/client/src/main.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # tt client 3 | //! 4 | 5 | #![warn(missing_docs, unused_import_braces, unused_extern_crates)] 6 | 7 | pub mod cfg_file; 8 | pub mod cmd_line; 9 | mod ops; 10 | 11 | pub use cfg_file::*; 12 | use ruc::*; 13 | use std::process; 14 | 15 | /// Generate log message from error 16 | pub fn genlog(err: E) -> String { 17 | format!("{}", err) 18 | } 19 | 20 | fn main() { 21 | pnk!(cfg_file::cfg_init()); 22 | pnk!(cmd_line::parse_and_exec()); 23 | } 24 | 25 | #[inline(always)] 26 | fn cmd_exec(cmd: &str, args: &[&str]) -> Result<()> { 27 | cmd_exec_with_output(cmd, args).c(d!()).map(|_| ()) 28 | } 29 | 30 | #[inline(always)] 31 | fn cmd_exec_with_output(cmd: &str, args: &[&str]) -> Result { 32 | let res = process::Command::new(cmd).args(args).output().c(d!())?; 33 | if res.status.success() { 34 | Ok(String::from_utf8_lossy(&res.stdout).into_owned()) 35 | } else { 36 | Err(eg!(String::from_utf8_lossy(&res.stderr))) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/client/src/ops/config.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Config Request 3 | //! 4 | 5 | use super::*; 6 | use crate::{read_cfg, write_cfg, Server}; 7 | use ruc::*; 8 | use std::{env, net::SocketAddr, process}; 9 | 10 | /////////////////////////////// 11 | #[derive(Default, Debug)] 12 | pub struct Config<'a> { 13 | pub server_addr: &'a str, 14 | pub server_port: u16, 15 | pub client_id: &'a str, 16 | } 17 | /////////////////////////////// 18 | 19 | impl<'a> Config<'a> { 20 | /// - 本地信息直接写入配置文件 21 | /// - 更新 server 信息时, 需验证其有效性 22 | pub fn do_req(&mut self) -> Result<()> { 23 | if "" == self.client_id && "" == read_cfg().c(d!())?.client_id { 24 | return Err(eg!("Client ID can't be empty !!!")); 25 | } 26 | 27 | if "" != self.client_id { 28 | read_cfg().c(d!()).and_then(|mut cfg| { 29 | // 尽可能避免分布式环境中的重复事件: 30 | // 用户指定的ID + timestamp + processId + $USER 31 | cfg.client_id = format!( 32 | "{}@{}@{}@{}", 33 | self.client_id, 34 | ts!(), 35 | process::id(), 36 | env::var("USER").unwrap_or_else(|_| "".to_owned()) 37 | ); 38 | write_cfg(&cfg).c(d!()) 39 | })?; 40 | } 41 | 42 | if "" != self.server_addr { 43 | alt!(0 == self.server_port, self.server_port = 9527); 44 | get_ops_id("register_client_id") 45 | .c(d!()) 46 | .and_then(|ops_id| { 47 | format!("{}:{}", self.server_addr, self.server_port) 48 | .parse::() 49 | .map(|addr| (ops_id, addr)) 50 | .c(d!("Invalid server_addr OR server_port")) 51 | }) 52 | .and_then(|(ops_id, addr)| { 53 | send_req::<&str>(ops_id, gen_req(""), addr).c(d!()) 54 | }) 55 | .and_then(|_| read_cfg().c(d!())) 56 | .and_then(|mut cfg| { 57 | cfg.server_list = 58 | Server::new(self.server_addr, self.server_port); 59 | write_cfg(&cfg).c(d!()) 60 | })?; 61 | } 62 | 63 | Ok(()) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/client/src/ops/env/add.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Env 3 | //! 4 | //! ```shell 5 | //! tt env add ... 6 | //! ``` 7 | //! 8 | 9 | use super::super::*; 10 | use crate::{get_servaddr, resp_print}; 11 | use ruc::*; 12 | 13 | /////////////////////////////// 14 | #[derive(Default)] 15 | pub struct EnvAdd<'a> { 16 | pub env_id: &'a str, 17 | pub os_prefix: Vec<&'a str>, 18 | pub vm_port: Vec, 19 | /// 0 代表使用服务端预设的默认值 20 | pub life_time: u64, 21 | /// 0 代表使用服务端预设的默认值 22 | pub cpu_num: u8, 23 | /// 0 代表使用服务端预设的默认值 24 | pub mem_size: u16, 25 | /// 0 代表使用服务端预设的默认值 26 | pub disk_size: u32, 27 | /// 0 代表使用服务端预设的默认值 28 | pub dup_each: u16, 29 | /// 是否禁止 VM 对外连网 30 | pub deny_outgoing: bool, 31 | /// VM uuid 是否随机化(唯一) 32 | pub rand_uuid: bool, 33 | } 34 | /////////////////////////////// 35 | 36 | impl<'a> EnvAdd<'a> { 37 | /// 发送请求并打印结果 38 | pub fn do_req(self) -> Result<()> { 39 | if 0 < self.life_time && 30 > self.life_time { 40 | return Err(eg!("Life time too short(min: 30) !")); 41 | } 42 | 43 | if 12 < self.cpu_num { 44 | return Err(eg!("Cpu number too large(max: 12) !")); 45 | } 46 | 47 | if 0 < self.mem_size && 50 > self.mem_size { 48 | return Err(eg!("Memory size too small(min: 50MB) !")); 49 | } 50 | 51 | if 48 * 1024 < self.mem_size { 52 | return Err(eg!("Memory size too large(max: 48GB) !")); 53 | } 54 | 55 | if 0 < self.disk_size && 100 > self.disk_size { 56 | return Err(eg!("Disk size too small(min: 100) !")); 57 | } 58 | 59 | if 1024 * 1024 < self.disk_size { 60 | return Err(eg!("Disk size too large(max: 1TB) !")); 61 | } 62 | 63 | get_ops_id("add_env") 64 | .c(d!()) 65 | .and_then(|ops_id| { 66 | get_servaddr().c(d!()).and_then(|addr| { 67 | send_req(ops_id, gen_req(ReqAddEnv::from(self)), addr) 68 | .c(d!()) 69 | }) 70 | }) 71 | .and_then(|resp| resp_print!(resp, String)) 72 | } 73 | } 74 | 75 | impl<'a> From> for ReqAddEnv { 76 | fn from(v: EnvAdd<'a>) -> Self { 77 | ReqAddEnv { 78 | env_id: v.env_id.to_owned(), 79 | os_prefix: v.os_prefix.into_iter().map(|s| s.to_owned()).collect(), 80 | life_time: alt!(0 == v.life_time, None, Some(v.life_time)), 81 | cpu_num: alt!(0 == v.cpu_num, None, Some(v.cpu_num as i32)), 82 | mem_size: alt!(0 == v.mem_size, None, Some(v.mem_size as i32)), 83 | disk_size: alt!(0 == v.disk_size, None, Some(v.disk_size as i32)), 84 | port_set: v.vm_port, 85 | dup_each: alt!(0 == v.dup_each, None, Some(v.dup_each as u32)), 86 | deny_outgoing: v.deny_outgoing, 87 | rand_uuid: v.rand_uuid, 88 | vmcfg: None, 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/client/src/ops/env/del.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Env 3 | //! 4 | //! ```shell 5 | //! tt env del ... 6 | //! ``` 7 | //! 8 | 9 | use super::super::*; 10 | use crate::{get_servaddr, resp_print}; 11 | use ruc::*; 12 | 13 | /////////////////////////////// 14 | #[derive(Default)] 15 | pub struct EnvDel<'a> { 16 | pub env_set: Vec<&'a EnvIdRef>, 17 | } 18 | /////////////////////////////// 19 | 20 | impl<'a> EnvDel<'a> { 21 | /// 发送请求并打印结果 22 | pub fn do_req(&self) -> Result<()> { 23 | self.env_set.iter().for_each(|env| { 24 | info_omit!( 25 | get_ops_id("del_env") 26 | .c(d!()) 27 | .and_then(|ops_id| { 28 | get_servaddr().c(d!()).and_then(|addr| { 29 | send_req( 30 | ops_id, 31 | gen_req(ReqDelEnv { 32 | env_id: env.to_string(), 33 | }), 34 | addr, 35 | ) 36 | .c(d!()) 37 | }) 38 | }) 39 | .and_then(|resp| resp_print!(resp, String)) 40 | ) 41 | }); 42 | Ok(()) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/client/src/ops/env/get/mod.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Get file from ENV 3 | //! 4 | //! ```shell 5 | //! tt env get ... 6 | //! ``` 7 | //! 8 | 9 | mod scp; 10 | mod ttrexec; 11 | 12 | use super::{ 13 | super::EnvIdRef, 14 | run::{get_conn_info, print_to_user, VmConnInfo}, 15 | }; 16 | use ruc::*; 17 | use std::{sync::mpsc::Receiver, time}; 18 | 19 | /////////////////////////////// 20 | #[derive(Default)] 21 | pub struct EnvGet<'a> { 22 | pub use_ssh: bool, 23 | pub file_path: &'a str, 24 | pub env_set: Vec<&'a EnvIdRef>, 25 | pub time_out: u64, 26 | pub filter_os_prefix: Vec<&'a str>, 27 | pub filter_vm_id: Vec, 28 | } 29 | /////////////////////////////// 30 | 31 | impl<'a> EnvGet<'a> { 32 | /// 发送请求并打印结果 33 | pub fn do_req(&self) -> Result<()> { 34 | self.get_res().c(d!()).and_then(|(n, r)| { 35 | for _ in 0..n { 36 | r.recv_timeout(time::Duration::from_secs(self.time_out)) 37 | .map(print_to_user) 38 | .c(d!())?; 39 | } 40 | Ok(()) 41 | }) 42 | } 43 | 44 | /// 发送请求并获取结果 45 | pub fn get_res(&self) -> Result<(usize, Receiver)> { 46 | if "" == self.file_path { 47 | return Err(eg!("Empty file_path!")); 48 | } 49 | 50 | if self.env_set.is_empty() { 51 | return Err(eg!("Empty env_set!")); 52 | } 53 | 54 | get_conn_info(&self.env_set).c(d!()).map(|mut vci_set| { 55 | if !self.filter_vm_id.is_empty() 56 | || !self.filter_os_prefix.is_empty() 57 | { 58 | vci_set = vci_set 59 | .into_iter() 60 | .filter(|vci| { 61 | self.filter_vm_id.iter().any(|id| vci.id == *id) 62 | || self.filter_os_prefix.iter().any(|prefix| { 63 | vci.os 64 | .to_lowercase() 65 | .starts_with(&prefix.to_lowercase()) 66 | }) 67 | }) 68 | .collect(); 69 | } 70 | if self.use_ssh { 71 | (vci_set.len(), scp::exec(self.file_path, vci_set)) 72 | } else { 73 | (vci_set.len(), ttrexec::exec(self.file_path, vci_set)) 74 | } 75 | }) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/client/src/ops/env/get/scp.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! A slow implementation based on SSH. 3 | //! 4 | 5 | use super::super::{ 6 | run::{ssh, VmConnInfo}, 7 | POOL, 8 | }; 9 | use ruc::*; 10 | use std::sync::mpsc::{channel, Receiver}; 11 | 12 | /// 执行外部的 scp 命令, 13 | /// 收集远端的输出内容并返回之 14 | pub(super) fn exec( 15 | file_path: &str, 16 | vm_conn_info: Vec, 17 | ) -> Receiver { 18 | let (s, r) = channel(); 19 | 20 | vm_conn_info 21 | .into_iter() 22 | .filter_map(|vci| { 23 | let remote_path = 24 | format!("{}@{}:{}", ssh::USER, &vci.addr, file_path); 25 | let port = vci.ssh_port.to_string(); 26 | let local_file = format!( 27 | "{}{{{}#{}#{}}}", 28 | file_path.rsplitn(2, '/').next().unwrap(), 29 | &vci.os, 30 | vci.addr, 31 | vci.ssh_port, 32 | ); 33 | let args = &["-P", &port, &remote_path, &local_file]; 34 | info!(ssh::do_exec("scp", args).c(d!()).map(|child| (child, vci))) 35 | .ok() 36 | }) 37 | .collect::>() 38 | .into_iter() 39 | .for_each(|(child, mut vci)| { 40 | let sender = s.clone(); 41 | POOL.spawn(move || { 42 | child 43 | .wait_with_output() 44 | .c(d!()) 45 | .map(|output| { 46 | vci.stdout = String::from_utf8_lossy(&output.stdout) 47 | .into_owned(); 48 | vci.stderr = String::from_utf8_lossy(&output.stderr) 49 | .into_owned(); 50 | }) 51 | .unwrap_or_else(|e| { 52 | vci.stderr = genlog(e); 53 | }); 54 | 55 | info_omit!(sender.send(vci)); 56 | }); 57 | }); 58 | 59 | r 60 | } 61 | -------------------------------------------------------------------------------- /src/client/src/ops/env/get/ttrexec.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! A fast implementation based on ttrexec. 3 | //! 4 | 5 | use super::super::{run::VmConnInfo, POOL}; 6 | use ruc::*; 7 | use std::sync::mpsc::{channel, Receiver}; 8 | use ttrexec::{ 9 | client::req_transfer, 10 | common::{Direction, TransReq}, 11 | }; 12 | 13 | pub(super) fn exec( 14 | file_path: &str, 15 | vm_conn_info: Vec, 16 | ) -> Receiver { 17 | let (s, r) = channel(); 18 | 19 | vm_conn_info.into_iter().for_each(|mut vci| { 20 | let fpath = file_path.to_owned(); 21 | let sender = s.clone(); 22 | POOL.spawn(move || { 23 | let local_file = format!( 24 | "{}{{{}#{}#{}}}", 25 | fpath.rsplitn(2, '/').next().unwrap(), 26 | &vci.os, 27 | vci.addr, 28 | vci.ssh_port, 29 | ); 30 | 31 | TransReq::new(Direction::Get, &local_file, &fpath) 32 | .and_then(|req| { 33 | let addr = format!("{}:{}", vci.addr, vci.ttrexec_port); 34 | let resp = req_transfer(&addr, req, None).c(d!())?; 35 | vci.stdout = resp.stdout.into_owned(); 36 | vci.stderr = resp.stderr.into_owned(); 37 | Ok(()) 38 | }) 39 | .unwrap_or_else(|e| { 40 | let log = genlog(e); 41 | println!("{}", &log); 42 | vci.stderr = log; 43 | }); 44 | info_omit!(sender.send(vci)); 45 | }); 46 | }); 47 | 48 | r 49 | } 50 | -------------------------------------------------------------------------------- /src/client/src/ops/env/list.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Env 3 | //! 4 | //! ```shell 5 | //! tt env list 6 | //! ``` 7 | //! 8 | 9 | use super::super::*; 10 | use crate::{get_servaddr, resp_parse, resp_print}; 11 | use ruc::*; 12 | use std::collections::HashMap; 13 | 14 | /////////////////////////////// 15 | pub struct EnvList; 16 | /////////////////////////////// 17 | 18 | impl EnvList { 19 | /// 发送请求并打印结果 20 | pub fn do_req(&self) -> Result<()> { 21 | self.get_res().c(d!()).map(|r| resp_print!(r)) 22 | } 23 | 24 | /// 发送请求并获取结果 25 | pub fn get_res(&self) -> Result> { 26 | get_ops_id("get_env_list") 27 | .c(d!()) 28 | .and_then(|ops_id| { 29 | get_servaddr().c(d!()).and_then(|addr| { 30 | send_req::<&str>(ops_id, gen_req(""), addr).c(d!()) 31 | }) 32 | }) 33 | .and_then( 34 | |resp| resp_parse!(resp, HashMap), 35 | ) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/client/src/ops/env/listall.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Env 3 | //! 4 | //! ```shell 5 | //! tt env listall 6 | //! ``` 7 | //! 8 | 9 | use super::super::*; 10 | use crate::{get_servaddr, resp_parse, resp_print}; 11 | use ruc::*; 12 | use std::collections::HashMap; 13 | 14 | /////////////////////////////// 15 | pub struct EnvListAll; 16 | /////////////////////////////// 17 | 18 | impl EnvListAll { 19 | /// 发送请求并打印结果 20 | pub fn do_req(&self) -> Result<()> { 21 | self.get_res().c(d!()).map(|r| resp_print!(r)) 22 | } 23 | 24 | /// 发送请求并获取结果 25 | pub fn get_res(&self) -> Result> { 26 | get_ops_id("get_env_list_all") 27 | .c(d!()) 28 | .and_then(|ops_id| { 29 | get_servaddr().c(d!()).and_then(|addr| { 30 | send_req::<&str>(ops_id, gen_req(""), addr).c(d!()) 31 | }) 32 | }) 33 | .and_then( 34 | |resp| resp_parse!(resp, HashMap), 35 | ) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/client/src/ops/env/mod.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Env Request 3 | //! 4 | 5 | pub mod add; 6 | pub mod del; 7 | pub mod get; 8 | pub mod list; 9 | pub mod listall; 10 | pub mod push; 11 | pub mod run; 12 | pub mod show; 13 | pub mod start; 14 | pub mod stop; 15 | pub mod update; 16 | 17 | pub use add::*; 18 | pub use del::*; 19 | pub use get::*; 20 | pub use list::*; 21 | pub use listall::*; 22 | pub use push::*; 23 | pub use run::*; 24 | pub use show::*; 25 | pub use start::*; 26 | pub use stop::*; 27 | pub use update::*; 28 | 29 | use std::sync::{Arc, Mutex, LazyLock}; 30 | use threadpool::ThreadPool; 31 | 32 | // 传输文件时并发太多没有意义, 33 | // 使用小规模的线程池限制并发数量 34 | static POOL: LazyLock = LazyLock::new(|| Pool::new(3)); 35 | 36 | struct Pool { 37 | inner: Arc>, 38 | } 39 | 40 | impl Pool { 41 | #[inline(always)] 42 | fn new(n: usize) -> Pool { 43 | Pool { 44 | inner: Arc::new(Mutex::new(ThreadPool::new(n))), 45 | } 46 | } 47 | 48 | #[inline(always)] 49 | fn spawn(&self, job: F) 50 | where 51 | F: FnOnce() + Send + 'static, 52 | { 53 | self.inner.lock().unwrap().execute(job) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/client/src/ops/env/push/mod.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Push file to ENV 3 | //! 4 | //! ```shell 5 | //! tt env push ... 6 | //! ``` 7 | //! 8 | 9 | mod scp; 10 | mod ttrexec; 11 | 12 | use super::{ 13 | super::EnvIdRef, 14 | run::{get_conn_info, print_to_user, VmConnInfo}, 15 | }; 16 | use ruc::*; 17 | use std::{sync::mpsc::Receiver, time}; 18 | 19 | /////////////////////////////// 20 | #[derive(Default)] 21 | pub struct EnvPush<'a> { 22 | pub use_ssh: bool, 23 | pub file_path: &'a str, 24 | pub env_set: Vec<&'a EnvIdRef>, 25 | pub time_out: u64, 26 | pub filter_os_prefix: Vec<&'a str>, 27 | pub filter_vm_id: Vec, 28 | } 29 | /////////////////////////////// 30 | 31 | impl<'a> EnvPush<'a> { 32 | /// 发送请求并打印结果 33 | pub fn do_req(&self) -> Result<()> { 34 | self.get_res().c(d!()).and_then(|(n, r)| { 35 | for _ in 0..n { 36 | r.recv_timeout(time::Duration::from_secs(self.time_out)) 37 | .map(print_to_user) 38 | .c(d!())?; 39 | } 40 | Ok(()) 41 | }) 42 | } 43 | 44 | /// 发送请求并获取结果 45 | pub fn get_res(&self) -> Result<(usize, Receiver)> { 46 | if "" == self.file_path { 47 | return Err(eg!("Empty file_path!")); 48 | } 49 | 50 | if self.env_set.is_empty() { 51 | return Err(eg!("Empty env_set!")); 52 | } 53 | 54 | get_conn_info(&self.env_set).c(d!()).map(|mut vci_set| { 55 | if !self.filter_vm_id.is_empty() 56 | || !self.filter_os_prefix.is_empty() 57 | { 58 | vci_set = vci_set 59 | .into_iter() 60 | .filter(|vci| { 61 | self.filter_vm_id.iter().any(|id| vci.id == *id) 62 | || self.filter_os_prefix.iter().any(|prefix| { 63 | vci.os 64 | .to_lowercase() 65 | .starts_with(&prefix.to_lowercase()) 66 | }) 67 | }) 68 | .collect(); 69 | } 70 | if self.use_ssh { 71 | (vci_set.len(), scp::exec(self.file_path, vci_set)) 72 | } else { 73 | (vci_set.len(), ttrexec::exec(self.file_path, vci_set)) 74 | } 75 | }) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/client/src/ops/env/push/scp.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! A slow implementation based on SSH. 3 | //! 4 | 5 | use super::super::{ 6 | run::{ssh, VmConnInfo}, 7 | POOL, 8 | }; 9 | use ruc::*; 10 | use std::sync::mpsc::{channel, Receiver}; 11 | 12 | /// 执行外部的 scp 命令, 13 | /// 收集远端的输出内容并返回之 14 | pub(super) fn exec( 15 | file_path: &str, 16 | vm_conn_info: Vec, 17 | ) -> Receiver { 18 | let (s, r) = channel(); 19 | 20 | vm_conn_info 21 | .into_iter() 22 | .filter_map(|vci| { 23 | let remote_path = format!("{}@{}:/tmp/", ssh::USER, &vci.addr); 24 | let port = vci.ssh_port.to_string(); 25 | let args = &["-P", &port, &file_path, &remote_path]; 26 | info!(ssh::do_exec("scp", args).c(d!()).map(|child| (child, vci))) 27 | .ok() 28 | }) 29 | .collect::>() 30 | .into_iter() 31 | .for_each(|(child, mut vci)| { 32 | let sender = s.clone(); 33 | POOL.spawn(move || { 34 | child 35 | .wait_with_output() 36 | .c(d!()) 37 | .map(|output| { 38 | vci.stdout = String::from_utf8_lossy(&output.stdout) 39 | .into_owned(); 40 | vci.stderr = String::from_utf8_lossy(&output.stderr) 41 | .into_owned(); 42 | }) 43 | .unwrap_or_else(|e| { 44 | vci.stderr = genlog(e); 45 | }); 46 | 47 | info_omit!(sender.send(vci)); 48 | }); 49 | }); 50 | 51 | r 52 | } 53 | -------------------------------------------------------------------------------- /src/client/src/ops/env/push/ttrexec.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! A fast implementation based on ttrexec. 3 | //! 4 | 5 | use super::super::{run::VmConnInfo, POOL}; 6 | use ruc::*; 7 | use std::sync::mpsc::{channel, Receiver}; 8 | use ttrexec::{ 9 | client::req_transfer, 10 | common::{Direction, TransReq}, 11 | }; 12 | 13 | pub(super) fn exec( 14 | file_path: &str, 15 | vm_conn_info: Vec, 16 | ) -> Receiver { 17 | let (s, r) = channel(); 18 | 19 | vm_conn_info.into_iter().for_each(|mut vci| { 20 | let fpath = file_path.to_owned(); 21 | let sender = s.clone(); 22 | POOL.spawn(move || { 23 | TransReq::new( 24 | Direction::Push, 25 | &fpath, 26 | &format!("/tmp/{}", fpath.rsplitn(2, '/').next().unwrap()), 27 | ) 28 | .and_then(|req| { 29 | let addr = format!("{}:{}", vci.addr, vci.ttrexec_port); 30 | let resp = req_transfer(&addr, req, None).c(d!())?; 31 | vci.stdout = resp.stdout.into_owned(); 32 | vci.stderr = resp.stderr.into_owned(); 33 | Ok(()) 34 | }) 35 | .unwrap_or_else(|e| { 36 | vci.stderr = genlog(e); 37 | }); 38 | 39 | info_omit!(sender.send(vci)); 40 | }); 41 | }); 42 | 43 | r 44 | } 45 | -------------------------------------------------------------------------------- /src/client/src/ops/env/run/ssh.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! A slow implementation based on SSH. 3 | //! 4 | 5 | use super::VmConnInfo; 6 | use crate::{SSH_VM_KEY, genlog}; 7 | use ruc::*; 8 | use nix::unistd::execv; 9 | use std::ffi::CString; 10 | use std::{ 11 | process::{Child, Command, Stdio}, 12 | sync::mpsc::{channel, Receiver}, 13 | thread, 14 | }; 15 | 16 | pub const USER: &str = "root"; 17 | 18 | /// Execute external ssh command, 19 | /// Collect remote output content and return it 20 | #[allow(clippy::mutex_atomic)] 21 | pub(super) fn exec( 22 | remote_cmd: &str, 23 | vm_conn_info: Vec, 24 | ) -> Receiver { 25 | let (s, r) = channel(); 26 | 27 | vm_conn_info.into_iter().for_each(|mut vci| { 28 | let conninfo = format!("{}@{}", USER, &vci.addr); 29 | let port = vci.ssh_port.to_string(); 30 | let cmd = remote_cmd.to_owned(); 31 | let args = ["-p".to_owned(), port, conninfo, cmd].to_vec(); 32 | 33 | let sender = s.clone(); 34 | thread::spawn(move || { 35 | exec_run("ssh", args) 36 | .c(d!()) 37 | .and_then(|child| { 38 | child.wait_with_output().c(d!()).map(|output| { 39 | vci.stdout = String::from_utf8_lossy(&output.stdout) 40 | .into_owned(); 41 | vci.stderr = String::from_utf8_lossy(&output.stderr) 42 | .into_owned(); 43 | vci.status_code = output.status.code().unwrap_or(255); 44 | }) 45 | }) 46 | .unwrap_or_else(|e| { 47 | vci.stderr = genlog(e); 48 | vci.status_code = 255; 49 | }); 50 | 51 | info_omit!(sender.send(vci)); 52 | }); 53 | }); 54 | 55 | r 56 | } 57 | 58 | #[inline(always)] 59 | fn exec_run(cmd: &'static str, args: Vec) -> Result { 60 | do_exec( 61 | &cmd, 62 | args.iter() 63 | .map(|s| s.as_str()) 64 | .collect::>() 65 | .as_slice(), 66 | ) 67 | } 68 | 69 | #[inline(always)] 70 | pub fn do_exec(cmd: &str, args: &[&str]) -> Result { 71 | Command::new(cmd) 72 | .args(&["-o", "ConnectTimeout=3", "-i", SSH_VM_KEY.as_str()]) 73 | .args(args) 74 | .stdout(Stdio::piped()) 75 | .stderr(Stdio::piped()) 76 | .spawn() 77 | .c(d!()) 78 | } 79 | 80 | /// Serial execution of SSH interactive operations 81 | #[allow(clippy::mutex_atomic)] 82 | pub(super) fn exec_interactive(vm_conn_info: Vec) -> ! { 83 | pnk!( 84 | gen_interactive_cmds(&vm_conn_info) 85 | .c(d!()) 86 | .and_then(|cmds| execv( 87 | &CString::new("/bin/sh").unwrap(), 88 | &[ 89 | &CString::new("/bin/sh").unwrap(), 90 | &CString::new("-c").unwrap(), 91 | &cmds 92 | ] 93 | ) 94 | .c(d!())) 95 | ); 96 | 97 | panic!() 98 | } 99 | 100 | /// Generate interactive SSH serial command set 101 | fn gen_interactive_cmds(vm_conn_info: &[VmConnInfo]) -> Result { 102 | let mut res = vct![]; 103 | vm_conn_info.iter().for_each(|vci| { 104 | res.push( 105 | format!("printf '\x1b[31;01mConnecting: {}\x1b[00m\n'; ssh -p {} -i {} -o ConnectTimeout=3 {}@{};", 106 | vci.os, 107 | vci.ssh_port, 108 | SSH_VM_KEY.as_str(), 109 | USER, 110 | vci.addr 111 | )); 112 | }); 113 | CString::new(res.join("")).c(d!()) 114 | } 115 | -------------------------------------------------------------------------------- /src/client/src/ops/env/run/ttrexec.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! A fast implementation based on ttrexec. 3 | //! 4 | 5 | use super::{ssh::USER, VmConnInfo}; 6 | use ruc::*; 7 | use std::{ 8 | sync::mpsc::{channel, Receiver}, 9 | thread, 10 | }; 11 | use ttrexec::client::req_exec; 12 | 13 | pub(crate) fn exec( 14 | remote_cmd: &str, 15 | vm_conn_info: Vec, 16 | ) -> Receiver { 17 | let (s, r) = channel(); 18 | vm_conn_info.into_iter().for_each(|mut vci| { 19 | // keep the same-workdir with SSH 20 | let cmd = format!("cd ~{};{}", USER, remote_cmd); 21 | let sender = s.clone(); 22 | thread::spawn(move || { 23 | match req_exec(&format!("{}:{}", vci.addr, vci.ttrexec_port), &cmd) 24 | { 25 | Ok(resp) => { 26 | vci.stdout = resp.stdout.into_owned(); 27 | vci.stderr = resp.stderr.into_owned(); 28 | vci.status_code = resp.code; 29 | } 30 | Err(e) => { 31 | vci.stderr = genlog(e); 32 | vci.status_code = 255; 33 | } 34 | } 35 | info_omit!(sender.send(vci)); 36 | }); 37 | }); 38 | 39 | r 40 | } 41 | -------------------------------------------------------------------------------- /src/client/src/ops/env/show.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Env 3 | //! 4 | //! ```shell 5 | //! tt env show ... 6 | //! ``` 7 | //! 8 | 9 | use super::super::*; 10 | use crate::{get_servaddr, resp_parse, resp_print}; 11 | use ruc::*; 12 | use std::collections::BTreeMap; 13 | 14 | /////////////////////////////// 15 | #[derive(Default)] 16 | pub struct EnvShow<'a> { 17 | /// 目前会忽略此参数, 总是返回所有 Env 的信息 18 | pub env_set: Vec<&'a EnvIdRef>, 19 | } 20 | /////////////////////////////// 21 | 22 | impl<'a> EnvShow<'a> { 23 | /// 发送请求并打印结果 24 | #[inline(always)] 25 | pub fn do_req(self) -> Result<()> { 26 | get_res(&self.env_set).map(|r| resp_print!(r)) 27 | } 28 | } 29 | 30 | /// 拆出此功能函数供 run 和 deploy 复用 31 | pub(super) fn get_res( 32 | env_set: &[&EnvIdRef], 33 | ) -> Result> { 34 | get_ops_id("get_env_info") 35 | .c(d!()) 36 | .and_then(|ops_id| get_servaddr().c(d!()).map(|addr| (ops_id, addr))) 37 | .and_then(|(ops_id, addr)| { 38 | send_req( 39 | ops_id, 40 | gen_req(ReqGetEnvInfo { 41 | env_set: env_set.iter().map(|id| id.to_string()).collect(), 42 | }), 43 | addr, 44 | ) 45 | .c(d!()) 46 | }) 47 | .and_then( 48 | |resp| resp_parse!(resp, BTreeMap), 49 | ) 50 | } 51 | -------------------------------------------------------------------------------- /src/client/src/ops/env/start.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Env 3 | //! 4 | //! ```shell 5 | //! tt env del ... 6 | //! ``` 7 | //! 8 | 9 | use super::super::*; 10 | use crate::{get_servaddr, resp_print}; 11 | use ruc::*; 12 | 13 | /////////////////////////////// 14 | #[derive(Default)] 15 | pub struct EnvStart<'a> { 16 | pub env_set: Vec<&'a EnvIdRef>, 17 | } 18 | /////////////////////////////// 19 | 20 | impl<'a> EnvStart<'a> { 21 | /// 发送请求并打印结果 22 | pub fn do_req(&self) -> Result<()> { 23 | self.env_set.iter().for_each(|env| { 24 | info_omit!( 25 | get_ops_id("start_env") 26 | .c(d!()) 27 | .and_then(|ops_id| { 28 | get_servaddr().c(d!()).and_then(|addr| { 29 | send_req( 30 | ops_id, 31 | gen_req(ReqStartEnv { 32 | env_id: env.to_string(), 33 | }), 34 | addr, 35 | ) 36 | .c(d!()) 37 | }) 38 | }) 39 | .and_then(|resp| resp_print!(resp, String)) 40 | ) 41 | }); 42 | Ok(()) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/client/src/ops/env/stop.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Env 3 | //! 4 | //! ```shell 5 | //! tt env del ... 6 | //! ``` 7 | //! 8 | 9 | use super::super::*; 10 | use crate::{get_servaddr, resp_print}; 11 | use ruc::*; 12 | 13 | /////////////////////////////// 14 | #[derive(Default)] 15 | pub struct EnvStop<'a> { 16 | pub env_set: Vec<&'a EnvIdRef>, 17 | } 18 | /////////////////////////////// 19 | 20 | impl<'a> EnvStop<'a> { 21 | /// 发送请求并打印结果 22 | pub fn do_req(&self) -> Result<()> { 23 | self.env_set.iter().for_each(|env| { 24 | info_omit!( 25 | get_ops_id("stop_env") 26 | .c(d!()) 27 | .and_then(|ops_id| { 28 | get_servaddr().c(d!()).and_then(|addr| { 29 | send_req( 30 | ops_id, 31 | gen_req(ReqStopEnv { 32 | env_id: env.to_string(), 33 | }), 34 | addr, 35 | ) 36 | .c(d!()) 37 | }) 38 | }) 39 | .and_then(|resp| resp_print!(resp, String)) 40 | ) 41 | }); 42 | Ok(()) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/client/src/ops/env/update.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Env 3 | //! 4 | //! ```shell 5 | //! tt env update --kick-vm=[OS_PREFIX] --update-life=[SECS] ... 6 | //! ``` 7 | //! 8 | 9 | use super::{super::*, run}; 10 | use crate::{get_servaddr, resp_print}; 11 | use ruc::*; 12 | use std::{collections::HashSet, mem, time}; 13 | 14 | /////////////////////////////// 15 | #[derive(Default)] 16 | pub struct EnvUpdate<'a> { 17 | pub env_set: Vec<&'a EnvIdRef>, 18 | pub cpu_num: Option, 19 | pub mem_size: Option, 20 | pub vm_port: Vec, 21 | /// for kick-vm 22 | pub kick_dead: bool, 23 | /// for kick-vm 24 | pub vm_id: Vec, 25 | /// for kick-vm 26 | pub os_prefix: Vec, 27 | /// for life-time 28 | pub life_time: Option, 29 | /// for life-time 30 | pub is_fucker: bool, 31 | /// deny VM to do outgoing network-ops 32 | pub deny_outgoing: Option, 33 | } 34 | /////////////////////////////// 35 | 36 | impl<'a> EnvUpdate<'a> { 37 | /// 发送请求并打印结果 38 | pub fn do_req(mut self) -> Result<()> { 39 | mem::take(&mut self.env_set).iter().for_each(|env| { 40 | if let Some(life_time) = self.life_time { 41 | let res = get_ops_id("update_env_lifetime") 42 | .c(d!()) 43 | .and_then(|ops_id| { 44 | get_servaddr().c(d!()).and_then(|addr| { 45 | send_req( 46 | ops_id, 47 | gen_req(ReqUpdateEnvLife { 48 | env_id: env.to_string(), 49 | life_time, 50 | is_fucker: self.is_fucker, 51 | }), 52 | addr, 53 | ) 54 | .c(d!()) 55 | }) 56 | }) 57 | .and_then(|resp| resp_print!(resp, String)); 58 | info_omit!(res); 59 | } 60 | 61 | if !self.vm_port.is_empty() 62 | || self.cpu_num.is_some() 63 | || self.mem_size.is_some() 64 | || self.deny_outgoing.is_some() 65 | { 66 | let res = get_ops_id("update_env_resource") 67 | .c(d!()) 68 | .and_then(|ops_id| { 69 | get_servaddr().c(d!()).and_then(|addr| { 70 | send_req( 71 | ops_id, 72 | gen_req(ReqUpdateEnvResource { 73 | env_id: env.to_string(), 74 | cpu_num: self.cpu_num.map(|n| n as i32), 75 | mem_size: self.mem_size.map(|i| i as i32), 76 | disk_size: None, 77 | vm_port: self.vm_port.clone(), 78 | deny_outgoing: self.deny_outgoing, 79 | }), 80 | addr, 81 | ) 82 | .c(d!()) 83 | }) 84 | }) 85 | .and_then(|resp| resp_print!(resp, String)); 86 | info_omit!(res); 87 | } 88 | 89 | if !self.os_prefix.is_empty() || !self.vm_id.is_empty() { 90 | let res = get_ops_id("update_env_kick_vm") 91 | .c(d!()) 92 | .and_then(|ops_id| { 93 | get_servaddr().c(d!()).and_then(|addr| { 94 | send_req( 95 | ops_id, 96 | gen_req(ReqUpdateEnvKickVm { 97 | env_id: env.to_string(), 98 | vm_id: self.vm_id.clone(), 99 | os_prefix: self.os_prefix.clone(), 100 | }), 101 | addr, 102 | ) 103 | .c(d!()) 104 | }) 105 | }) 106 | .and_then(|resp| resp_print!(resp, String)); 107 | info_omit!(res); 108 | } 109 | 110 | // - `tt env run -c 'date' -t 5`, 111 | // - 收集所有超时未返回的实例集合, 112 | // - `tt env update --kick-vm=...` 113 | if self.kick_dead { 114 | let res = 115 | run::get_conn_info(&[env]).c(d!()).and_then(|vci_set| { 116 | let vmid_set = vci_set 117 | .iter() 118 | .map(|vci| vci.id) 119 | .collect::>(); 120 | 121 | let hdr = run::ttrexec::exec("date", vci_set); 122 | let running_set = if hdr 123 | .recv_timeout(time::Duration::from_secs(8)) 124 | .is_err() 125 | { 126 | HashSet::new() 127 | } else { 128 | (1..vmid_set.len()) 129 | .filter_map(|_| { 130 | hdr.recv_timeout( 131 | time::Duration::from_secs(5), 132 | ) 133 | .ok() 134 | }) 135 | .filter(|vci| 0 == vci.status_code) 136 | .map(|vci| vci.id) 137 | .collect::>() 138 | }; 139 | 140 | let to_kick = vmid_set 141 | .difference(&running_set) 142 | .copied() 143 | .collect::>(); 144 | 145 | EnvUpdate { 146 | env_set: vct![&env], 147 | cpu_num: None, 148 | mem_size: None, 149 | vm_port: vct![], 150 | kick_dead: false, 151 | vm_id: to_kick, 152 | os_prefix: vct![], 153 | life_time: None, 154 | is_fucker: false, 155 | deny_outgoing: None, 156 | } 157 | .do_req() 158 | .c(d!()) 159 | }); 160 | info_omit!(res); 161 | } 162 | }); 163 | 164 | Ok(()) 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/client/src/ops/mod.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! 请求类别, 对应服务端的处理函数: 3 | //! 4 | //! ```rust 5 | //! const OPS_MAP: &[Ops] = &[ 6 | //! register_client_id, 7 | //! get_server_info, 8 | //! get_env_list, 9 | //! get_env_info, 10 | //! add_env, 11 | //! del_env, 12 | //! update_env_lifetime, 13 | //! update_env_kick_vm, 14 | //! stop_env, 15 | //! start_env, 16 | //! update_env_resource, 17 | //! ]; 18 | //! ``` 19 | //! 20 | 21 | pub mod config; 22 | pub mod env; 23 | pub mod status; 24 | 25 | use crate::CFG; 26 | use ruc::*; 27 | use serde::Serialize; 28 | use std::{ 29 | collections::HashMap, 30 | net::{SocketAddr, UdpSocket}, 31 | sync::LazyLock, 32 | time::Duration, 33 | }; 34 | pub(self) use ttserver_def::*; 35 | use ttutils::zlib; 36 | 37 | static OPS_MAP: LazyLock> = LazyLock::new(|| map! { 38 | "register_client_id" => 0, 39 | "get_server_info" => 1, 40 | "get_env_list" => 2, 41 | "get_env_info" => 3, 42 | "add_env" => 4, 43 | "del_env" => 5, 44 | "update_env_lifetime" => 6, 45 | "update_env_kick_vm" => 7, 46 | "get_env_list_all" => 8, 47 | "stop_env" => 9, 48 | "start_env" => 10, 49 | "update_env_resource" => 11, 50 | }); 51 | static SOCK: LazyLock = LazyLock::new(|| pnk!(gen_sock(8))); 52 | 53 | /// 解析返回的结果 54 | #[macro_export] 55 | macro_rules! resp_parse { 56 | ($resp_orig: expr, $body_type: ty) => { 57 | match $resp_orig.status { 58 | RetStatus::Success => { 59 | serde_json::from_slice::<$body_type>(&$resp_orig.msg).c(d!()) 60 | } 61 | RetStatus::Fail => { 62 | Err(eg!(String::from_utf8_lossy(&$resp_orig.msg))) 63 | } 64 | } 65 | }; 66 | } 67 | 68 | /// 打印返回的结果 69 | #[macro_export] 70 | macro_rules! resp_print { 71 | ($body: expr) => {{ 72 | dbg!($body); 73 | }}; 74 | ($resp_orig: expr, $body_type: ty) => { 75 | $crate::resp_parse!($resp_orig, $body_type).map(|body| { 76 | dbg!(body); 77 | }) 78 | }; 79 | } 80 | 81 | #[inline(always)] 82 | fn get_ops_id(ops: &str) -> Result { 83 | OPS_MAP 84 | .get(ops) 85 | .copied() 86 | .ok_or(eg!(format!("Unknown request: {}", ops))) 87 | } 88 | 89 | fn gen_sock(timeout: u64) -> Result { 90 | let mut addr; 91 | for port in (20_000 + ts!() % 10_000)..60_000 { 92 | addr = SocketAddr::from(([0, 0, 0, 0], port as u16)); 93 | if let Ok(sock) = UdpSocket::bind(addr) { 94 | sock.set_read_timeout(Some(Duration::from_secs(timeout))) 95 | .c(d!())?; 96 | return Ok(sock); 97 | } 98 | } 99 | Err(eg!()) 100 | } 101 | 102 | /// 发送请求信息 103 | pub fn send_req( 104 | ops_id: u8, 105 | req: Req, 106 | peeraddr: SocketAddr, 107 | ) -> Result { 108 | let mut req_bytes = serde_json::to_vec(&req) 109 | .c(d!()) 110 | .and_then(|req| zlib::encode(&req[..]).c(d!()))?; 111 | let mut body = 112 | format!("{id:>0width$}", id = ops_id, width = OPS_ID_LEN).into_bytes(); 113 | body.append(&mut req_bytes); 114 | 115 | SOCK.send_to(&body, peeraddr).c(d!()).and_then(|_| { 116 | // At most 64KB can be sent on UDP(inet/inet6) 117 | let mut buf = vct![0; 64 * 1024]; 118 | let size = SOCK.recv(&mut buf).c(d!())?; 119 | zlib::decode(&buf[0..size]) 120 | .c(d!()) 121 | .and_then(|data| serde_json::from_slice(&data).c(d!())) 122 | }) 123 | } 124 | 125 | /// 生成 Req 结构 126 | #[inline(always)] 127 | pub fn gen_req(msg: T) -> Req { 128 | Req::new(0, CFG.client_id.clone(), msg) 129 | } 130 | -------------------------------------------------------------------------------- /src/client/src/ops/status.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Status Request 3 | //! 4 | 5 | use super::*; 6 | use crate::{get_servaddr, resp_parse, resp_print, CFG}; 7 | use ruc::*; 8 | use std::collections::HashMap; 9 | 10 | /////////////////////////////// 11 | #[derive(Default)] 12 | pub struct Status { 13 | #[allow(dead_code)] 14 | pub client: bool, 15 | pub server: bool, 16 | } 17 | /////////////////////////////// 18 | 19 | impl Status { 20 | /// 发送请求并打印结果 21 | pub fn do_req(&self) -> Result<()> { 22 | if self.server { 23 | self.get_res().c(d!()).map(|mut r| { 24 | r.values_mut().for_each(|si| { 25 | si.supported_list.sort(); 26 | }); 27 | resp_print!(r) 28 | }) 29 | } else { 30 | CFG.print_to_user(); 31 | Ok(()) 32 | } 33 | } 34 | 35 | fn get_res(&self) -> Result> { 36 | let addr = get_servaddr().c(d!())?; 37 | get_ops_id("get_server_info") 38 | .c(d!()) 39 | .and_then(|ops_id| { 40 | send_req::<&str>(ops_id, gen_req(""), addr).c(d!()) 41 | }) 42 | .and_then(|resp| resp_parse!(resp, HashMap)) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/core/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ttcore" 3 | version = "0.3.2" 4 | 5 | edition = "2024" 6 | authors = ["FanHui ", "FanHui "] 7 | description = "Lightweight private cloud solution for SME scenarios." 8 | license = "MIT OR Apache-2.0" 9 | keywords = ["firecracker", "qemu", "kvm", "openstack", "k8s", "cloud"] 10 | categories = ["command-line-utilities"] 11 | 12 | [dependencies] 13 | nix = { version = "0.29.0", features = ["socket", "uio", "zerocopy", "fs"] } 14 | libc = "0.2.167" 15 | parking_lot = "0.12.3" 16 | ttcore_def = { path = "../core_def" } 17 | 18 | base64 = "0.22.1" 19 | serde_json = "1.0.133" 20 | serde = { version = "1.0.216", features = ["derive"] } 21 | 22 | ruc = "8.1.2" 23 | futures = { version = "0.3.31", features = ["thread-pool"] } 24 | futures-timer = "3.0.3" 25 | 26 | [features] 27 | default = [ "nft", "zfs" ] 28 | cow = [] 29 | nft = [] 30 | zfs = [] 31 | testmock = [] 32 | -------------------------------------------------------------------------------- /src/core/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # TT Core Implementation 3 | //! 4 | 5 | #![warn(missing_docs, unused_import_braces, unused_extern_crates)] 6 | 7 | mod def; 8 | pub use def::*; 9 | 10 | #[cfg(all(target_os = "linux", not(feature = "testmock")))] 11 | mod linux; 12 | #[cfg(all(target_os = "linux", not(feature = "testmock")))] 13 | pub use linux::*; 14 | 15 | #[cfg(all(target_os = "linux", not(feature = "testmock")))] 16 | mod common { 17 | use crate::Vm; 18 | use futures::executor::{ThreadPool, ThreadPoolBuilder}; 19 | use ruc::*; 20 | use std::{path::PathBuf, sync::LazyLock}; 21 | 22 | pub(crate) const CLONE_MARK: &str = "clone_"; 23 | 24 | pub(crate) static POOL: LazyLock = 25 | LazyLock::new(|| pnk!(ThreadPoolBuilder::new().pool_size(1).create())); 26 | 27 | pub(crate) async fn asleep(sec: u64) { 28 | futures_timer::Delay::new(std::time::Duration::from_secs(sec)).await; 29 | } 30 | 31 | #[cfg(feature = "zfs")] 32 | pub(crate) static ZFS_ROOT: LazyLock<&'static str> = 33 | LazyLock::new(|| pnk!(imgroot_register(None))); 34 | 35 | #[cfg(feature = "zfs")] 36 | pub(crate) fn imgroot_register( 37 | imgpath: Option<&str>, 38 | ) -> Option<&'static str> { 39 | static mut ROOT: Option = None; 40 | if let Some(path) = imgpath { 41 | unsafe { 42 | ROOT.replace( 43 | path.trim_start_matches("/dev/zvol/") 44 | .trim_end_matches('/') 45 | .to_owned(), 46 | ); 47 | } 48 | } 49 | 50 | unsafe { ROOT.as_deref() } 51 | } 52 | 53 | // VM image naming format: 54 | // - ${CLONE_MARK}_VmId 55 | #[inline(always)] 56 | pub(crate) fn vmimg_path(vm: &Vm) -> PathBuf { 57 | let mut vmimg_path = vm.image_path.clone(); 58 | let vmimg_name = format!("{}{}", CLONE_MARK, vm.id); 59 | vmimg_path.set_file_name(vmimg_name); 60 | vmimg_path 61 | } 62 | } 63 | 64 | #[cfg(all(target_os = "linux", not(feature = "testmock")))] 65 | pub(crate) use common::*; 66 | 67 | mod test; 68 | 69 | // Use mocker for testmock feature OR non-Linux platforms 70 | #[cfg(any(feature = "testmock", not(target_os = "linux")))] 71 | mod mocker; 72 | #[cfg(any(feature = "testmock", not(target_os = "linux")))] 73 | pub use mocker::*; 74 | 75 | // Provide common constants for non-Linux platforms 76 | #[cfg(not(target_os = "linux"))] 77 | mod non_linux_common { 78 | use crate::Vm; 79 | use std::path::PathBuf; 80 | 81 | pub(crate) const CLONE_MARK: &str = "clone_"; 82 | 83 | // VM image naming format: 84 | // - ${CLONE_MARK}_VmId 85 | #[inline(always)] 86 | pub(crate) fn vmimg_path(vm: &Vm) -> PathBuf { 87 | let mut vmimg_path = vm.image_path.clone(); 88 | let vmimg_name = format!("{}{}", CLONE_MARK, vm.id()); 89 | vmimg_path.set_file_name(vmimg_name); 90 | vmimg_path 91 | } 92 | } 93 | 94 | 95 | -------------------------------------------------------------------------------- /src/core/src/linux/mod.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # tt, Temporary Test. 3 | //! 4 | //! core 模块实现服务端的核心逻辑. 5 | //! 6 | //! use Qemu\FireCracker + Nftables on Linux. 7 | //! 8 | 9 | pub(crate) mod nat; 10 | pub(crate) mod vm; 11 | 12 | #[cfg(feature = "zfs")] 13 | use crate::{imgroot_register, CLONE_MARK}; 14 | use crate::{ImagePath, OsName, Vm, VmId, VmKind}; 15 | use ruc::*; 16 | use nix::sched::{clone, CloneFlags}; 17 | use std::collections::HashMap; 18 | use std::{ 19 | fs, 20 | path::{Path, PathBuf}, 21 | }; 22 | 23 | ///////////////// 24 | // Entry Point // 25 | ///////////////// 26 | 27 | /// 全局入口, 必须首先调用 28 | #[cfg(feature = "zfs")] 29 | #[inline(always)] 30 | pub fn exec( 31 | imgpath: &str, 32 | cb: fn() -> Result<()>, 33 | serv_ip: &str, 34 | ) -> Result<()> { 35 | imgroot_register(Some(imgpath)); 36 | do_exec(cb, serv_ip).c(d!()) 37 | } 38 | 39 | /// 全局入口, 必须首先调用 40 | #[cfg(not(feature = "zfs"))] 41 | #[inline(always)] 42 | pub fn exec( 43 | _imgpath: &str, 44 | cb: fn() -> Result<()>, 45 | serv_ip: &str, 46 | ) -> Result<()> { 47 | do_exec(cb, serv_ip).c(d!()) 48 | } 49 | 50 | fn do_exec(cb: fn() -> Result<()>, serv_ip: &str) -> Result<()> { 51 | const STACK_SIZ: usize = 1024 * 1024; 52 | let mut stack = Vec::with_capacity(STACK_SIZ); 53 | unsafe { 54 | stack.set_len(STACK_SIZ); 55 | } 56 | 57 | let mut flags = CloneFlags::empty(); 58 | flags.insert(CloneFlags::CLONE_NEWNS); 59 | flags.insert(CloneFlags::CLONE_NEWPID); 60 | 61 | let ops = || -> isize { 62 | info!( 63 | vm::util::mount_make_rprivate() 64 | .c(d!()) 65 | .and_then(|_| vm::util::mount_dynfs_proc().c(d!())) 66 | .and_then(|_| vm::util::mount_tmp_tmpfs().c(d!())) 67 | .and_then(|_| vm::engine::init().c(d!())) 68 | .and_then(|_| vm::cgroup::init().c(d!())) 69 | .and_then(|_| nat::init(serv_ip).c(d!())) 70 | .and_then(|_| cb().c(d!())) 71 | ) 72 | .and(Ok(0)) 73 | .or::>(Ok(-1)) 74 | .unwrap() 75 | }; 76 | 77 | clone( 78 | Box::new(ops), 79 | stack.as_mut_slice(), 80 | flags, 81 | Some(libc::SIGCHLD), 82 | ) 83 | .c(d!()) 84 | .map(|_| ()) 85 | } 86 | 87 | ////////////////// 88 | // Support List // 89 | ////////////////// 90 | 91 | /// 获取服务端支持的系统列表和对应的 Vm 镜像路径, 92 | /// 排除基础快照、镜像内部分区、Clone 临时镜像三类对象 93 | #[cfg(feature = "zfs")] 94 | pub fn get_os_info(img_path: &str) -> Result> { 95 | get_image_path(img_path).c(d!()).map(|path| { 96 | path.iter() 97 | .filter_map(|i| { 98 | i.file_name() 99 | .map(|j| j.to_str()) 100 | .flatten() 101 | .map(|os| (os, i)) 102 | }) 103 | .filter(|(os, _)| { 104 | vm_kind(os).is_ok() 105 | && !(os.contains('@') 106 | || os.contains("-part") 107 | || os.starts_with(CLONE_MARK)) 108 | }) 109 | .map(|(os, i)| { 110 | (os.to_lowercase(), i.to_string_lossy().into_owned()) 111 | }) 112 | .collect() 113 | }) 114 | } 115 | 116 | /// 获取服务端支持的系统列表和对应的 Vm 镜像路径 117 | #[cfg(not(feature = "zfs"))] 118 | pub fn get_os_info(img_path: &str) -> Result> { 119 | get_image_path(img_path).c(d!()).map(|path| { 120 | path.iter() 121 | .filter_map(|i| { 122 | i.file_name() 123 | .map(|j| j.to_str()) 124 | .flatten() 125 | .map(|os| (os, i)) 126 | }) 127 | .filter(|(os, _)| vm_kind(os).is_ok()) 128 | .map(|(os, i)| { 129 | (os.to_lowercase(), i.to_string_lossy().into_owned()) 130 | }) 131 | .collect() 132 | }) 133 | } 134 | 135 | /// 读取 zfs snapshot 集合 136 | #[cfg(feature = "zfs")] 137 | fn get_image_path(img_path: &str) -> Result> { 138 | let mut res = vct![]; 139 | let dir = Path::new(img_path); 140 | if dir.is_dir() { 141 | for entry in fs::read_dir(dir).c(d!())? { 142 | let entry = entry.c(d!())?; 143 | let path = entry.path(); 144 | if let Some(p) = path.to_str() { 145 | res.push(PathBuf::from(p)); 146 | } 147 | } 148 | } 149 | Ok(res) 150 | } 151 | 152 | /// 递归读取 ImagePath 下的所有以 ".qemu" 结尾的文件 153 | #[inline(always)] 154 | #[cfg(not(feature = "zfs"))] 155 | fn get_image_path(img_path: &str) -> Result> { 156 | walk_gen(&Path::new(img_path)).c(d!()) 157 | } 158 | 159 | // recursive function 160 | #[inline(always)] 161 | #[cfg(not(feature = "zfs"))] 162 | fn walk_gen(dir: &Path) -> Result> { 163 | let mut res = vct![]; 164 | 165 | if dir.is_dir() { 166 | for entry in fs::read_dir(dir).c(d!())? { 167 | let entry = entry.c(d!())?; 168 | let path = entry.path(); 169 | if path.is_dir() { 170 | res.append(&mut walk_gen(&path).c(d!())?); 171 | } else if let Some(p) = path.to_str() { 172 | res.push(PathBuf::from(p)); 173 | } 174 | } 175 | } 176 | 177 | Ok(res) 178 | } 179 | 180 | /// stop an env 181 | #[inline(always)] 182 | pub fn pause(id: VmId) -> Result<()> { 183 | vm::cgroup::kill_vm(id).c(d!()) 184 | } 185 | 186 | /// restart an env 187 | #[inline(always)] 188 | pub fn resume(vm: &Vm) -> Result<()> { 189 | vm::start(vm).c(d!()) 190 | } 191 | 192 | /// 根据镜像前缀识别虚拟机引擎 193 | pub fn vm_kind(os: &str) -> Result { 194 | let os = os.to_lowercase(); 195 | for (prefix, kind) in 196 | &[("qemu:", VmKind::Qemu), ("fire:", VmKind::FireCracker)] 197 | { 198 | if os.starts_with(prefix) { 199 | return Ok(*kind); 200 | } 201 | } 202 | Err(eg!( 203 | "Invalid OS name, it should starts with one of [ qemu, fire[cracker] ]." 204 | )) 205 | } 206 | -------------------------------------------------------------------------------- /src/core/src/linux/vm/cgroup.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Cgroup2 3 | //! 4 | //! 每个 Vm 分配一个独立的 Cgroup, 通达 Drop Trait 自动清理资源. 5 | //! 6 | 7 | use crate::{asleep, linux::vm::util, VmId, POOL}; 8 | use ruc::*; 9 | use nix::{ 10 | sys::signal::{self, kill}, 11 | unistd::Pid, 12 | }; 13 | use std::{fs, io::Write, path::PathBuf, process, sync::LazyLock}; 14 | 15 | // 采用 private mount, 多服务可使用同一挂载点 16 | const CGROUP_ROOT_PATH: &str = "/tmp/.ttcgroup"; 17 | 18 | // 管理进程的 Cgroup 路径 19 | const CGROUP_ADMIN_PATH: &str = "/tmp/.ttcgroup/ttadmin"; 20 | 21 | // 重置管理进程的 Cgroup 状态 22 | fn cgrp_reset_admin() -> Result<()> { 23 | static CGRP_PROCS: LazyLock = LazyLock::new(|| 24 | format!("{}/cgroup.procs", CGROUP_ADMIN_PATH) 25 | ); 26 | 27 | fs::OpenOptions::new() 28 | .append(true) 29 | .open(CGRP_PROCS.as_str()) 30 | .c(d!())? 31 | .write(process::id().to_string().as_bytes()) 32 | .c(d!()) 33 | .map(|_| ()) 34 | } 35 | 36 | /// 挂载 Cgroup2 根目录结构, 37 | /// **MUST** do `mount` first! (before CGROUP_ADMIN_PATH) 38 | pub(in crate::linux) fn init() -> Result<()> { 39 | fs::create_dir_all(CGROUP_ROOT_PATH) 40 | .c(d!()) 41 | .and_then(|_| util::mount_cgroup2(CGROUP_ROOT_PATH).c(d!())) 42 | .and_then(|_| fs::create_dir_all(CGROUP_ADMIN_PATH).c(d!())) 43 | } 44 | 45 | // 确认 Cgroup 根路径已挂载 46 | fn cgroup2_ready() -> bool { 47 | let mut path = PathBuf::from(CGROUP_ROOT_PATH); 48 | path.push("cgroup.procs"); 49 | path.is_file() 50 | } 51 | 52 | /// 按 '/VmId' 分配挂载点 53 | pub(in crate::linux) fn alloc_mnt_point(id: VmId) -> Result { 54 | if !cgroup2_ready() { 55 | return Err(eg!("The fucking world is over!")); 56 | } 57 | 58 | let mut path = PathBuf::from(CGROUP_ROOT_PATH); 59 | path.push(id.to_string()); 60 | if !path.exists() { 61 | fs::create_dir(&path).c(d!()).map(|_| path) 62 | } else if 0 == get_proc_meta_path(id).c(d!())?.metadata().c(d!())?.len() { 63 | // cgroup.procs 文件为空, 即: 64 | // 挂载点已存在, 但没有进程参与其中 65 | Ok(path) 66 | } else { 67 | Err(eg!("The fucking world is over!")) 68 | } 69 | } 70 | 71 | /// 将 vm 进程加入到指定的 Cgroup 中 72 | #[inline(always)] 73 | pub(in crate::linux) fn add_vm(id: VmId, pid: crate::Pid) -> Result<()> { 74 | add_proc(id, pid).c(d!()) 75 | } 76 | 77 | // 将指定进程加入到指定的 Cgroup 中 78 | fn add_proc(id: VmId, pid: crate::Pid) -> Result<()> { 79 | get_proc_meta_path(id) 80 | .c(d!()) 81 | .and_then(|meta| { 82 | fs::OpenOptions::new().append(true).open(meta).c(d!()) 83 | }) 84 | .and_then(|mut f| { 85 | f.write(pid.to_string().as_bytes()).c(d!()).map(|_| ()) 86 | }) 87 | } 88 | 89 | /// 清理 Vm 进程[组] 90 | pub(crate) fn kill_vm(id: VmId) -> Result<()> { 91 | get_proc_meta_path(id) 92 | .c(d!()) 93 | .and_then(|p| kill_cgrp(p).c(d!())) 94 | } 95 | 96 | // 对指定 Cgroup 下的所有进程, 执行 `kill -9` 97 | fn kill_cgrp(cgpath: PathBuf) -> Result<()> { 98 | fs::read(&cgpath) 99 | .c(d!()) 100 | .and_then(|b| String::from_utf8(b).c(d!())) 101 | .and_then(|s| { 102 | let mut failed_list = vct![]; 103 | s.lines().for_each(|pid| { 104 | let pid = pnk!(pid.parse::()); 105 | // 启动 Vm 进程之前, 控制进程会先进入对应的 Cgroup 106 | if process::id() == pid { 107 | info_omit!(cgrp_reset_admin()); 108 | return; 109 | } 110 | kill(Pid::from_raw(pid as libc::pid_t), signal::SIGTERM) 111 | .c(d!()) 112 | .unwrap_or_else(|e| failed_list.push((pid, e))); 113 | }); 114 | alt!( 115 | failed_list.is_empty(), 116 | Ok(()), 117 | Err(eg!(format!("{:#?}", failed_list))) 118 | ) 119 | }) 120 | .and_then(|_| cgpath.parent().ok_or(eg!("shit!"))) 121 | .map(|dir| { 122 | let dir = dir.to_owned(); 123 | POOL.spawn_ok(async move { 124 | asleep(5).await; 125 | info_omit!(fs::remove_dir(&dir)); 126 | }) 127 | }) 128 | } 129 | 130 | // 获取 Cgroup2 中存放PID列表的文件路径 131 | #[inline(always)] 132 | fn get_proc_meta_path(id: VmId) -> Result { 133 | let mut mnt = get_mnt_point(id).c(d!())?; 134 | mnt.push("cgroup.procs"); 135 | alt!(mnt.is_file(), Ok(mnt), Err(eg!())) 136 | } 137 | 138 | // 获取指定 Vm 的 Cgroup 挂载点 139 | fn get_mnt_point(id: VmId) -> Result { 140 | let mut path = PathBuf::from(CGROUP_ROOT_PATH); 141 | path.push(id.to_string()); 142 | 143 | if path.exists() && path.is_dir() { 144 | Ok(path) 145 | } else { 146 | Err(eg!("Not exists!")) 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /src/core/src/linux/vm/engine/firecracker/mod.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # FireCracker Engine 3 | //! 4 | 5 | #[cfg(all(feature = "nft", any(feature = "cow", feature = "zfs")))] 6 | mod suitable_env; 7 | #[cfg(any(not(feature = "nft"), not(any(feature = "cow", feature = "zfs"))))] 8 | mod unsuitable_env; 9 | 10 | #[cfg(all(feature = "nft", any(feature = "cow", feature = "zfs")))] 11 | pub(super) use suitable_env::*; 12 | #[cfg(any( 13 | not(feature = "nft"), 14 | not(any(feature = "cow", feature = "zfs")) 15 | ))] 16 | pub(super) use unsuitable_env::*; 17 | -------------------------------------------------------------------------------- /src/core/src/linux/vm/engine/firecracker/unsuitable_env.rs: -------------------------------------------------------------------------------- 1 | use crate::linux::vm::Vm; 2 | use ruc::*; 3 | 4 | pub(crate) const LOG_DIR: &str = ""; 5 | 6 | pub(crate) fn start(_vm: &Vm) -> Result<()> { 7 | Err(eg!("Unsuitable environment!")) 8 | } 9 | 10 | pub(crate) fn pre_starter(_vm: &Vm) -> Result<()> { 11 | Err(eg!("Unsuitable environment!")) 12 | } 13 | 14 | pub(crate) fn remove_image(_vm: &Vm) -> Result<()> { 15 | Err(eg!("Unsuitable environment!")) 16 | } 17 | 18 | #[cfg(feature = "nft")] 19 | pub(crate) fn remove_tap(_vm: &Vm) -> Result<()> { 20 | Err(eg!("Unsuitable environment!")) 21 | } 22 | -------------------------------------------------------------------------------- /src/core/src/linux/vm/engine/mod.rs: -------------------------------------------------------------------------------- 1 | mod firecracker; 2 | mod qemu; 3 | 4 | use crate::{Vm, VmKind}; 5 | use ruc::*; 6 | use std::fs; 7 | 8 | // TODO: support more vm-engine 9 | #[inline(always)] 10 | pub(super) fn start(vm: &Vm) -> Result<()> { 11 | match vm.kind { 12 | VmKind::Qemu => qemu::start(vm).c(d!()), 13 | VmKind::FireCracker => firecracker::start(vm).c(d!()), 14 | _ => Err(eg!("Unsupported VmKind!")), 15 | } 16 | } 17 | 18 | #[inline(always)] 19 | pub(in crate::linux) fn init() -> Result<()> { 20 | fs::create_dir_all(firecracker::LOG_DIR) 21 | .c(d!()) 22 | .and_then(|_| { 23 | // firecracker also need this! 24 | qemu::init().c(d!()) 25 | }) 26 | } 27 | 28 | #[inline(always)] 29 | pub(super) fn get_pre_starter(vm: &Vm) -> Result Result<()>> { 30 | match vm.kind { 31 | VmKind::Qemu => Ok(qemu::pre_starter), 32 | VmKind::FireCracker => Ok(firecracker::pre_starter), 33 | _ => Err(eg!("Unsupported VmKind!")), 34 | } 35 | } 36 | 37 | #[inline(always)] 38 | pub(super) fn remove_image(vm: &Vm) -> Result<()> { 39 | match vm.kind { 40 | VmKind::Qemu => qemu::remove_image(vm).c(d!()), 41 | VmKind::FireCracker => firecracker::remove_image(vm).c(d!()), 42 | _ => Err(eg!("The fucking world is over!")), 43 | } 44 | } 45 | 46 | #[cfg(feature = "nft")] 47 | #[inline(always)] 48 | pub(super) fn remove_tap(vm: &Vm) -> Result<()> { 49 | match vm.kind { 50 | VmKind::Qemu => qemu::remove_tap(vm).c(d!()), 51 | VmKind::FireCracker => firecracker::remove_tap(vm).c(d!()), 52 | _ => Err(eg!("The fucking world is over!")), 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/core/src/linux/vm/mod.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Virtual Machine Mgmt 3 | //! 4 | //! - Qemu 5 | //! - FireCracker 6 | //! - ... 7 | //! 8 | 9 | pub(crate) mod cgroup; 10 | pub(crate) mod engine; 11 | pub(crate) mod util; 12 | 13 | use crate::Vm; 14 | use ruc::*; 15 | #[cfg(all(feature = "nft", any(feature = "cow", feature = "zfs")))] 16 | use nix::unistd::{fork, ForkResult}; 17 | #[cfg(all(feature = "nft", any(feature = "cow", feature = "zfs")))] 18 | use std::os::unix::process::CommandExt; 19 | use std::process; 20 | #[cfg(all(feature = "nft", any(feature = "cow", feature = "zfs")))] 21 | use std::process::Stdio; 22 | 23 | #[inline(always)] 24 | pub(crate) fn start(vm: &Vm) -> Result<()> { 25 | // 1. 首先, 分配 Cgroup 挂载点 26 | // 2. 之后, 控制进制先进入 Vm 的 Cgroup 27 | // - 其创建的 Vm 进程会自动归属于相同的 Cgroup 28 | // - 在清理 Vm 进程时要跳过可能存在的控制进程 PID 29 | cgroup::alloc_mnt_point(vm.id) 30 | .c(d!()) 31 | .and_then(|_| cgroup::add_vm(vm.id, process::id()).c(d!())) 32 | .and_then(|_| engine::start(vm).c(d!())) 33 | } 34 | 35 | // Avoid this by using "sh -c ..." to start Qemu? 36 | #[inline(always)] 37 | pub(crate) fn zobmie_clean() { 38 | util::wait_pid() 39 | } 40 | 41 | #[inline(always)] 42 | pub(crate) fn post_clean(vm: &Vm) { 43 | // 停止 Vm 进程及关联的 Cgroup 44 | info_omit!(cgroup::kill_vm(vm.id)); 45 | 46 | // 清理为 Vm 创建的临时 image 47 | if !vm.image_cached { 48 | info_omit!(engine::remove_image(vm)); 49 | } 50 | 51 | // 清理为 Vm 创建的 TAP 设备 52 | #[cfg(feature = "nft")] 53 | info_omit!(engine::remove_tap(vm)); 54 | } 55 | 56 | #[inline(always)] 57 | pub(crate) fn get_pre_starter(vm: &Vm) -> Result Result<()>> { 58 | engine::get_pre_starter(vm).c(d!()) 59 | } 60 | 61 | // 执行命令 62 | #[inline(always)] 63 | fn cmd_exec(cmd: &str, args: &[&str]) -> Result<()> { 64 | let res = process::Command::new(cmd).args(args).output().c(d!())?; 65 | if res.status.success() { 66 | Ok(()) 67 | } else { 68 | Err(eg!(String::from_utf8_lossy(&res.stderr))) 69 | } 70 | } 71 | 72 | // 必须后台执行 73 | #[inline(always)] 74 | #[cfg(all(feature = "nft", any(feature = "cow", feature = "zfs")))] 75 | fn cmd_exec_daemonize(cmd: &str, args: &[&str]) -> Result<()> { 76 | match unsafe { fork() } { 77 | Ok(ForkResult::Child) => pnk!(Err(eg!(process::Command::new(cmd) 78 | .stdin(Stdio::null()) 79 | .stdout(Stdio::null()) 80 | .stderr(Stdio::null()) 81 | .args(args) 82 | .exec()))), 83 | Ok(_) => Ok(()), 84 | Err(e) => Err(e).c(d!()), 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/core/src/linux/vm/util.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Utils for linux. 3 | //! 4 | 5 | use ruc::*; 6 | use nix::{ 7 | mount::{self, MsFlags}, 8 | sys::wait, 9 | unistd, 10 | }; 11 | 12 | pub(in crate::linux) fn mount_make_rprivate() -> Result<()> { 13 | mountx( 14 | None, 15 | "/", 16 | None, 17 | pnk!(MsFlags::from_bits( 18 | MsFlags::MS_REC.bits() | MsFlags::MS_PRIVATE.bits() 19 | )), 20 | None, 21 | ) 22 | .c(d!()) 23 | } 24 | 25 | #[cfg(not(feature = "testmock"))] 26 | pub(in crate::linux) fn mount_cgroup2(path: &str) -> Result<()> { 27 | mountx(None, path, Some("cgroup2"), MsFlags::empty(), None).c(d!()) 28 | } 29 | 30 | pub(in crate::linux) fn mount_dynfs_proc() -> Result<()> { 31 | let mut flags = MsFlags::empty(); 32 | flags.insert(MsFlags::MS_NODEV); 33 | flags.insert(MsFlags::MS_NOEXEC); 34 | flags.insert(MsFlags::MS_NOSUID); 35 | flags.insert(MsFlags::MS_RELATIME); 36 | 37 | mountx(None, "/proc", Some("proc"), flags, None).c(d!()) 38 | } 39 | 40 | pub(in crate::linux) fn mount_tmp_tmpfs() -> Result<()> { 41 | let mut flags = MsFlags::empty(); 42 | flags.insert(MsFlags::MS_RELATIME); 43 | 44 | mountx(None, "/tmp", Some("tmpfs"), flags, None).c(d!()) 45 | } 46 | 47 | #[inline(always)] 48 | fn mountx( 49 | from: Option<&str>, 50 | to: &str, 51 | fstype: Option<&str>, 52 | flags: MsFlags, 53 | data: Option<&str>, 54 | ) -> Result<()> { 55 | mount::mount(from, to, fstype, flags, data).c(d!()) 56 | } 57 | 58 | pub(in crate::linux) fn wait_pid() { 59 | while let Ok(st) = wait::waitpid( 60 | unistd::Pid::from_raw(-1), 61 | Some(wait::WaitPidFlag::WNOHANG), 62 | ) { 63 | if st == wait::WaitStatus::StillAlive { 64 | break; 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/core/src/mocker/mod.rs: -------------------------------------------------------------------------------- 1 | #![warn(unused_import_braces, unused_extern_crates)] 2 | #![allow(missing_docs)] 3 | 4 | pub(crate) mod nat; 5 | pub(crate) mod vm; 6 | 7 | use crate::{ImagePath, OsName, Vm, VmId, VmKind}; 8 | use ruc::*; 9 | use std::collections::HashMap; 10 | 11 | ///////////////// 12 | // Entry Point // 13 | ///////////////// 14 | pub fn exec( 15 | _imgpath: &str, 16 | cb: fn() -> Result<()>, 17 | _serv_ip: &str, 18 | ) -> Result<()> { 19 | cb().c(d!()) 20 | } 21 | 22 | ////////////////// 23 | // Support List // 24 | ////////////////// 25 | 26 | pub fn get_os_info(img_path: &str) -> Result> { 27 | use std::fs; 28 | 29 | let res = map! { 30 | "centos7.0".to_string() => format!("{}/centos7.{}", img_path, 0), 31 | "centos7.1".to_string() => format!("{}/centos7.{}", img_path, 1), 32 | "centos7.2".to_string() => format!("{}/centos7.{}", img_path, 2), 33 | "centos7.3".to_string() => format!("{}/centos7.{}", img_path, 3), 34 | "ubuntu20.04".to_string() => format!("{}/ubuntu20.04", img_path), 35 | "ubuntu22.04".to_string() => format!("{}/ubuntu22.04", img_path), 36 | "qemu:centos7.0".to_string() => format!("{}/qemu:centos7.{}", img_path, 0), 37 | "qemu:centos7.1".to_string() => format!("{}/qemu:centos7.{}", img_path, 1), 38 | "qemu:ubuntu20.04".to_string() => format!("{}/qemu:ubuntu20.04", img_path), 39 | "qemu:ubuntu22.04".to_string() => format!("{}/qemu:ubuntu22.04", img_path), 40 | }; 41 | 42 | // Create empty mock files for testing 43 | for path in res.values() { 44 | info_omit!(fs::File::create(path)); 45 | } 46 | 47 | Ok(res) 48 | } 49 | 50 | #[inline(always)] 51 | pub fn pause(_id: VmId) -> Result<()> { 52 | Ok(()) 53 | } 54 | 55 | #[inline(always)] 56 | pub fn resume(_vm: &Vm) -> Result<()> { 57 | Ok(()) 58 | } 59 | 60 | pub fn vm_kind(os: &str) -> Result { 61 | let os = os.to_lowercase(); 62 | if os.starts_with("qemu:") { 63 | Ok(VmKind::Qemu) 64 | } else if os.starts_with("fire:") { 65 | Ok(VmKind::FireCracker) 66 | } else { 67 | Ok(VmKind::Qemu) // Default to Qemu for compatibility 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/core/src/mocker/nat.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # NAT mocker 3 | //! 4 | 5 | use crate::Vm; 6 | use ruc::*; 7 | 8 | pub(crate) fn set_rule(_vm: &Vm) -> Result<()> { 9 | Ok(()) 10 | } 11 | 12 | pub(crate) fn clean_rule(_vm_set: &[&Vm]) -> Result<()> { 13 | Ok(()) 14 | } 15 | 16 | pub(crate) fn deny_outgoing(_vm_set: &[&Vm]) -> Result<()> { 17 | Ok(()) 18 | } 19 | 20 | pub(crate) fn allow_outgoing(_vm_set: &[&Vm]) -> Result<()> { 21 | Ok(()) 22 | } 23 | -------------------------------------------------------------------------------- /src/core/src/mocker/vm.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Mocker for Virtual Machine Mgmt 3 | //! 4 | 5 | use crate::Vm; 6 | use ruc::*; 7 | 8 | pub(crate) fn start(_: &Vm) -> Result<()> { 9 | Ok(()) 10 | } 11 | 12 | // Do nothing on freebsd. 13 | pub(crate) fn zobmie_clean() {} 14 | 15 | pub(crate) fn post_clean(_: &Vm) {} 16 | 17 | pub(crate) fn get_pre_starter(_vm: &Vm) -> Result Result<()>> { 18 | Ok(pre_start) 19 | } 20 | 21 | fn pre_start(_vm: &Vm) -> Result<()> { 22 | Ok(()) 23 | } 24 | -------------------------------------------------------------------------------- /src/core/src/test.rs: -------------------------------------------------------------------------------- 1 | #![cfg(feature = "testmock")] 2 | 3 | use crate::{ImagePath, OsName}; 4 | use ruc::*; 5 | use std::{collections::HashMap, fs}; 6 | 7 | pub(super) fn get_os_info( 8 | img_path: &str, 9 | ) -> Result> { 10 | let res = map! { 11 | "CentOS7.0".to_lowercase() => format!("{}/CentOS7.{}", img_path, 0), 12 | "CentOS7.1".to_lowercase() => format!("{}/CentOS7.{}", img_path, 1), 13 | "CentOS7.2".to_lowercase() => format!("{}/CentOS7.{}", img_path, 2), 14 | "CentOS7.3".to_lowercase() => format!("{}/CentOS7.{}", img_path, 3), 15 | "CentOS7.4".to_lowercase() => format!("{}/CentOS7.{}", img_path, 4), 16 | "CentOS7.5".to_lowercase() => format!("{}/CentOS7.{}", img_path, 5), 17 | "CentOS7.6".to_lowercase() => format!("{}/CentOS7.{}", img_path, 6), 18 | "CentOS7.7".to_lowercase() => format!("{}/CentOS7.{}", img_path, 7), 19 | "CentOS7.8".to_lowercase() => format!("{}/CentOS7.{}", img_path, 8), 20 | }; 21 | 22 | res.values().for_each(|i| { 23 | info_omit!(fs::File::create(i)); 24 | }); 25 | 26 | Ok(res) 27 | } 28 | -------------------------------------------------------------------------------- /src/core_def/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ttcore_def" 3 | version = "0.3.2" 4 | 5 | edition = "2024" 6 | authors = ["FanHui ", "FanHui "] 7 | description = "Lightweight private cloud solution for SME scenarios." 8 | license = "MIT OR Apache-2.0" 9 | keywords = ["firecracker", "qemu", "kvm", "openstack", "k8s", "cloud"] 10 | categories = ["command-line-utilities"] 11 | 12 | [dependencies] 13 | serde = { version = "1.0.216", features = ["derive"] } 14 | -------------------------------------------------------------------------------- /src/core_def/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Basic Type Definitions 3 | //! 4 | 5 | use serde::{Deserialize, Serialize}; 6 | use std::{ 7 | collections::{BTreeMap, HashMap}, 8 | fmt, 9 | }; 10 | 11 | /// Default number of VM CPU cores 12 | pub const CPU_DEFAULT: i32 = 2; 13 | /// Default VM memory capacity, unit: MB 14 | pub const MEM_DEFAULT: i32 = 1024; 15 | /// Default VM disk capacity, unit: MB 16 | pub const DISK_DEFAULT: i32 = 40 * 1024; 17 | 18 | /// Cli ID 19 | pub type CliId = String; 20 | /// Cli ID as `&str` 21 | pub type CliIdRef = str; 22 | /// Env ID 23 | pub type EnvId = String; 24 | /// Env ID as `&str` 25 | pub type EnvIdRef = str; 26 | 27 | /// Uses the product of the last two segments of VM's MAC address, max value: 256 * 256 28 | pub type VmId = i32; 29 | /// process id 30 | pub type Pid = u32; 31 | 32 | /// VM default open port (sshd) 33 | pub const SSH_PORT: u16 = 22; 34 | /// VM default open port (ttrexec-daemon) 35 | pub const TTREXEC_PORT: u16 = 22000; 36 | 37 | /// eg: 10.10.123.110 38 | #[derive(Clone, Default, Debug, Deserialize, Serialize)] 39 | pub struct Ipv4 { 40 | addr: String, 41 | } 42 | 43 | impl Ipv4 { 44 | /// create a new one 45 | pub fn new(addr: String) -> Ipv4 { 46 | Ipv4 { addr } 47 | } 48 | 49 | /// convert to string 50 | pub fn as_str(&self) -> &str { 51 | self.addr.as_str() 52 | } 53 | } 54 | 55 | impl fmt::Display for Ipv4 { 56 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 57 | write!(f, "{}", &self.addr) 58 | } 59 | } 60 | 61 | /// eg: 22 62 | pub type Port = u16; 63 | /// Port from VM internal perspective, such as standard ports 80, 443 64 | pub type VmPort = Port; 65 | /// Port from external perspective, such as NAT mapped ports 8080, 8443 66 | pub type PubPort = Port; 67 | 68 | /// May support more container engines in the future 69 | /// - [Y] Qemu 70 | /// - [Y] Bhyve 71 | /// - [Y] Firecracker 72 | /// - [N] Systemd Nspawn 73 | /// - [N] Docker 74 | #[derive(Clone, Copy, Debug, Deserialize, Serialize)] 75 | #[allow(missing_docs)] 76 | pub enum VmKind { 77 | Qemu, 78 | Bhyve, 79 | FireCracker, 80 | Unknown, 81 | } 82 | 83 | impl fmt::Display for VmKind { 84 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 85 | match self { 86 | VmKind::Qemu => write!(f, "Qemu"), 87 | VmKind::Bhyve => write!(f, "Bhyve"), 88 | VmKind::FireCracker => write!(f, "FireCracker"), 89 | _ => write!(f, "Unknown"), 90 | } 91 | } 92 | } 93 | 94 | #[cfg(target_os = "linux")] 95 | #[allow(clippy::derivable_impls)] 96 | impl Default for VmKind { 97 | fn default() -> VmKind { 98 | VmKind::Qemu 99 | } 100 | } 101 | 102 | #[cfg(target_os = "freebsd")] 103 | #[allow(clippy::derivable_impls)] 104 | impl Default for VmKind { 105 | fn default() -> VmKind { 106 | VmKind::Bhyve 107 | } 108 | } 109 | 110 | #[cfg(not(any(target_os = "linux", target_os = "freebsd")))] 111 | #[allow(clippy::derivable_impls)] 112 | impl Default for VmKind { 113 | fn default() -> VmKind { 114 | VmKind::Unknown 115 | } 116 | } 117 | 118 | /// Metadata for display purposes 119 | #[derive(Clone, Debug, Deserialize, Serialize)] 120 | pub struct EnvMeta { 121 | /// Ensures global uniqueness 122 | pub id: EnvId, 123 | /// Start time cannot be changed once set 124 | pub start_timestamp: u64, 125 | /// End time can be changed to control VM lifecycle 126 | pub end_timestamp: u64, 127 | /// Number of VMs inside 128 | pub vm_cnt: usize, 129 | /// Whether in stopped state 130 | /// `tt env stop ...` 131 | pub is_stopped: bool, 132 | } 133 | 134 | /// Detailed information of environment instance 135 | #[derive(Clone, Debug, Deserialize, Serialize)] 136 | pub struct EnvInfo { 137 | /// Ensures global uniqueness 138 | pub id: EnvId, 139 | /// Start time cannot be changed once set 140 | pub start_timestamp: u64, 141 | /// End time can be changed to control VM lifecycle 142 | pub end_timestamp: u64, 143 | /// Collection of all VMs under the same Env 144 | pub vm: BTreeMap, 145 | /// Whether in stopped state 146 | /// `tt env stop ...` 147 | pub is_stopped: bool, 148 | } 149 | 150 | /// Use this structure to respond to client requests, preventing Drop actions 151 | #[derive(Clone, Debug, Deserialize, Serialize)] 152 | pub struct VmInfo { 153 | /// System name 154 | pub os: String, 155 | /// Number of CPUs 156 | pub cpu_num: i32, 157 | /// Unit: MB 158 | pub mem_size: i32, 159 | /// Unit: MB 160 | pub disk_size: i32, 161 | /// VM IP determined by VmId, using '10.10.x.x/8' network segment 162 | pub ip: Ipv4, 163 | /// Internal-external port mapping relationship for DNAT 164 | pub port_map: HashMap, 165 | } 166 | 167 | /// Parameters provided by caller for VM creation 168 | #[derive(Clone, Debug)] 169 | pub struct VmCfg { 170 | /// System image path 171 | pub image_path: String, 172 | /// All VMs under the same Env have identical internal ports 173 | pub port_list: Vec, 174 | /// Type of virtual instance 175 | pub kind: VmKind, 176 | /// Number of CPUs 177 | pub cpu_num: Option, 178 | /// Unit: MB 179 | pub mem_size: Option, 180 | /// Unit: MB 181 | pub disk_size: Option, 182 | /// VM UUID randomization (unique) 183 | pub rand_uuid: bool, 184 | } 185 | 186 | /// Parameters provided by caller for VM creation, Proxy-specific 187 | #[derive(Clone, Debug, Deserialize, Serialize)] 188 | pub struct VmCfgProxy { 189 | /// Complete system image name 190 | pub os: String, 191 | /// All VMs under the same Env have identical internal ports 192 | pub port_list: Vec, 193 | /// Number of CPUs 194 | pub cpu_num: Option, 195 | /// Unit: MB 196 | pub mem_size: Option, 197 | /// Unit: MB 198 | pub disk_size: Option, 199 | /// VM UUID randomization (unique) 200 | pub rand_uuid: bool, 201 | } 202 | -------------------------------------------------------------------------------- /src/proxy/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ttproxy" 3 | version = "0.3.2" 4 | 5 | edition = "2024" 6 | authors = ["FanHui ", "FanHui "] 7 | description = "Lightweight private cloud solution for SME scenarios." 8 | license = "MIT OR Apache-2.0" 9 | keywords = ["firecracker", "qemu", "kvm", "openstack", "k8s", "cloud"] 10 | categories = ["command-line-utilities"] 11 | 12 | [dependencies] 13 | clap = "4.5.26" 14 | parking_lot = "0.12.3" 15 | 16 | serde = { version = "1.0.216", features = ["derive"] } 17 | serde_json = "1.0.133" 18 | 19 | axum = "0.8.1" 20 | tide = "0.16.0" 21 | tokio = { version = "1.42.0", features = ["full"] } 22 | async-std = "1.13.0" 23 | nix = { version = "0.29.0", features = ["socket", "uio", "zerocopy", "fs"] } 24 | 25 | ttutils = { path = "../utils" } 26 | ttserver_def = { path = "../server_def" } 27 | ruc = "8.1.2" 28 | 29 | [dev-dependencies] 30 | reqwest = "0.12.9" 31 | ttserver = { path = "../server" } 32 | 33 | [features] 34 | default = [] 35 | testmock = [ "ttserver/testmock" ] 36 | -------------------------------------------------------------------------------- /src/proxy/src/bin/ttproxy.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # tt-proxy 3 | //! 4 | //! 为多个 tt-server 做前端代理, 统一调度全局资源. 5 | //! 6 | 7 | use clap::{Arg, Command}; 8 | use ruc::*; 9 | use std::net::SocketAddr; 10 | use ttproxy::cfg::Cfg; 11 | 12 | fn main() { 13 | pnk!(ttproxy::start(pnk!(parse_cfg()))) 14 | } 15 | 16 | /// 解析命令行参数 17 | pub(crate) fn parse_cfg() -> Result { 18 | // 要添加 "--ignored" 等兼容 `cargo test` 的选项 19 | let matches = Command::new(env!("CARGO_PKG_NAME")) 20 | .version(env!("CARGO_PKG_VERSION")) 21 | .author(env!("CARGO_PKG_AUTHORS")) 22 | .about(env!("CARGO_PKG_DESCRIPTION")) 23 | .arg(Arg::new("proxy-addr") 24 | .long("proxy-addr") 25 | .value_name("ADDR") 26 | .help("ttproxy 地址, eg: 127.0.0.1:19527.")) 27 | .arg(Arg::new("server-set") 28 | .long("server-set") 29 | .value_name("ADDR") 30 | .action(clap::ArgAction::Append) 31 | .help("ttserver 地址, eg: 127.0.0.1:9527,10.10.10.101:9527.")) 32 | .get_matches(); 33 | 34 | match ( 35 | matches.get_one::("proxy-addr"), 36 | matches.get_many::("server-set"), 37 | ) { 38 | (Some(proxy_addr), Some(server_set)) => { 39 | let (proxy_serv_at, server_addr_set, server_set) = { 40 | let mut set = vct![]; 41 | let mut orig_set = vct![]; 42 | for s in server_set { 43 | set.push(s.parse::().c(d!())?); 44 | orig_set.push(s.to_owned()); 45 | } 46 | (proxy_addr.clone(), set, orig_set) 47 | }; 48 | Ok(Cfg { 49 | proxy_serv_at, 50 | server_addr_set, 51 | server_set, 52 | }) 53 | } 54 | _ => Err(eg!()), 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/proxy/src/cfg.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Config Parse 3 | //! 4 | 5 | use std::net::SocketAddr; 6 | 7 | /// 配置信息 8 | #[derive(Debug)] 9 | pub struct Cfg { 10 | /// Proxy 服务地址, 11 | /// eg: '0.0.0.0:19527' 12 | pub proxy_serv_at: String, 13 | /// Server 服务地址, SocketAddr 格式 14 | pub server_addr_set: Vec, 15 | /// Server 服务地址, 原始格式 16 | /// eg: '[ "127.0.0.1:9527", "10.10.10.101:9527" ]' 17 | pub server_set: Vec, 18 | } 19 | 20 | pub(crate) fn register_cfg(cfg: Option) -> Option<&'static Cfg> { 21 | static mut CFG: Option = None; 22 | if cfg.is_some() { 23 | unsafe { 24 | CFG = cfg; 25 | } 26 | } 27 | unsafe { CFG.as_ref() } 28 | } 29 | -------------------------------------------------------------------------------- /src/proxy/src/def.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Proxy Data Structure 3 | //! 4 | //! Uses async req/resp model, 5 | //! SlaveServer responses share the same network infrastructure with C/S logic, 6 | //! No waiting after request is sent (including async waiting and timeout waiting), subsequent processing based on UUID matching. 7 | //! 8 | 9 | use ruc::*; 10 | use nix::sys::socket::SockaddrStorage; 11 | use std::{collections::HashMap, mem, net::SocketAddr}; 12 | use ttserver_def::{Resp, UUID}; 13 | 14 | /// UNIX timestamp 15 | pub type TS = u64; 16 | 17 | /// Proxy Bucket index 18 | pub type IDX = usize; 19 | 20 | /// Request timeout from Proxy to SlaveServer 21 | pub const TIMEOUT_SECS: usize = 5; 22 | 23 | /// Poll once per second, 24 | /// Discard data in timed-out buckets entirely, 25 | /// Trigger Drop mechanism, implement Client reply logic within it 26 | /// 27 | /// bucket index calculation method: 28 | /// - `idx = ts!() % TIMEOUT_SECS` 29 | #[derive(Default)] 30 | pub struct Proxy { 31 | /// Query bucket corresponding to UUID 32 | pub idx_map: HashMap, 33 | /// Store by second division for easier resource cleanup, 34 | /// Requests generated within the same second are all in the same bucket 35 | pub buckets: [Bucket; TIMEOUT_SECS], 36 | } 37 | 38 | impl Proxy { 39 | /// Usually cleaned once per second, 40 | /// but scheduled tasks cannot guarantee strict time alignment, 41 | /// buckets more than 5 seconds later than current timestamp need to be cleaned 42 | pub fn clean_timeout(&mut self) { 43 | let ts_deadline = ts!() - TIMEOUT_SECS as u64; 44 | (0..TIMEOUT_SECS) 45 | .filter(|&i| self.buckets[i].ts < ts_deadline) 46 | .collect::>() 47 | .into_iter() 48 | .for_each(|i| { 49 | mem::take(&mut self.buckets[i]).res.keys().for_each(|k| { 50 | self.idx_map.remove(k); 51 | }); 52 | }) 53 | } 54 | } 55 | 56 | /// Basic unit of polling 57 | pub struct Bucket { 58 | /// Timestamp 59 | pub ts: TS, 60 | /// Slave result set 61 | pub res: HashMap, 62 | } 63 | 64 | impl Default for Bucket { 65 | fn default() -> Self { 66 | Bucket { 67 | ts: 0, 68 | res: HashMap::new(), 69 | } 70 | } 71 | } 72 | 73 | /// 请求信息发出之前, 74 | /// 注册此结至 Proxy 中; 75 | /// 除 msg 字段外, 76 | /// 其余字段创建时预置 77 | pub struct SlaveRes { 78 | /// Slave 回复的消息 79 | pub msg: HashMap, 80 | /// 还没收到回复的 Slave 数量, 81 | /// 每次收到回复减一, 减至 0 时丢弃该结构, 82 | /// 触发 Drop 机制处理数据并回复 Client 端 83 | pub num_to_wait: usize, 84 | /// 请求发起时间 85 | pub start_ts: u64, 86 | /// 全部回复或超时后, 87 | /// 调用此函数做最后的处理 88 | pub do_resp: fn(&mut SlaveRes), 89 | /// Client 的地址, 90 | /// do_resp 处理完后回复到此地址 91 | pub peeraddr: SockaddrStorage, 92 | /// Clent 的 ReqId, 93 | /// 回复 Client 时会用到 94 | pub uuid: UUID, 95 | } 96 | 97 | /// 巧用 Drop 98 | impl Drop for SlaveRes { 99 | fn drop(&mut self) { 100 | (self.do_resp)(self) 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/proxy/src/hdr/add_env.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # add_env() 3 | //! 4 | //! 逻辑较复杂, 实现为一个单独的模块 5 | //! 6 | 7 | use super::*; 8 | use crate::util::genlog; 9 | use std::{collections::HashSet, mem}; 10 | 11 | #[derive(Clone, Debug, Deserialize)] 12 | struct MyReq { 13 | uuid: u64, 14 | cli_id: Option, 15 | msg: ReqAddEnv, 16 | } 17 | 18 | /// 创建新的 Env, 19 | /// 不能简单的转发请求, 20 | /// 要分割资源之后再分发 21 | pub(super) fn add_env( 22 | ops_id: usize, 23 | peeraddr: SockaddrStorage, 24 | request: Vec, 25 | ) -> ruc::Result<()> { 26 | let mut req = serde_json::from_slice::(&request).c(d!())?; 27 | if ENV_MAP.read().get(&req.msg.env_id).is_some() { 28 | return Err(eg!("Aready exists!")); 29 | } 30 | 31 | let slave_info = SLAVE_INFO.read().clone(); 32 | 33 | let mut resource_pool = slave_info 34 | .clone() 35 | .into_iter() 36 | .map(|(k, mut v)| { 37 | let supported_set = mem::take(&mut v.supported_list) 38 | .into_iter() 39 | .collect::>(); 40 | (k, v, vec![], supported_set) 41 | }) 42 | .collect::>(); 43 | 44 | req.msg.set_os_lowercase(); 45 | let me = &req.msg; 46 | let dup_each = me.check_dup().c(d!())?; 47 | let rsc_wanted = slave_info 48 | .into_iter() 49 | .map(|(_, i)| i.supported_list.into_iter()) 50 | .flatten() 51 | .collect::>() 52 | .into_iter() 53 | .filter(|os| req.msg.os_prefix.iter().any(|pre| os.starts_with(pre))) 54 | .map(|os| { 55 | (0..(1 + dup_each)).map(move |_| VmCfgProxy { 56 | os: os.clone(), 57 | port_list: me.port_set.clone(), 58 | cpu_num: me.cpu_num, 59 | mem_size: me.mem_size, 60 | disk_size: me.disk_size, 61 | rand_uuid: me.rand_uuid, 62 | }) 63 | }) 64 | .flatten(); 65 | 66 | let cpu_need = me.cpu_num.unwrap_or(CPU_DEFAULT); 67 | let mem_need = me.mem_size.unwrap_or(MEM_DEFAULT); 68 | let disk_need = me.disk_size.unwrap_or(DISK_DEFAULT); 69 | 'x: for w in rsc_wanted { 70 | resource_pool.sort_by_key(|s| s.1.mem_used - s.1.mem_total); 71 | for i in resource_pool.iter_mut() { 72 | if i.3.contains(&w.os) 73 | && i.1.cpu_total - i.1.cpu_used >= cpu_need 74 | && i.1.mem_total - i.1.mem_used >= mem_need 75 | && i.1.disk_total - i.1.disk_used >= disk_need 76 | { 77 | i.1.cpu_used += cpu_need; 78 | i.1.mem_used += mem_need; 79 | i.1.disk_used += disk_need; 80 | i.2.push(w); 81 | continue 'x; 82 | } 83 | } 84 | 85 | return send_err!( 86 | req.uuid, 87 | eg!("Server has not enough resources to meet your needs!"), 88 | peeraddr 89 | ); 90 | } 91 | 92 | // 任务分配完成, 执行分发 93 | let jobs = resource_pool 94 | .into_iter() 95 | .map(|(sa, _, v, _)| (sa, v)) 96 | .filter(|(_, v)| !v.is_empty()) 97 | .collect::>(); 98 | send_req(ops_id, req, peeraddr, jobs).c(d!()) 99 | } 100 | 101 | fn send_req( 102 | ops_id: usize, 103 | mut req: MyReq, 104 | peeraddr: SockaddrStorage, 105 | jobs: Vec<(SocketAddr, Vec)>, 106 | ) -> ruc::Result<()> { 107 | let num_to_wait = jobs.len(); 108 | let proxy_uuid = gen_proxy_uuid(); 109 | let cli_id = req.cli_id.take().unwrap_or_else(|| peeraddr.to_string()); 110 | 111 | register_resp_hdr( 112 | num_to_wait, 113 | resp_cb_simple, 114 | peeraddr, 115 | req.uuid, 116 | proxy_uuid, 117 | ); 118 | 119 | ENV_MAP.write().insert( 120 | req.msg.env_id.clone(), 121 | jobs.iter().map(|(sa, _)| sa).copied().collect(), 122 | ); 123 | 124 | // 清理不需要的字段, 减少网络数据量 125 | req.msg.os_prefix = vec![]; 126 | req.msg.cpu_num = None; 127 | req.msg.mem_size = None; 128 | req.msg.disk_size = None; 129 | req.msg.port_set = vec![]; 130 | 131 | let mut m; 132 | for (slave_addr, vmcfg) in jobs.into_iter() { 133 | m = req.msg.clone(); 134 | m.vmcfg = Some(vmcfg); 135 | 136 | send_req_to_slave( 137 | ops_id, 138 | Req::new(proxy_uuid, cli_id.clone(), m), 139 | &[slave_addr], 140 | ) 141 | .c(d!()) 142 | .or_else(|e| send_err!(req.uuid, e, peeraddr))?; 143 | } 144 | 145 | Ok(()) 146 | } 147 | -------------------------------------------------------------------------------- /src/proxy/src/hdr/sync.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! Periodically synchronize resource information of various Slave Servers 3 | //! 4 | 5 | use super::*; 6 | use crate::{fwd_to_slave, CFG}; 7 | use async_std::net::{Ipv4Addr, SocketAddrV4}; 8 | use nix::sys::socket::SockaddrStorage; 9 | use std::{thread, time::Duration}; 10 | use ttserver_def::*; 11 | 12 | // Synchronize once per second 13 | const SYNC_ITV: u64 = 1; 14 | 15 | pub(crate) fn start_cron() { 16 | // An empty request body is sufficient 17 | let mut req = Req::new(0, format!("SYSTEM-CRON-{}", ts!()), ""); 18 | 19 | // Mock an address 20 | let peeraddr = mock_addr(); 21 | 22 | // Request information from all background Slave Servers 23 | let addr_set = CFG.server_addr_set.clone(); 24 | 25 | loop { 26 | // get_server_info 27 | info_omit!( 28 | fwd_to_slave!(@@@1, req, peeraddr, server_info_cb, &addr_set) 29 | ); 30 | 31 | // get_env_list_all 32 | info_omit!(fwd_to_slave!(@@@8, req, peeraddr, env_list_cb, &addr_set)); 33 | 34 | thread::sleep(Duration::from_secs(SYNC_ITV)); 35 | } 36 | } 37 | 38 | fn server_info_cb(r: &mut SlaveRes) { 39 | let res = r 40 | .msg 41 | .iter() 42 | .filter(|(_, raw)| raw.status == RetStatus::Success) 43 | .filter_map(|(slave, raw)| { 44 | info!(serde_json::from_slice::< 45 | HashMap, 46 | >(&raw.msg)) 47 | .ok() 48 | .and_then(|resp| resp.into_iter().next()) 49 | .map(|resp| (*slave, resp.1)) 50 | }) 51 | .collect::>(); 52 | 53 | *SLAVE_INFO.write() = res; 54 | } 55 | 56 | fn env_list_cb(r: &mut SlaveRes) { 57 | let res = r 58 | .msg 59 | .iter() 60 | .filter(|(_, raw)| raw.status == RetStatus::Success) 61 | .filter_map(|(slave, raw)| { 62 | info!( 63 | serde_json::from_slice::>( 64 | &raw.msg 65 | ) 66 | ) 67 | .ok() 68 | .and_then(|resp| resp.into_iter().next()) 69 | .map(move |resp| resp.1.into_iter().map(move |ei| (ei.id, *slave))) 70 | }) 71 | .flatten() 72 | .fold(map! {}, |mut base: HashMap>, new| { 73 | if let Some(slave) = base.get_mut(&new.0) { 74 | slave.push(new.1); 75 | } else { 76 | base.insert(new.0, vec![new.1]); 77 | } 78 | base 79 | }); 80 | 81 | // Unreachable Slaves remain in metadata, 82 | // which is a distraction, only valid information should be retained. 83 | // Additionally, if not fully replaced, ENVs automatically cleaned up on Slave side when expired, 84 | // will still remain in Proxy, causing errors when creating ENVs with the same name. 85 | *ENV_MAP.write() = res; 86 | } 87 | 88 | #[inline(always)] 89 | fn mock_addr() -> SockaddrStorage { 90 | let socket_addr_v4 = SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), 35107); 91 | SockaddrStorage::from(socket_addr_v4) 92 | } 93 | -------------------------------------------------------------------------------- /src/proxy/src/http.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Http Interfaces 3 | //! 4 | //! Commiunication with client-end. 5 | //! 6 | 7 | use crate::{hdr, util, DEFAULT_REQ_ID, RECV_TO_SECS, UAU_ID}; 8 | use async_std::future; 9 | use ruc::*; 10 | use std::{sync::atomic::Ordering, time::Duration}; 11 | use tide::{Body, Error, Request}; 12 | 13 | /// Generate log message from error 14 | pub fn genlog(err: E) -> String { 15 | format!("{}", err) 16 | } 17 | 18 | macro_rules! err { 19 | (@$e: expr) => { 20 | Error::from_str(500, util::gen_resp_err(DEFAULT_REQ_ID, &genlog($e))) 21 | }; 22 | ($e: expr) => { 23 | Err(err!(@$e)) 24 | }; 25 | } 26 | 27 | macro_rules! gen_hdr { 28 | ($ops: ident, $idx: expr) => { 29 | pub(super) async fn $ops( 30 | mut req: Request<()>, 31 | ) -> tide::Result { 32 | const OPS_ID: usize = $idx; 33 | let msg = req.body_bytes().await?; 34 | let uau_addr = UAU_ID.fetch_add(1, Ordering::Relaxed).to_ne_bytes(); 35 | 36 | let (mysock, myaddr) = match util::gen_uau_socket(&uau_addr) { 37 | Ok((s, a)) => (s, a), 38 | Err(e) => return err!(e), 39 | }; 40 | 41 | hdr::OPS_MAP[OPS_ID](OPS_ID, myaddr.into(), msg) 42 | .c(d!()) 43 | .map_err(|e| err!(@e))?; 44 | 45 | let mut buf = vec![0; 128 * 1024]; 46 | future::timeout(Duration::from_secs(RECV_TO_SECS), mysock.recv(&mut buf)) 47 | .await 48 | .c(d!()) 49 | .map_err(|e| err!(@e))? 50 | .c(d!()) 51 | .map(|siz| { 52 | unsafe { 53 | buf.set_len(siz); 54 | } 55 | Body::from_bytes(buf) 56 | }) 57 | .map_err(|e| err!(@e)) 58 | } 59 | } 60 | } 61 | 62 | gen_hdr!(register_client_id, 0); 63 | gen_hdr!(get_server_info, 1); 64 | gen_hdr!(get_env_list, 2); 65 | gen_hdr!(get_env_info, 3); 66 | gen_hdr!(add_env, 4); 67 | gen_hdr!(del_env, 5); 68 | gen_hdr!(update_env_lifetime, 6); 69 | gen_hdr!(update_env_kick_vm, 7); 70 | gen_hdr!(get_env_list_all, 8); 71 | gen_hdr!(stop_env, 9); 72 | gen_hdr!(start_env, 10); 73 | gen_hdr!(update_env_resource, 11); 74 | -------------------------------------------------------------------------------- /src/proxy/src/uau.addr: -------------------------------------------------------------------------------- 1 | e96d36287c8cfece722db1ce842e9f6 -------------------------------------------------------------------------------- /src/proxy/src/util.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Commiuncation With Client 3 | //! 4 | 5 | use async_std::{ 6 | net::{SocketAddr, UdpSocket}, 7 | task, 8 | }; 9 | use ruc::*; 10 | use nix::sys::socket::{ 11 | bind, sendto, setsockopt, socket, sockopt, AddressFamily, MsgFlags, 12 | SockFlag, SockType, UnixAddr, SockaddrStorage, 13 | }; 14 | use serde::Serialize; 15 | use std::{ 16 | os::unix::io::{FromRawFd, RawFd, IntoRawFd, BorrowedFd}, 17 | time::Duration, 18 | }; 19 | use ttserver_def::*; 20 | use ttutils::zlib; 21 | 22 | /// UAU(uau), Unix(Unix Domain Socket) Abstract Udp 23 | /// 24 | /// An abstract socket address is distinguished (from a pathname socket) 25 | /// by the fact that sun_path[0] is a null byte ('\0'). 26 | /// The socket's address in this namespace is given 27 | /// by the additional bytes in sun_path that are covered 28 | /// by the specified length of the address structure. 29 | /// Null bytes in the name have no special significance. 30 | /// The name has no connection with filesystem pathnames. 31 | /// When the address of an abstract socket is returned, 32 | /// the returned addrlen is greater than sizeof(sa_family_t) (i.e., greater than 2), 33 | /// and the name of the socket is contained in the first (addrlen - sizeof(sa_family_t)) bytes of sun_path. 34 | /// 35 | /// `man unix(7)` for more infomation. 36 | /// 37 | /// NOTE: 38 | /// Unix Socket that needs to receive messages must explicitly bind address; 39 | /// If sent anonymously, unable to receive reply messages from the other party. 40 | pub(crate) fn gen_uau_socket(addr: &[u8]) -> ruc::Result<(UdpSocket, UnixAddr)> { 41 | let owned_fd = socket( 42 | AddressFamily::Unix, 43 | SockType::Datagram, 44 | SockFlag::empty(), 45 | None, 46 | ) 47 | .c(d!())?; 48 | 49 | let raw_fd = owned_fd.into_raw_fd(); 50 | 51 | let borrowed_fd = unsafe { BorrowedFd::borrow_raw(raw_fd) }; 52 | setsockopt(&borrowed_fd, sockopt::ReuseAddr, &true).c(d!())?; 53 | setsockopt(&borrowed_fd, sockopt::ReusePort, &true).c(d!())?; 54 | 55 | let sa = UnixAddr::new(addr).c(d!())?; 56 | bind(raw_fd, &sa).c(d!())?; 57 | 58 | Ok((unsafe { UdpSocket::from_raw_fd(raw_fd) }, sa)) 59 | } 60 | 61 | /// Send back success information 62 | #[macro_export] 63 | macro_rules! send_ok { 64 | ($uuid: expr, $msg: expr, $peeraddr: expr) => { 65 | $crate::util::send_back( 66 | *$crate::SOCK_UAU, 67 | $crate::util::gen_resp_ok($uuid, $msg), 68 | $peeraddr, 69 | ) 70 | }; 71 | } 72 | 73 | /// Generate reply body marking 'success' 74 | pub(crate) fn gen_resp_ok(uuid: u64, msg: impl Serialize) -> Resp { 75 | Resp { 76 | uuid, 77 | status: RetStatus::Success, 78 | msg: info!(serde_json::to_vec(&msg)).unwrap_or_default(), 79 | } 80 | } 81 | 82 | /// Send back failure information 83 | #[macro_export] 84 | macro_rules! send_err { 85 | ($uuid: expr, $err: expr, $peeraddr: expr) => {{ 86 | let log = genlog($err); 87 | $crate::util::send_back( 88 | *$crate::SOCK_UAU, 89 | $crate::util::gen_resp_err($uuid, &log), 90 | $peeraddr, 91 | ) 92 | .c(d!(&log)) 93 | .map_err(|e| { p(eg!(log)); e }) 94 | }}; 95 | // Errors generated directly at the top level, no longer forwarded internally 96 | (@$uuid: expr, $err: expr, $peeraddr: expr) => {{ 97 | let log = genlog($err); 98 | $crate::util::send_out( 99 | &*$crate::SOCK, 100 | $crate::util::gen_resp_err($uuid, &log), 101 | $peeraddr, 102 | ) 103 | .c(d!(&log)) 104 | .map_err(|e| { p(eg!(log)); e }) 105 | }}; 106 | } 107 | 108 | /// Generate reply body marking 'error' 109 | pub(crate) fn gen_resp_err(uuid: u64, msg: &str) -> Resp { 110 | Resp { 111 | uuid, 112 | status: RetStatus::Fail, 113 | msg: msg.as_bytes().to_vec(), 114 | } 115 | } 116 | 117 | /// Generate log message from error 118 | pub(crate) fn genlog(err: E) -> String { 119 | format!("{}", err) 120 | } 121 | 122 | /// Print function for logging/debugging 123 | pub(crate) fn p(msg: T) -> T { 124 | eprintln!("{}", msg); 125 | msg 126 | } 127 | 128 | /// Send information back to 'outbound hub' 129 | #[inline(always)] 130 | pub(crate) fn send_back( 131 | sock: RawFd, 132 | resp: Resp, 133 | peeraddr: SockaddrStorage, 134 | ) -> ruc::Result<()> { 135 | serde_json::to_vec(&resp) 136 | .c(d!()) 137 | .and_then(|resp| zlib::encode(&resp[..]).c(d!())) 138 | .and_then(|resp_compressed| { 139 | sendto(sock, &resp_compressed, &peeraddr, MsgFlags::empty()) 140 | .c(d!()) 141 | .map(|_| ()) 142 | }) 143 | } 144 | 145 | /// Send information back to client 146 | #[inline(always)] 147 | pub(crate) fn send_out( 148 | sock: &'static UdpSocket, 149 | resp: Resp, 150 | peeraddr: SocketAddr, 151 | ) -> ruc::Result<()> { 152 | serde_json::to_vec(&resp) 153 | .c(d!()) 154 | .and_then(|resp| zlib::encode(&resp[..]).c(d!())) 155 | .map(|resp_compressed| { 156 | task::spawn(async move { 157 | info_omit!(sock.send_to(&resp_compressed, peeraddr).await); 158 | }); 159 | }) 160 | } 161 | 162 | #[inline(always)] 163 | pub(crate) async fn asleep(secs: u64) { 164 | task::sleep(Duration::from_secs(secs)).await 165 | } 166 | -------------------------------------------------------------------------------- /src/proxy/tests/env/mod.rs: -------------------------------------------------------------------------------- 1 | use lazy_static::lazy_static; 2 | use ruc::*; 3 | use nix::unistd::{fork, getuid, ForkResult}; 4 | use serde::Serialize; 5 | use std::{ 6 | collections::HashMap, 7 | fs, 8 | net::{SocketAddr, UdpSocket}, 9 | process, thread, 10 | time::Duration, 11 | }; 12 | use ttproxy::cfg::Cfg; 13 | use ttserver::cfg::Cfg as SlaveCfg; 14 | use ttserver_def::*; 15 | use ttutils::zlib; 16 | 17 | pub(super) const CPU_TOTAL: i32 = 48; 18 | pub(super) const MEM_TOTAL: i32 = 64 * 1024; 19 | pub(super) const DISK_TOTAL: i32 = 1000 * 1024; 20 | 21 | const IP_TEST: &str = "127.0.0.1"; 22 | 23 | lazy_static! { 24 | static ref PROXY_ADDR: String = [IP_TEST, ":19527"].concat(); 25 | static ref CLI_SOCK: UdpSocket = pnk!(gen_sock(3)); 26 | static ref SERV_ADDR: SocketAddr = pnk!(PROXY_ADDR.parse::()); 27 | static ref OPS_MAP: HashMap<&'static str, u8> = map! { 28 | "register_client_id" => 0, 29 | "get_server_info" => 1, 30 | "get_env_list" => 2, 31 | "get_env_info" => 3, 32 | "add_env" => 4, 33 | "del_env" => 5, 34 | "update_env_lifetime" => 6, 35 | "update_env_kick_vm" => 7, 36 | }; 37 | } 38 | 39 | pub(super) fn get_uid() -> u32 { 40 | getuid().as_raw() 41 | } 42 | 43 | // slave server[s] and proxy server 44 | pub(super) fn start_proxy() { 45 | assert_eq!( 46 | 0, 47 | get_uid(), 48 | "\x1b[31;1mMust be root to run this test!\x1b[0m" 49 | ); 50 | 51 | let (proxy, server1, server2) = mock_cfg(); 52 | 53 | start_slave(server1); 54 | start_slave(server2); 55 | 56 | thread::spawn(|| { 57 | pnk!(ttproxy::start(proxy)); 58 | }); 59 | 60 | thread::sleep(Duration::from_secs(3)); 61 | } 62 | 63 | fn start_slave(cfg: SlaveCfg) { 64 | match unsafe { fork() } { 65 | Ok(ForkResult::Child) => { 66 | pnk!(ttserver::start(cfg)); 67 | process::exit(1); 68 | } 69 | Ok(_) => {} 70 | e => { 71 | pnk!(e); 72 | } 73 | } 74 | } 75 | 76 | fn mock_cfg() -> (Cfg, SlaveCfg, SlaveCfg) { 77 | let s1 = [IP_TEST, ":29527"].concat(); 78 | let s2 = [IP_TEST, ":39527"].concat(); 79 | 80 | let cfgdb_path1 = format!("/tmp/{}", ts!()); 81 | pnk!(fs::create_dir_all(&cfgdb_path1)); 82 | 83 | let cfgdb_path2 = format!("/tmp/{}", 2 * ts!()); 84 | pnk!(fs::create_dir_all(&cfgdb_path2)); 85 | 86 | ( 87 | Cfg { 88 | proxy_serv_at: PROXY_ADDR.to_owned(), 89 | server_addr_set: vct![ 90 | pnk!(s1.parse::()), 91 | pnk!(s2.parse::()) 92 | ], 93 | server_set: vct![s1.clone(), s2.clone()], 94 | }, 95 | SlaveCfg { 96 | log_path: Some("/tmp/1_ttserver.log".to_owned()), 97 | serv_ip: IP_TEST.to_owned(), 98 | serv_at: s1, 99 | image_path: "/mnt/".to_owned(), 100 | cfgdb_path: cfgdb_path1, 101 | cpu_total: CPU_TOTAL, 102 | mem_total: MEM_TOTAL, 103 | disk_total: DISK_TOTAL, 104 | }, 105 | SlaveCfg { 106 | log_path: Some("/tmp/2_ttserver.log".to_owned()), 107 | serv_ip: IP_TEST.to_owned(), 108 | serv_at: s2, 109 | image_path: "/mnt/".to_owned(), 110 | cfgdb_path: cfgdb_path2, 111 | cpu_total: CPU_TOTAL, 112 | mem_total: MEM_TOTAL, 113 | disk_total: DISK_TOTAL, 114 | }, 115 | ) 116 | } 117 | 118 | pub(super) type Sender = fn(&str, Req) -> Result; 119 | 120 | /// 发送请求信息, UDP 协议 121 | pub(super) fn send_req(ops: &str, req: Req) -> Result { 122 | let ops_id = OPS_MAP 123 | .get(ops) 124 | .copied() 125 | .ok_or(eg!(format!("Unknown request: {}", ops)))?; 126 | 127 | let mut req_bytes = serde_json::to_vec(&req) 128 | .c(d!()) 129 | .and_then(|req| zlib::encode(&req[..]).c(d!()))?; 130 | let mut body = 131 | format!("{id:>0width$}", id = ops_id, width = OPS_ID_LEN).into_bytes(); 132 | body.append(&mut req_bytes); 133 | 134 | CLI_SOCK.send_to(&body, *SERV_ADDR).c(d!()).and_then(|_| { 135 | let mut buf = vct![0; 8 * 4096]; 136 | let size = CLI_SOCK.recv(&mut buf).c(d!())?; 137 | 138 | zlib::decode(&buf[..size]) 139 | .c(d!()) 140 | .and_then(|resp_decompressed| { 141 | serde_json::from_slice(&resp_decompressed).c(d!()) 142 | }) 143 | }) 144 | } 145 | 146 | /// 发送请求信息, HTTP/TCP 协议 147 | pub(super) fn send_req_http( 148 | ops: &str, 149 | req: Req, 150 | ) -> Result { 151 | let url = format!("http://127.0.0.1:19527/{}", ops); 152 | let body = serde_json::to_vec(&req).c(d!())?; 153 | 154 | attohttpc::post(&url) 155 | .bytes(&body) 156 | .send() 157 | .c(d!()) 158 | .and_then(|resp| resp.bytes().c(d!())) 159 | .and_then(|body| { 160 | zlib::decode(&body[..]) 161 | .c(d!()) 162 | .and_then(|resp_decompressed| { 163 | serde_json::from_slice(&resp_decompressed).c(d!()) 164 | }) 165 | }) 166 | } 167 | 168 | fn gen_sock(timeout: u64) -> Result { 169 | let mut addr; 170 | for port in (40_000 + ts!() % 10_000)..60_000 { 171 | addr = SocketAddr::from(([127, 0, 0, 1], port as u16)); 172 | if let Ok(sock) = UdpSocket::bind(addr) { 173 | sock.set_read_timeout(Some(Duration::from_secs(timeout))) 174 | .c(d!())?; 175 | return Ok(sock); 176 | } 177 | } 178 | Err(eg!()) 179 | } 180 | -------------------------------------------------------------------------------- /src/proxy/tests/integration.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Integration Tests. 3 | //! 4 | 5 | #![cfg(feature = "testmock")] 6 | 7 | mod env; 8 | mod knead; 9 | mod standalone; 10 | 11 | #[test] 12 | fn i_ttproxy() { 13 | if 0 == env::get_uid() { 14 | env::start_proxy(); 15 | standalone::test(); 16 | knead::test(); 17 | } else { 18 | println!("\x1b[31;01mNOT root, ignore...\x1b[00m"); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/proxy/tests/standalone/mod.rs: -------------------------------------------------------------------------------- 1 | use super::env::*; 2 | use ruc::*; 3 | use std::collections::HashMap; 4 | use ttserver_def::*; 5 | 6 | const CUSTOM_CLI_ID: &str = "ErHa"; 7 | 8 | // 孤立测试每一个接口 9 | pub(super) fn test() { 10 | // UDP protocol tests 11 | t_register_client_id(send_req); 12 | t_get_server_info(send_req); 13 | t_get_env_list(send_req); 14 | t_get_env_info(send_req); 15 | t_add_env(send_req); 16 | t_update_env_lifetime(send_req); 17 | t_update_env_kick_vm(send_req); 18 | t_del_env(send_req); 19 | 20 | // HTTP/TCP protocol tests 21 | t_register_client_id(send_req_http); 22 | t_get_server_info(send_req_http); 23 | t_get_env_list(send_req_http); 24 | t_get_env_info(send_req_http); 25 | t_add_env(send_req_http); 26 | t_update_env_lifetime(send_req_http); 27 | t_update_env_kick_vm(send_req_http); 28 | t_del_env(send_req_http); 29 | } 30 | 31 | fn t_register_client_id(send_req: Sender<&str>) { 32 | assert!( 33 | send_req( 34 | "register_client_id", 35 | Req::new(0, CUSTOM_CLI_ID.to_owned(), "") 36 | ) 37 | .is_ok() 38 | ); 39 | assert!( 40 | send_req( 41 | "register_client_id", 42 | Req::new(0, CUSTOM_CLI_ID.to_owned(), "") 43 | ) 44 | .is_ok() 45 | ); 46 | } 47 | 48 | fn t_get_server_info(send_req: Sender<&str>) { 49 | let uuid = 1; 50 | let resp = pnk!(send_req( 51 | "get_server_info", 52 | Req::new(uuid, CUSTOM_CLI_ID.to_owned(), "") 53 | )); 54 | 55 | assert_eq!(resp.uuid, uuid); 56 | assert_eq!(resp.status, RetStatus::Success); 57 | 58 | let body = pnk!(serde_json::from_slice::< 59 | HashMap, 60 | >(&resp.msg)); 61 | assert_eq!(2, body.len()); 62 | 63 | let mut body = body.into_iter().map(|(_, v)| v).fold( 64 | RespGetServerInfo::default(), 65 | |mut base, mut new| { 66 | base.vm_total += new.vm_total; 67 | base.cpu_total += new.cpu_total; 68 | base.mem_total += new.mem_total; 69 | base.disk_total += new.disk_total; 70 | base.cpu_used += new.cpu_used; 71 | base.mem_used += new.mem_used; 72 | base.disk_used += new.disk_used; 73 | base.supported_list.append(&mut new.supported_list); 74 | base 75 | }, 76 | ); 77 | body.supported_list.sort(); 78 | body.supported_list.dedup(); 79 | 80 | assert_eq!(body.vm_total, 0); 81 | assert_eq!(body.cpu_total, 2 * CPU_TOTAL); 82 | assert_eq!(body.mem_total, 2 * MEM_TOTAL); 83 | assert_eq!(body.disk_total, 2 * DISK_TOTAL); 84 | assert_eq!(body.cpu_used, 0); 85 | assert_eq!(body.mem_used, 0); 86 | assert_eq!(body.disk_used, 0); 87 | assert!(!body.supported_list.is_empty()); 88 | } 89 | 90 | // 在 add_env 之前调用 91 | fn t_get_env_list(send_req: Sender<&str>) { 92 | let uuid = 2; 93 | 94 | let resp = pnk!(send_req( 95 | "get_env_list", 96 | Req::new(uuid, CUSTOM_CLI_ID.to_owned(), "") 97 | )); 98 | 99 | assert_eq!(resp.uuid, uuid); 100 | assert_eq!(resp.status, RetStatus::Success); 101 | 102 | let body = pnk!( 103 | serde_json::from_slice::>(&resp.msg) 104 | ); 105 | assert_eq!(2, body.len()); 106 | 107 | body.iter().for_each(|b| { 108 | assert!(b.1.is_empty()); 109 | }); 110 | } 111 | 112 | fn t_get_env_info(send_req: Sender) { 113 | let uuid = 3; 114 | 115 | let msg = ReqGetEnvInfo { 116 | env_set: vct!["abcxxx".to_owned(), "xxxabc".to_owned()], 117 | }; 118 | 119 | let resp = pnk!(send_req( 120 | "get_env_info", 121 | Req::new(uuid, CUSTOM_CLI_ID.to_owned(), msg) 122 | )); 123 | 124 | assert_eq!(resp.uuid, uuid); 125 | assert_eq!(resp.status, RetStatus::Success); 126 | 127 | let body = pnk!( 128 | serde_json::from_slice::>(&resp.msg) 129 | ); 130 | 131 | // 参见 get_env_info 的实现 132 | assert_eq!(0, body.len()); 133 | } 134 | 135 | fn t_add_env(send_req: Sender) { 136 | let uuid = 4; 137 | 138 | let msg = ReqAddEnv { 139 | env_id: "UselessEnv".to_owned(), 140 | os_prefix: vct!["c".to_owned(), "u".to_owned()], 141 | life_time: None, 142 | cpu_num: None, 143 | mem_size: Some(512), 144 | disk_size: None, 145 | port_set: vct![], 146 | dup_each: None, 147 | deny_outgoing: false, 148 | rand_uuid: true, 149 | vmcfg: None, 150 | }; 151 | 152 | let resp = pnk!(send_req( 153 | "add_env", 154 | Req::new(uuid, CUSTOM_CLI_ID.to_owned(), msg) 155 | )); 156 | 157 | assert_eq!(resp.uuid, uuid); 158 | assert_eq!(resp.status, RetStatus::Success); 159 | 160 | let body = pnk!(serde_json::from_slice::(&resp.msg)); 161 | assert_eq!("Success!", &body); 162 | } 163 | 164 | fn t_update_env_lifetime(send_req: Sender) { 165 | let uuid = 5; 166 | 167 | let msg = ReqUpdateEnvLife { 168 | env_id: "UselessEnv".to_owned(), 169 | life_time: 88888888888888, 170 | is_fucker: true, 171 | }; 172 | 173 | let resp = pnk!(send_req( 174 | "update_env_lifetime", 175 | Req::new(uuid, CUSTOM_CLI_ID.to_owned(), msg) 176 | )); 177 | 178 | assert_eq!(resp.uuid, uuid); 179 | assert_eq!(resp.status, RetStatus::Success); 180 | } 181 | 182 | fn t_update_env_kick_vm(send_req: Sender) { 183 | let uuid = 6; 184 | 185 | let msg = ReqUpdateEnvKickVm { 186 | env_id: "UselessEnv".to_owned(), 187 | vm_id: vct![], 188 | os_prefix: vct!["c".to_owned(), "u".to_owned()], 189 | }; 190 | 191 | let resp = pnk!(send_req( 192 | "update_env_kick_vm", 193 | Req::new(uuid, CUSTOM_CLI_ID.to_owned(), msg) 194 | )); 195 | 196 | assert_eq!(resp.uuid, uuid); 197 | assert_eq!(resp.status, RetStatus::Success); 198 | 199 | let body = pnk!(serde_json::from_slice::(&resp.msg)); 200 | assert_eq!("Success!", &body); 201 | } 202 | 203 | fn t_del_env(send_req: Sender) { 204 | let uuid = 7; 205 | 206 | let msg = ReqDelEnv { 207 | env_id: "UselessEnv".to_owned(), 208 | }; 209 | 210 | let resp = pnk!(send_req( 211 | "del_env", 212 | Req::new(uuid, CUSTOM_CLI_ID.to_owned(), msg) 213 | )); 214 | 215 | assert_eq!(resp.uuid, uuid); 216 | assert_eq!(resp.status, RetStatus::Success); 217 | 218 | let body = pnk!(serde_json::from_slice::(&resp.msg)); 219 | assert_eq!("Success!", &body); 220 | } 221 | -------------------------------------------------------------------------------- /src/rexec/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ttrexec" 3 | version = "0.3.2" 4 | 5 | edition = "2024" 6 | authors = ["FanHui ", "FanHui "] 7 | description = "Lightweight private cloud solution for SME scenarios." 8 | license = "MIT OR Apache-2.0" 9 | keywords = ["firecracker", "qemu", "kvm", "openstack", "k8s", "cloud"] 10 | categories = ["command-line-utilities"] 11 | 12 | [dependencies] 13 | nix = { version = "0.29.0", features = ["socket", "uio", "zerocopy", "net"] } 14 | clap = "4.5.26" 15 | serde_json = "1.0.133" 16 | serde = { version = "1.0.216", features = ["derive"] } 17 | jemallocator = { optional = true, version = "0.5.4" } 18 | 19 | ruc = "8.1.2" 20 | 21 | [features] 22 | default = [ "server", "client" ] 23 | server = [ "jemallocator" ] 24 | client = [] 25 | 26 | [[bin]] 27 | name = "ttrexec-daemon" 28 | path = "src/bin/daemon.rs" 29 | 30 | [[bin]] 31 | name = "ttrexec-cli" 32 | path = "src/bin/cli.rs" 33 | -------------------------------------------------------------------------------- /src/rexec/README.md: -------------------------------------------------------------------------------- 1 | # ttrexec 2 | 3 | tt-r-exec, tt remote exector. 4 | 5 | 快速执行远程命令及双向传输文件, 用以替代 SSH. 6 | -------------------------------------------------------------------------------- /src/rexec/src/bin/cli.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # A client-end implementation. 3 | //! 4 | 5 | use clap::{Arg, ArgMatches, Command}; 6 | use ruc::*; 7 | use ttrexec::{ 8 | client::{req_exec, req_transfer}, 9 | common::{Direction, TransReq}, 10 | }; 11 | 12 | fn main() { 13 | let m = Command::new(env!("CARGO_PKG_NAME")) 14 | .version(env!("CARGO_PKG_VERSION")) 15 | .author(env!("CARGO_PKG_AUTHORS")) 16 | .about(env!("CARGO_PKG_DESCRIPTION")) 17 | .subcommands(vct![ 18 | Command::new("exec") 19 | .arg(Arg::new("server-addr") 20 | .short('a') 21 | .long("server-addr") 22 | .value_name("ADDR") 23 | .help("服务端的监听地址.")) 24 | .arg(Arg::new("server-port") 25 | .short('p') 26 | .long("server-port") 27 | .value_name("PORT") 28 | .help("服务端的监听端口.")) 29 | .arg(Arg::new("remote-cmd") 30 | .short('c') 31 | .long("remote-cmd") 32 | .value_name("CMD") 33 | .help("待执行的远程命令.")) 34 | .arg(Arg::new("time-out") 35 | .short('t') 36 | .long("time-out") 37 | .value_name("TIME") 38 | .help("执行超时时间.")), 39 | Command::new("push") 40 | .arg(Arg::new("server-addr") 41 | .short('a') 42 | .long("server-addr") 43 | .value_name("ADDR") 44 | .help("服务端的监听地址.")) 45 | .arg(Arg::new("server-port") 46 | .short('p') 47 | .long("server-port") 48 | .value_name("PORT") 49 | .help("服务端的监听端口.")) 50 | .arg(Arg::new("local-path") 51 | .short('l') 52 | .long("local-path") 53 | .value_name("PATH") 54 | .help("客户端本地文件路径.")) 55 | .arg(Arg::new("remote-path") 56 | .short('r') 57 | .long("remote-path") 58 | .value_name("PATH") 59 | .help("服务端文件路径.")) 60 | .arg(Arg::new("time-out") 61 | .short('t') 62 | .long("time-out") 63 | .value_name("TIME") 64 | .help("执行超时时间.")), 65 | Command::new("get") 66 | .arg(Arg::new("server-addr") 67 | .short('a') 68 | .long("server-addr") 69 | .value_name("ADDR") 70 | .help("服务端的监听地址.")) 71 | .arg(Arg::new("server-port") 72 | .short('p') 73 | .long("server-port") 74 | .value_name("PORT") 75 | .help("服务端的监听端口.")) 76 | .arg(Arg::new("local-path") 77 | .short('l') 78 | .long("local-path") 79 | .value_name("PATH") 80 | .help("客户端本地文件路径.")) 81 | .arg(Arg::new("remote-path") 82 | .short('r') 83 | .long("remote-path") 84 | .value_name("PATH") 85 | .help("服务端文件路径.")) 86 | .arg(Arg::new("time-out") 87 | .short('t') 88 | .long("time-out") 89 | .value_name("TIME") 90 | .help("执行超时时间.")), 91 | ]) 92 | .get_matches(); 93 | 94 | pnk!(parse_and_exec(m)); 95 | } 96 | 97 | fn parse_and_exec(m: ArgMatches) -> Result<()> { 98 | macro_rules! err { 99 | () => { 100 | return Err(eg!(format!("{:#?}", m))); 101 | }; 102 | } 103 | 104 | macro_rules! trans { 105 | ($m: expr, $drct: tt) => { 106 | match ( 107 | $m.get_one::("server-addr"), 108 | $m.get_one::("server-port"), 109 | $m.get_one::("local-path"), 110 | $m.get_one::("remote-path"), 111 | $m.get_one::("time-out"), 112 | ) { 113 | ( 114 | Some(addr), 115 | port, 116 | Some(local_path), 117 | Some(remote_path), 118 | time_out, 119 | ) => { 120 | let servaddr = 121 | format!("{}:{}", addr, port.map(|s| s.as_str()).unwrap_or("22000")); 122 | TransReq::new(Direction::$drct, local_path, remote_path) 123 | .c(d!()) 124 | .and_then(|req| { 125 | req_transfer( 126 | &servaddr, 127 | req, 128 | Some( 129 | time_out 130 | .unwrap_or("3") 131 | .parse::() 132 | .c(d!())?, 133 | ), 134 | ) 135 | .c(d!()) 136 | }) 137 | .and_then(|resp| { 138 | alt!(0 == dbg!(resp).code, Ok(()), err!()) 139 | }) 140 | } 141 | _ => err!(), 142 | } 143 | }; 144 | } 145 | 146 | match m.subcommand() { 147 | ("exec", Some(exec_m)) => { 148 | match ( 149 | exec_m.get_one::("server-addr"), 150 | exec_m.get_one::("server-port"), 151 | exec_m.get_one::("remote-cmd"), 152 | ) { 153 | (Some(addr), port, Some(cmd)) => req_exec( 154 | &format!("{}:{}", addr, port.map(|s| s.as_str()).unwrap_or("22000")), 155 | cmd, 156 | ) 157 | .c(d!()) 158 | .and_then(|resp| alt!(0 == dbg!(resp).code, Ok(()), err!())), 159 | _ => err!(), 160 | } 161 | } 162 | ("push", Some(push_m)) => trans!(push_m, Push), 163 | ("get", Some(get_m)) => trans!(get_m, Get), 164 | _ => err!(), 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/rexec/src/bin/daemon.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # A server-end implementation. 3 | //! 4 | //! default listen at "0.0.0.0:22000", UDP and TCP. 5 | //! 6 | 7 | use clap::{Arg, Command}; 8 | use ruc::*; 9 | use std::thread; 10 | use ttrexec::server::{serv_cmd, serv_transfer}; 11 | 12 | fn main() { 13 | let m = Command::new(env!("CARGO_PKG_NAME")) 14 | .version(env!("CARGO_PKG_VERSION")) 15 | .author(env!("CARGO_PKG_AUTHORS")) 16 | .about(env!("CARGO_PKG_DESCRIPTION")) 17 | .arg(Arg::new("server-addr") 18 | .short('a') 19 | .long("server-addr") 20 | .value_name("ADDR") 21 | .help("服务端的监听地址.")) 22 | .arg(Arg::new("server-port") 23 | .short('p') 24 | .long("server-port") 25 | .value_name("PORT") 26 | .help("服务端的监听端口.")) 27 | .get_matches(); 28 | 29 | let servaddr = format!( 30 | "{}:{}", 31 | m.get_one::("server-addr").map(|s| s.as_str()).unwrap_or("0.0.0.0"), 32 | m.get_one::("server-port").map(|s| s.as_str()).unwrap_or("22000") 33 | ); 34 | let servaddr1 = servaddr.clone(); 35 | 36 | thread::spawn(move || { 37 | pnk!(serv_cmd(&servaddr)); 38 | }); 39 | 40 | pnk!(serv_transfer(&servaddr1)); 41 | } 42 | -------------------------------------------------------------------------------- /src/rexec/src/client.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Client 3 | //! 4 | 5 | use crate::{common::*, sendfile::sendfile}; 6 | use ruc::*; 7 | use nix::sys::socket::{self, MsgFlags, SockaddrIn}; 8 | use std::io::IoSlice; 9 | use std::{ 10 | fs::{self, File}, 11 | io::Write, 12 | net::{SocketAddr, TcpStream}, 13 | os::unix::io::IntoRawFd, 14 | time::Duration, 15 | }; 16 | 17 | /// Send execution command request to remote, 18 | /// - @ remote_addr: remote address, eg: "8.8.8.8:80" 19 | /// - @ cmd: command to be executed, directly passed to shell for parsing and execution 20 | /// - @ wait_timeout: maximum time to wait for remote response, default 10 seconds 21 | pub fn req_exec<'a>(remote_addr: &'a str, cmd: &'a str) -> ruc::Result> { 22 | let socket = gen_udp_sock().c(d!())?; 23 | let sock = *socket; 24 | let socket_addr = remote_addr.parse::().c(d!())?; 25 | let peeraddr: SockaddrIn = match socket_addr { 26 | SocketAddr::V4(addr) => addr.into(), 27 | SocketAddr::V6(_) => return Err(eg!("IPv6 addresses not supported")), 28 | }; 29 | let req = cmd.as_bytes(); 30 | 31 | socket::sendto(sock, req, &peeraddr, MsgFlags::empty()) 32 | .c(d!()) 33 | .and_then(|_| { 34 | let mut buf = vec![0; 4 * 4096]; 35 | let recvd = 36 | socket::recv(sock, &mut buf, MsgFlags::empty()).c(d!())?; 37 | serde_json::from_slice::(&buf[..recvd]).c(d!()) 38 | }) 39 | } 40 | 41 | /// Bidirectional file transfer 42 | /// - @ remote_addr: remote address, eg: "8.8.8.8:80" 43 | /// - @ request: request information, which will indicate transfer direction 44 | /// - @ wait_timeout: maximum time to wait for remote response, default 10 seconds 45 | pub fn req_transfer<'a>( 46 | remote_addr: &'a str, 47 | request: TransReq, 48 | wait_timeout: Option, 49 | ) -> ruc::Result> { 50 | let addr_std = remote_addr.parse::().c(d!())?; 51 | 52 | // Connection time at most 2 seconds 53 | let tcpstream = 54 | TcpStream::connect_timeout(&addr_std, Duration::from_secs(2)) 55 | .c(d!()) 56 | .and_then(|stream| { 57 | stream 58 | .set_read_timeout(wait_timeout.map(Duration::from_secs)) 59 | .c(d!()) 60 | .map(|_| stream) 61 | })?; 62 | 63 | // Single read wait at most 3 seconds 64 | tcpstream 65 | .set_read_timeout(Some(Duration::from_secs(3))) 66 | .c(d!())?; 67 | 68 | // Single write wait at most 3 seconds 69 | tcpstream 70 | .set_write_timeout(Some(Duration::from_secs(3))) 71 | .c(d!())?; 72 | 73 | // Take over socket lifecycle 74 | let socket = FileHdr::new(tcpstream.into_raw_fd()); 75 | let sock = *socket; 76 | 77 | let req = serde_json::to_vec(&request).c(d!())?; 78 | let meta = 79 | format!("{d:>0w$}", d = req.len(), w = TRANS_META_WIDTH).into_bytes(); 80 | socket::sendmsg::<()>( 81 | sock, 82 | &[IoSlice::new(&meta), IoSlice::new(&req)], 83 | &[], 84 | MsgFlags::empty(), 85 | None, 86 | ) 87 | .c(d!())?; 88 | 89 | // Transfer local file to remote 90 | if Direction::Push == request.drct { 91 | let fd = FileHdr::new( 92 | File::open(request.local_file_path).c(d!())?.into_raw_fd(), 93 | ); 94 | let local_fd = *fd; 95 | 96 | sendfile(local_fd, sock, request.file_size).c(d!())?; 97 | } 98 | 99 | let mut buf = vec![0u8; TRANS_META_WIDTH]; 100 | let recvd = socket::recv(sock, &mut buf, MsgFlags::empty()).c(d!())?; 101 | let resp_size = String::from_utf8_lossy(&buf[..recvd]) 102 | .parse::() 103 | .c(d!())?; 104 | 105 | if buf.capacity() < resp_size { 106 | return Err(eg!("The fucking world is over!")); 107 | } else { 108 | unsafe { 109 | buf.set_len(resp_size); 110 | } 111 | } 112 | let recvd = socket::recv(sock, &mut buf, MsgFlags::empty()).c(d!())?; 113 | let resp = serde_json::from_slice::(&buf[..recvd]).c(d!())?; 114 | 115 | // Store remote file locally 116 | if Direction::Get == request.drct && 0 == resp.code { 117 | let mut siz = resp.file_size as usize; 118 | let mut file = fs::OpenOptions::new() 119 | .write(true) 120 | .create(true) 121 | .truncate(true) 122 | .open(request.local_file_path) 123 | .c(d!())?; 124 | let mut recvd; 125 | unsafe { 126 | buf.set_len(buf.capacity()); 127 | } 128 | 129 | while siz > 0 { 130 | recvd = socket::recv(sock, &mut buf, MsgFlags::empty()).c(d!())?; 131 | if 0 == recvd { 132 | return Err(eg!(format!( 133 | "declared_size: {}, recvd: {}", 134 | resp.file_size, 135 | resp.file_size - siz 136 | ))); 137 | } 138 | file.write(&buf[..recvd]).c(d!())?; 139 | siz -= recvd; 140 | } 141 | } 142 | 143 | Ok(resp) 144 | } 145 | -------------------------------------------------------------------------------- /src/rexec/src/common.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Common utils 3 | //! 4 | 5 | use ruc::*; 6 | use serde::{Deserialize, Serialize}; 7 | use std::{ 8 | borrow::Cow, fs::File, ops::Deref, os::unix::io::{AsFd, BorrowedFd, IntoRawFd, RawFd}, path::Path, 9 | }; 10 | 11 | /// 服务端在回复 Resp 结构之前, 12 | /// 发送4个字节长度的字符串数字给客户端, 13 | /// 客户端据此接收 Resp 本体 14 | pub const TRANS_META_WIDTH: usize = "1234".len(); 15 | 16 | /// 接收文件时以此为分界线, 17 | /// - 低于此限, 按实际大小分配缓存区 18 | /// - 超过此限, 循环利用此缓存区 19 | #[cfg(feature = "server")] 20 | pub(crate) const SIZE_16MB: usize = 16 * 1024 * 1024; 21 | 22 | /// 传输方向 23 | #[derive(Debug, Deserialize, Serialize, Eq, PartialEq)] 24 | pub enum Direction { 25 | /// 从远程拉取文件 26 | Get, 27 | /// 推送文件至远程 28 | Push, 29 | } 30 | 31 | /// 文件传输的请求结构 32 | #[derive(Debug, Deserialize, Serialize)] 33 | pub struct TransReq<'a> { 34 | /// 传输方向 35 | pub drct: Direction, 36 | /// 客户端本地的文件路径, 37 | /// Push 时发送此文件至服务端, 38 | /// Get 时接收文件至本地的此路径, 39 | #[serde(skip)] 40 | pub local_file_path: &'a str, 41 | /// 服务端的文件路径 42 | pub remote_file_path: &'a str, 43 | /// 客户端发送的文件尺寸, 44 | /// 直接使用 local_file_path 的文件大小, 无需调方指定, 45 | /// Direction 为 Get 时忽略此项. 46 | pub(crate) file_size: usize, 47 | } 48 | 49 | impl<'a> TransReq<'a> { 50 | /// 创建实例的过程中, 51 | /// 会检查文件路径的有效性: 52 | /// - Push 时 local_file_path 必须存在并可读 53 | /// - Get 时 local_file_path 必须不存在, 防止覆盖已有文件 54 | pub fn new( 55 | drct: Direction, 56 | local_file_path: &'a str, 57 | remote_file_path: &'a str, 58 | ) -> ruc::Result { 59 | let file_size = if Direction::Push == drct { 60 | File::open(local_file_path) 61 | .c(d!())? 62 | .metadata() 63 | .c(d!())? 64 | .len() as usize 65 | } else if Path::new(local_file_path).exists() { 66 | return Err(eg!(format!( 67 | "File: `{}` already exists!", 68 | local_file_path 69 | ))); 70 | } else { 71 | 0 72 | }; 73 | 74 | Ok(TransReq { 75 | drct, 76 | local_file_path, 77 | remote_file_path, 78 | file_size, 79 | }) 80 | } 81 | } 82 | 83 | /// shell 执行命令的返回码, 84 | /// 0 代表成功, 其它数字代表出错 85 | pub type ShellCode = i32; 86 | 87 | /// 服务端返回的元信息 88 | #[derive(Debug, Default, Deserialize, Serialize)] 89 | pub struct Resp<'a> { 90 | /// 服务端执行结果 91 | pub code: ShellCode, 92 | /// 服务端执行过程中产生的标准输出内容 93 | pub stdout: Cow<'a, str>, 94 | /// 服务端执行过程中产生的标准错误内容 95 | pub stderr: Cow<'a, str>, 96 | /// 服务端发送的文件尺寸, 97 | /// 直接使用 local_file_path 的文件大小, 无需调方指定, 98 | /// Direction 为 Get 时忽略此项. 99 | pub(crate) file_size: usize, 100 | } 101 | 102 | /// 管理 RawFd 的 lifetime, 确保及时关闭之 103 | pub struct FileHdr(RawFd); 104 | 105 | impl FileHdr { 106 | /// 创建新实例 107 | pub fn new(sock: RawFd) -> Self { 108 | FileHdr(sock) 109 | } 110 | } 111 | 112 | impl Deref for FileHdr { 113 | type Target = RawFd; 114 | 115 | fn deref(&self) -> &Self::Target { 116 | &self.0 117 | } 118 | } 119 | 120 | impl AsFd for FileHdr { 121 | fn as_fd(&self) -> BorrowedFd<'_> { 122 | unsafe { BorrowedFd::borrow_raw(self.0) } 123 | } 124 | } 125 | 126 | impl Drop for FileHdr { 127 | fn drop(&mut self) { 128 | info_omit!(nix::unistd::close(self.0)); 129 | } 130 | } 131 | 132 | #[cfg(any(target_os = "linux", target_os = "freebsd"))] 133 | macro_rules! gen_sock { 134 | ($addr_family: tt, $sock_type: tt) => {{ 135 | nix::sys::socket::socket( 136 | nix::sys::socket::AddressFamily::$addr_family, 137 | nix::sys::socket::SockType::$sock_type, 138 | nix::sys::socket::SockFlag::SOCK_CLOEXEC, 139 | None, 140 | ) 141 | .map(|fd| FileHdr::new(fd.into_raw_fd())) 142 | .c(d!()) 143 | }}; 144 | } 145 | 146 | #[cfg(target_os = "macos")] 147 | macro_rules! gen_sock { 148 | ($addr_family: tt, $sock_type: tt) => {{ 149 | nix::sys::socket::socket( 150 | nix::sys::socket::AddressFamily::$addr_family, 151 | nix::sys::socket::SockType::$sock_type, 152 | nix::sys::socket::SockFlag::empty(), 153 | None, 154 | ) 155 | .map(|fd| FileHdr::new(fd.into_raw_fd())) 156 | .c(d!()) 157 | }}; 158 | } 159 | 160 | /// 创建 UDP 套接字 161 | #[inline(always)] 162 | pub fn gen_udp_sock() -> ruc::Result { 163 | gen_sock!(Inet, Datagram) 164 | } 165 | 166 | /// 创建 TCP 套接字 167 | #[inline(always)] 168 | pub fn gen_tcp_sock() -> ruc::Result { 169 | gen_sock!(Inet, Stream) 170 | } 171 | -------------------------------------------------------------------------------- /src/rexec/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # fastexec 3 | //! 4 | //! Fast execution of remote commands and bidirectional file transfer. 5 | //! 6 | //! Command execution uses UDP, file transfer uses TCP. 7 | //! 8 | 9 | #![warn(missing_docs, unused_import_braces, unused_extern_crates)] 10 | 11 | /// Generate log message from error 12 | pub fn genlog(err: E) -> String { 13 | format!("{}", err) 14 | } 15 | 16 | #[cfg(feature = "client")] 17 | pub mod client; 18 | 19 | pub mod common; 20 | mod sendfile; 21 | 22 | #[cfg(feature = "server")] 23 | pub mod server; 24 | 25 | #[cfg(feature = "server")] 26 | #[global_allocator] 27 | static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; 28 | -------------------------------------------------------------------------------- /src/rexec/src/sendfile.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Sendfile 3 | //! 4 | //! **注意**: 5 | //! 6 | //! - sendfile 单次最多发送 2G 内容, 需要循环 7 | //! - sendfile 系统接口前两个参数的位置, 在 Linux 与 FreeBSD/MacOS 两类平台上是相反的 8 | //! 9 | 10 | use ruc::*; 11 | use nix::sys::sendfile::sendfile as sf; 12 | #[cfg(target_os = "freebsd")] 13 | use nix::sys::sendfile::SfFlags; 14 | use std::os::unix::io::{RawFd, BorrowedFd}; 15 | 16 | #[cfg(target_os = "linux")] 17 | pub(crate) fn sendfile( 18 | file_fd: RawFd, 19 | sock_fd: RawFd, 20 | file_size: usize, 21 | ) -> ruc::Result<()> { 22 | let mut offset = 0; 23 | loop { 24 | let file_borrowed = unsafe { BorrowedFd::borrow_raw(file_fd) }; 25 | let sock_borrowed = unsafe { BorrowedFd::borrow_raw(sock_fd) }; 26 | let sendsiz = 27 | sf(sock_borrowed, file_borrowed, Some(&mut offset), file_size).c(d!())?; 28 | if 0 == sendsiz { 29 | break; 30 | } 31 | } 32 | Ok(()) 33 | } 34 | 35 | #[cfg(target_os = "freebsd")] 36 | pub(crate) fn sendfile( 37 | file_fd: RawFd, 38 | sock_fd: RawFd, 39 | file_size: usize, 40 | ) -> ruc::Result<()> { 41 | let mut offset = 0; 42 | loop { 43 | let file_borrowed = unsafe { BorrowedFd::borrow_raw(file_fd) }; 44 | let sock_borrowed = unsafe { BorrowedFd::borrow_raw(sock_fd) }; 45 | let (res, sendsiz) = sf( 46 | file_borrowed, 47 | sock_borrowed, 48 | offset, 49 | Some(file_size), 50 | None, 51 | None, 52 | SfFlags::empty(), 53 | 16, 54 | ); 55 | res.c(d!())?; 56 | if 0 == sendsiz { 57 | break; 58 | } else { 59 | offset += sendsiz; 60 | } 61 | } 62 | Ok(()) 63 | } 64 | 65 | #[cfg(target_os = "macos")] 66 | pub(crate) fn sendfile( 67 | file_fd: RawFd, 68 | sock_fd: RawFd, 69 | file_size: usize, 70 | ) -> ruc::Result<()> { 71 | let mut offset = 0; 72 | loop { 73 | let file_borrowed = unsafe { BorrowedFd::borrow_raw(file_fd) }; 74 | let sock_borrowed = unsafe { BorrowedFd::borrow_raw(sock_fd) }; 75 | let (res, sendsiz) = 76 | sf(file_borrowed, sock_borrowed, offset, Some(file_size as i64), None, None); 77 | res.c(d!())?; 78 | if 0 == sendsiz { 79 | break; 80 | } else { 81 | offset += sendsiz; 82 | } 83 | } 84 | Ok(()) 85 | } 86 | 87 | #[cfg(not(any( 88 | target_os = "linux", 89 | target_os = "freebsd", 90 | target_os = "macos" 91 | )))] 92 | pub(crate) fn sendfile( 93 | file_fd: RawFd, 94 | sock_fd: RawFd, 95 | file_size: usize, 96 | ) -> ruc::Result<()> { 97 | Err(eg!("Unsupported platform!")) 98 | } 99 | -------------------------------------------------------------------------------- /src/rexec/tests/integration.rs: -------------------------------------------------------------------------------- 1 | use ruc::*; 2 | use std::{fs, path::Path, thread, time::Duration}; 3 | use ttrexec::{ 4 | client, 5 | common::{Direction, TransReq}, 6 | server, 7 | }; 8 | 9 | const UDP_SERV_ADDR: &str = "127.0.0.1:49527"; 10 | const TCP_SERV_ADDR: &str = "127.0.0.1:49527"; 11 | 12 | #[test] 13 | fn i_ttrexec() { 14 | start_server(); 15 | do_client_ops(); 16 | } 17 | 18 | fn start_server() { 19 | thread::spawn(|| { 20 | pnk!(server::serv_cmd(UDP_SERV_ADDR)); 21 | }); 22 | 23 | thread::spawn(|| { 24 | pnk!(server::serv_transfer(TCP_SERV_ADDR)); 25 | }); 26 | 27 | thread::sleep(Duration::from_secs(1)); 28 | } 29 | 30 | fn do_client_ops() { 31 | exec_normal_success(); 32 | exec_normal_fail(); 33 | 34 | trans_normal_success_get(); 35 | trans_normal_success_push(); 36 | trans_normal_fail_get_client_err(); 37 | trans_normal_fail_get_server_err(); 38 | trans_normal_fail_push_client_err(); 39 | trans_normal_fail_push_server_err(); 40 | } 41 | 42 | fn exec_normal_success() { 43 | let resp = pnk!(client::req_exec(UDP_SERV_ADDR, "uname")); 44 | assert_eq!(0, resp.code); 45 | assert!(0 < resp.stdout.len()); 46 | assert_eq!(0, resp.stderr.len()); 47 | } 48 | 49 | fn exec_normal_fail() { 50 | let resp = pnk!(client::req_exec(UDP_SERV_ADDR, "ls /a;lfjkal;hjkf")); 51 | assert_ne!(0, resp.code); 52 | assert_eq!(0, resp.stdout.len()); 53 | assert!(0 < resp.stderr.len()); 54 | } 55 | 56 | fn trans_normal_success_get() { 57 | let local_file_path = "/tmp/passwd"; 58 | let remote_file_path = "/etc/passwd"; 59 | 60 | if Path::new(local_file_path).exists() { 61 | pnk!(fs::remove_file(local_file_path)); 62 | } 63 | 64 | let req = pnk!(TransReq::new( 65 | Direction::Get, 66 | local_file_path, 67 | remote_file_path 68 | )); 69 | let resp = pnk!(client::req_transfer(TCP_SERV_ADDR, req, None)); 70 | 71 | assert_eq!(0, resp.code); 72 | assert_eq!(0, resp.stderr.len()); 73 | 74 | let contents_local = pnk!(fs::read(local_file_path)); 75 | let contents_remote = pnk!(fs::read(remote_file_path)); 76 | assert_eq!(contents_local, contents_remote); 77 | } 78 | 79 | fn trans_normal_success_push() { 80 | let local_file_path = "/etc/passwd"; 81 | let remote_file_path = "/tmp/passwd"; 82 | 83 | if Path::new(remote_file_path).exists() { 84 | pnk!(fs::remove_file(remote_file_path)); 85 | } 86 | 87 | let req = pnk!(TransReq::new( 88 | Direction::Push, 89 | local_file_path, 90 | remote_file_path 91 | )); 92 | let resp = pnk!(client::req_transfer(TCP_SERV_ADDR, req, None)); 93 | 94 | assert_eq!(0, resp.code); 95 | assert_eq!(0, resp.stderr.len()); 96 | 97 | let contents_local = pnk!(fs::read(local_file_path)); 98 | let contents_remote = pnk!(fs::read(remote_file_path)); 99 | assert_eq!(contents_local, contents_remote); 100 | } 101 | 102 | fn trans_normal_fail_get_client_err() { 103 | let local_file_path = "/tmpppppppp/passwd"; 104 | let remote_file_path = "/etc/passwd"; 105 | 106 | if Path::new(local_file_path).exists() { 107 | pnk!(fs::remove_file(local_file_path)); 108 | } 109 | 110 | let req = pnk!(TransReq::new( 111 | Direction::Get, 112 | local_file_path, 113 | remote_file_path 114 | )); 115 | 116 | assert!(client::req_transfer(TCP_SERV_ADDR, req, None).is_err()); 117 | } 118 | 119 | fn trans_normal_fail_get_server_err() { 120 | let local_file_path = "/tmp/passwd"; 121 | let remote_file_path = "/a;lkfjal;kfl;akjfl;ahgalkj"; 122 | 123 | if Path::new(local_file_path).exists() { 124 | pnk!(fs::remove_file(local_file_path)); 125 | } 126 | 127 | let req = pnk!(TransReq::new( 128 | Direction::Get, 129 | local_file_path, 130 | remote_file_path 131 | )); 132 | let resp = pnk!(client::req_transfer(TCP_SERV_ADDR, req, None)); 133 | 134 | assert_ne!(0, resp.code); 135 | assert!(0 < resp.stderr.len()); 136 | } 137 | 138 | fn trans_normal_fail_push_client_err() { 139 | let local_file_path = "/a;lkfjajkf"; 140 | let remote_file_path = "/tmp/passwd"; 141 | 142 | if Path::new(remote_file_path).exists() { 143 | pnk!(fs::remove_file(remote_file_path)); 144 | } 145 | 146 | assert!( 147 | TransReq::new(Direction::Push, local_file_path, remote_file_path) 148 | .is_err() 149 | ); 150 | } 151 | 152 | fn trans_normal_fail_push_server_err() { 153 | let local_file_path = "/etc/passwd"; 154 | let remote_file_path = ""; 155 | 156 | if Path::new(remote_file_path).exists() { 157 | pnk!(fs::remove_file(remote_file_path)); 158 | } 159 | 160 | let req = pnk!(TransReq::new( 161 | Direction::Push, 162 | local_file_path, 163 | remote_file_path 164 | )); 165 | let resp = pnk!(client::req_transfer(TCP_SERV_ADDR, req, None)); 166 | 167 | assert_ne!(0, resp.code); 168 | assert!(0 < resp.stderr.len()); 169 | } 170 | -------------------------------------------------------------------------------- /src/server/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ttserver" 3 | version = "0.3.2" 4 | 5 | edition = "2024" 6 | authors = ["FanHui ", "FanHui "] 7 | description = "Lightweight private cloud solution for SME scenarios." 8 | license = "MIT OR Apache-2.0" 9 | keywords = ["firecracker", "qemu", "kvm", "openstack", "k8s", "cloud"] 10 | categories = ["command-line-utilities"] 11 | 12 | [dependencies] 13 | nix = { version = "0.29.0", features = ["socket", "uio", "zerocopy", "fs"] } 14 | clap = "4.5.26" 15 | 16 | serde_json = "1.0.133" 17 | serde = { version = "1.0.216", features = ["derive"] } 18 | 19 | futures = { version = "0.3.31", features = ["thread-pool"] } 20 | futures-timer = "3.0.3" 21 | 22 | num_cpus = "1.16.0" 23 | parking_lot = "0.12.3" 24 | 25 | ttutils = { path = "../utils" } 26 | ttcore = { path = "../core", default-features = false } 27 | ttserver_def = { path = "../server_def" } 28 | 29 | ruc = "8.1.2" 30 | 31 | [features] 32 | default = [ "nft", "zfs" ] 33 | cow = [ "ttcore/cow" ] 34 | nft = [ "ttcore/nft" ] 35 | zfs = [ "ttcore/zfs" ] 36 | testmock = [ "ttcore/testmock" ] 37 | -------------------------------------------------------------------------------- /src/server/src/bin/ttserver.rs: -------------------------------------------------------------------------------- 1 | use clap::{Arg, Command}; 2 | use ruc::*; 3 | use std::{path::Path, process}; 4 | use ttserver::cfg::Cfg; 5 | 6 | fn main() { 7 | pnk!(ttserver::start(pnk!(parse_cfg()))); 8 | } 9 | 10 | /// 解析命令行参数 11 | fn parse_cfg() -> Result { 12 | // 要添加 "--ignored" 等兼容 `cargo test` 的选项 13 | let matches = Command::new(env!("CARGO_PKG_NAME")) 14 | .version(env!("CARGO_PKG_VERSION")) 15 | .author(env!("CARGO_PKG_AUTHORS")) 16 | .about(env!("CARGO_PKG_DESCRIPTION")) 17 | .arg(Arg::new("serv-addr") 18 | .long("serv-addr") 19 | .value_name("ADDR") 20 | .help("服务监听地址.")) 21 | .arg(Arg::new("serv-port") 22 | .long("serv-port") 23 | .value_name("PORT") 24 | .help("服务监听端口.")) 25 | .arg(Arg::new("log-path") 26 | .long("log-path") 27 | .value_name("PATH") 28 | .help("日志存储路径.")) 29 | .arg(Arg::new("image-path") 30 | .long("image-path") 31 | .value_name("PATH") 32 | .help("镜像存放路径.")) 33 | .arg(Arg::new("cfgdb-path") 34 | .long("cfgdb-path") 35 | .value_name("PATH") 36 | .help("Env Config 存放路径.")) 37 | .arg(Arg::new("cpu-total") 38 | .long("cpu-total") 39 | .value_name("NUM") 40 | .help("可以使用的 CPU 核心总数.")) 41 | .arg(Arg::new("mem-total") 42 | .long("mem-total") 43 | .value_name("SIZE") 44 | .help("可以使用的内存总量, 单位: MB.")) 45 | .arg(Arg::new("disk-total") 46 | .long("disk-total") 47 | .value_name("SIZE") 48 | .help("可以使用的磁盘总量, 单位: MB.")) 49 | .get_matches(); 50 | 51 | match ( 52 | matches.get_one::("serv-addr"), 53 | matches.get_one::("serv-port"), 54 | matches.get_one::("log-path"), 55 | matches.get_one::("image-path"), 56 | matches.get_one::("cfgdb-path"), 57 | matches.get_one::("cpu-total"), 58 | matches.get_one::("mem-total"), 59 | matches.get_one::("disk-total"), 60 | ) { 61 | ( 62 | Some(addr), 63 | port, 64 | log_path, 65 | Some(img_path), 66 | Some(cfgdb_path), 67 | Some(cpu), 68 | Some(mem), 69 | Some(disk), 70 | ) => Ok(Cfg { 71 | serv_ip: addr.clone(), 72 | serv_at: format!("{}:{}", addr, port.map(|s| s.as_str()).unwrap_or("9527")), 73 | log_path: log_path.map(|lp| lp.clone()), 74 | image_path: check_image_path(img_path).c(d!())?.to_owned(), 75 | cfgdb_path: cfgdb_path.clone(), 76 | cpu_total: cpu.parse::().c(d!())?, 77 | mem_total: mem.parse::().c(d!())?, 78 | disk_total: disk.parse::().c(d!())?, 79 | }), 80 | (addr, _, _, img_path, cfgdb_path, cpu, mem, disk) => { 81 | let msg = format!( 82 | "\x1b[01mOption missing: [{}]\x1b[00m", 83 | [ 84 | ("--serv-addr", addr), 85 | ("--image-path", img_path), 86 | ("--cfgdb-path", cfgdb_path), 87 | ("--cpu-total", cpu), 88 | ("--mem-total", mem), 89 | ("--disk-total", disk) 90 | ] 91 | .iter() 92 | .filter(|(_, v)| v.is_none()) 93 | .map(|(k, _)| k) 94 | .cloned() 95 | .collect::>() 96 | .join(", ") 97 | ); 98 | 99 | eprintln!( 100 | "\n\x1b[31;01mInvalid arguments\x1b[00m\n\n{}\n\n{}\n", 101 | msg, 102 | matches.render_usage() 103 | ); 104 | 105 | process::exit(1); 106 | } 107 | } 108 | } 109 | 110 | /// 确认镜像路径可写并且是目录 111 | fn check_image_path(path: &str) -> Result<&str> { 112 | let p = Path::new(path); 113 | 114 | #[cfg(target_os = "Linux")] 115 | if p.metadata().c(d!())?.permissions().readonly() { 116 | return Err(eg!("无写权限!")); 117 | } 118 | 119 | if !p.is_dir() { 120 | return Err(eg!("不是目录!")); 121 | } 122 | 123 | Ok(path) 124 | } 125 | -------------------------------------------------------------------------------- /src/server/src/cfg.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Config Parse 3 | //! 4 | 5 | /// 配置信息 6 | #[derive(Debug)] 7 | pub struct Cfg { 8 | /// 日志存储路径 9 | pub log_path: Option, 10 | /// eg: '10.10.10.22' 11 | pub serv_ip: String, 12 | /// 服务地址和端口, 13 | /// eg: '10.10.10.22:9527' 14 | pub serv_at: String, 15 | /// 基础镜像的存放路径, 16 | /// 同时也是服务进程的工作路径; 17 | /// 需要可写权限, 18 | /// tap.sh 会创建在此路径下, 19 | /// Vm 镜像也会创建在相同的跟径下 20 | pub image_path: String, 21 | /// Env Config 存放路径 22 | pub cfgdb_path: String, 23 | /// CPU 核心总数 24 | pub cpu_total: i32, 25 | /// Mem 总容量, 单位: MB 26 | pub mem_total: i32, 27 | /// Disk 总容量, 单位: MB 28 | pub disk_total: i32, 29 | } 30 | 31 | pub(crate) fn register_cfg(cfg: Option) -> Option<&'static Cfg> { 32 | static mut CFG: Option = None; 33 | if cfg.is_some() { 34 | unsafe { 35 | CFG = cfg; 36 | } 37 | } 38 | unsafe { CFG.as_ref() } 39 | } 40 | -------------------------------------------------------------------------------- /src/server/src/def.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Common Data Structure 3 | //! 4 | 5 | pub(crate) use ttserver_def::*; 6 | -------------------------------------------------------------------------------- /src/server/src/hdr/server.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Server Information 3 | //! 4 | 5 | use crate::CFG; 6 | use ruc::*; 7 | use parking_lot::RwLock; 8 | use std::{collections::HashMap, sync::{Arc, LazyLock}}; 9 | use ttcore::{get_os_info, ImagePath, OsName}; 10 | 11 | pub(super) static OS_INFO: LazyLock>>> = 12 | LazyLock::new(|| Arc::new(RwLock::new(HashMap::new()))); 13 | 14 | /// 定时扫描镜像信息 15 | pub(crate) async fn refresh_os_info() -> ruc::Result<()> { 16 | get_os_info(&CFG.image_path) 17 | .map(|info| *OS_INFO.write() = info) 18 | .c(d!()) 19 | } 20 | -------------------------------------------------------------------------------- /src/server/src/init.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Environment Init. 3 | //! 4 | 5 | use crate::{hdr::server::refresh_os_info, util::asleep, CFG, POOL, SERV}; 6 | use ruc::*; 7 | use nix::unistd; 8 | #[cfg(not(feature = "testmock"))] 9 | use std::{fs, mem}; 10 | 11 | // 设置基本运行环境 12 | #[inline(always)] 13 | pub(super) fn setenv() -> ruc::Result<()> { 14 | unistd::chdir(CFG.image_path.as_str()) 15 | .c(d!()) 16 | .and_then(|_| log_init(CFG.log_path.as_deref()).c(d!())) 17 | .map(|_| set_total_resource()) 18 | } 19 | 20 | /// 设置可用的资源上限 21 | fn set_total_resource() { 22 | SERV.set_resource(ttcore::Resource::new( 23 | CFG.cpu_total, 24 | CFG.mem_total, 25 | CFG.disk_total, 26 | )); 27 | } 28 | 29 | /// 每 15 秒执行一次定时任务 30 | /// - 清理一次过期的 Vm 31 | /// - 扫描刷新基础镜像信息 32 | #[inline(always)] 33 | pub(super) fn start_cron() { 34 | POOL.spawn_ok(async { 35 | loop { 36 | info_omit!(refresh_os_info().await); 37 | clean_expired_env().await; 38 | asleep(15).await; 39 | } 40 | }); 41 | } 42 | 43 | // 清理过期的 Env, 44 | // 只需请理 Env 自身即可, 45 | // 其余附属数据会在 Drop 体制下被自动清理 46 | #[inline(always)] 47 | async fn clean_expired_env() { 48 | SERV.clean_expired_env() 49 | } 50 | 51 | // 输出日志至文件 52 | #[cfg(not(feature = "testmock"))] 53 | fn log_init(log_path: Option<&str>) -> ruc::Result<()> { 54 | const LOG_PATH: &str = "/tmp/ttserver.log"; 55 | 56 | let path = log_path.unwrap_or(LOG_PATH); 57 | let open = || { 58 | fs::OpenOptions::new() 59 | .read(true) 60 | .create(true) 61 | .append(true) 62 | .open(path) 63 | }; 64 | 65 | unistd::close(1) 66 | .c(d!()) 67 | .and_then(|_| open().c(d!())) 68 | .map(mem::forget) 69 | .and_then(|_| unistd::close(2).c(d!())) 70 | .and_then(|_| open().c(d!())) 71 | .map(mem::forget) 72 | } 73 | 74 | // 输出日志至文件 75 | #[cfg(feature = "testmock")] 76 | fn log_init(_log_path: Option<&str>) -> ruc::Result<()> { 77 | Ok(()) 78 | } 79 | -------------------------------------------------------------------------------- /src/server/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # tt-server 3 | //! 4 | //! 处理与 Client 端的交互逻辑. 5 | //! 6 | 7 | #![warn(missing_docs, unused_import_braces, unused_extern_crates)] 8 | 9 | pub mod cfg; 10 | mod def; 11 | mod hdr; 12 | mod init; 13 | mod util; 14 | 15 | use def::{DEFAULT_REQ_ID, OPS_ID_LEN}; 16 | use futures::executor::ThreadPool; 17 | use ruc::*; 18 | use std::{ 19 | mem, 20 | net::{SocketAddr, UdpSocket}, 21 | sync::{Arc, LazyLock}, 22 | }; 23 | use ttutils::zlib; 24 | use util::{genlog, p}; 25 | 26 | static POOL: LazyLock = LazyLock::new(|| pnk!(util::gen_thread_pool(Some(8)))); 27 | static CFG: LazyLock<&'static cfg::Cfg> = LazyLock::new(|| pnk!(cfg::register_cfg(None))); 28 | static SERV: LazyLock> = LazyLock::new(|| Arc::new(ttcore::Serv::new(&CFG.cfgdb_path))); 29 | static SOCK: LazyLock = LazyLock::new(|| pnk!(UdpSocket::bind(&CFG.serv_at).c(d!()))); 30 | 31 | /// 服务启动入口 32 | pub fn start(cfg: cfg::Cfg) -> ruc::Result<()> { 33 | pnk!(cfg::register_cfg(Some(cfg))); 34 | init::setenv() 35 | .c(d!()) 36 | .and_then(|_| ttcore::exec(&CFG.image_path, run, &CFG.serv_ip)) 37 | } 38 | 39 | #[inline(always)] 40 | fn run() -> ruc::Result<()> { 41 | // 必须在 clone 调用之后执行, 42 | // 否则会导致 POOL 在父进程中被初始化, 43 | // 进入子进程后只会保留一个主线程, 44 | // 且 LazyLock 不会再次初始化线程池. 45 | init::start_cron(); 46 | 47 | // 必须在 clone 调用之后执行, 48 | // 同样是因为 LazyLock 所限, 49 | load_exists().c(d!())?; 50 | 51 | // (C/S) 网络交互 52 | start_netserv(); 53 | } 54 | 55 | // 载入先前已存在的 ENV 实例 56 | fn load_exists() -> ruc::Result<()> { 57 | let mut vm_set; 58 | for (cli, env_set) in SERV.cfg_db.read_all().c(d!())?.into_iter() { 59 | for mut env in env_set.into_iter() { 60 | vm_set = mem::take(&mut env.vm) 61 | .into_iter() 62 | .map(|(_, vm_set)| vm_set) 63 | .collect(); 64 | env.load(&SERV) 65 | .c(d!()) 66 | .and_then(|mut env| { 67 | env.add_vm_set_complex(vec![], vm_set, true) 68 | .c(d!()) 69 | .map(|_| env) 70 | }) 71 | .and_then(|env| SERV.register_env(cli.clone(), env).c(d!()))?; 72 | } 73 | } 74 | 75 | Ok(()) 76 | } 77 | 78 | fn start_netserv() -> ! { 79 | let mut buf = vec![0; 8192]; 80 | loop { 81 | if let Ok((size, peeraddr)) = SOCK.recv_from(&mut buf) { 82 | if size < OPS_ID_LEN { 83 | continue; 84 | } 85 | parse_ops_id(&buf[..OPS_ID_LEN]) 86 | .c(d!()) 87 | .and_then(|ops_id| { 88 | let recvd = 89 | zlib::decode(&buf[OPS_ID_LEN..size]).c(d!())?; 90 | POOL.spawn_ok(async move { 91 | info_omit!(serv_it(ops_id, recvd, peeraddr).await); 92 | }); 93 | Ok(()) 94 | }) 95 | .unwrap_or_else(|e| { p(e); }); 96 | } 97 | } 98 | } 99 | 100 | /// 处理 Client 请求 101 | #[inline(always)] 102 | async fn serv_it( 103 | ops_id: usize, 104 | msg_body: Vec, 105 | peeraddr: SocketAddr, 106 | ) -> ruc::Result<()> { 107 | hdr::OPS_MAP 108 | .get(ops_id) 109 | .ok_or(eg!(format!("Unknown Request: {}", ops_id))) 110 | .and_then(|ops| ops(peeraddr, msg_body).c(d!())) 111 | .or_else(|e| send_err!(DEFAULT_REQ_ID, e, peeraddr)) 112 | } 113 | 114 | #[inline(always)] 115 | fn parse_ops_id(raw: &[u8]) -> ruc::Result { 116 | String::from_utf8_lossy(raw).parse::().c(d!()) 117 | } 118 | -------------------------------------------------------------------------------- /src/server/src/util.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Commiuncation With Client 3 | //! 4 | 5 | use crate::{def::*, POOL, SOCK}; 6 | use futures::executor::{ThreadPool, ThreadPoolBuilder}; 7 | use futures_timer::Delay; 8 | use ruc::*; 9 | use serde::Serialize; 10 | use std::{net::SocketAddr, time::Duration}; 11 | use ttutils::zlib; 12 | 13 | /// Generate log message from error 14 | pub(crate) fn genlog(err: E) -> String { 15 | format!("{}", err) 16 | } 17 | 18 | /// Print function for logging/debugging 19 | pub(crate) fn p(msg: T) -> T { 20 | eprintln!("{}", msg); 21 | msg 22 | } 23 | 24 | /// 生成异步框架底层的线程池 25 | pub(crate) fn gen_thread_pool(n: Option) -> ruc::Result { 26 | ThreadPoolBuilder::new() 27 | .pool_size(n.map(|n| n as usize).unwrap_or_else(num_cpus::get)) 28 | .create() 29 | .c(d!()) 30 | } 31 | 32 | /// 回送成功信息 33 | #[macro_export] 34 | macro_rules! send_ok { 35 | ($uuid: expr, $msg: expr, $peeraddr: expr) => { 36 | $crate::util::send_back( 37 | $crate::util::gen_resp_ok($uuid, $msg), 38 | $peeraddr, 39 | ) 40 | }; 41 | } 42 | 43 | /// 生成标志'成功'的回复体 44 | pub(crate) fn gen_resp_ok(uuid: u64, msg: impl Serialize) -> Resp { 45 | Resp { 46 | uuid, 47 | status: RetStatus::Success, 48 | msg: info!(serde_json::to_vec(&msg)).unwrap_or_default(), 49 | } 50 | } 51 | 52 | /// 回送失败信息 53 | #[macro_export] 54 | macro_rules! send_err { 55 | ($uuid: expr, $err: expr, $peeraddr: expr) => {{ 56 | let log = genlog($err); 57 | $crate::util::send_back( 58 | $crate::util::gen_resp_err($uuid, &log), 59 | $peeraddr, 60 | ) 61 | .c(d!(&log)) 62 | .map_err(|e| { p(eg!(log)); e }) 63 | }}; 64 | } 65 | 66 | /// 生成标志'出错'的回复体 67 | pub(crate) fn gen_resp_err(uuid: u64, msg: &str) -> Resp { 68 | Resp { 69 | uuid, 70 | status: RetStatus::Fail, 71 | msg: msg.as_bytes().to_vec(), 72 | } 73 | } 74 | 75 | /// 回送信息至请求方(Client/Proxy) 76 | #[inline(always)] 77 | pub(crate) fn send_back(resp: Resp, peeraddr: SocketAddr) -> ruc::Result<()> { 78 | serde_json::to_vec(&resp) 79 | .c(d!()) 80 | .and_then(|resp| zlib::encode(&resp[..]).c(d!())) 81 | .map(|resp_compressed| { 82 | POOL.spawn_ok(async move { 83 | info_omit!(SOCK.send_to(&resp_compressed, peeraddr)); 84 | }); 85 | }) 86 | } 87 | 88 | /// 异步 sleep 89 | pub(crate) async fn asleep(sec: u64) { 90 | Delay::new(Duration::from_secs(sec)).await; 91 | } 92 | -------------------------------------------------------------------------------- /src/server/tests/env/mod.rs: -------------------------------------------------------------------------------- 1 | use lazy_static::lazy_static; 2 | use ruc::*; 3 | use nix::unistd::getuid; 4 | use serde::Serialize; 5 | use std::{ 6 | collections::HashMap, 7 | fs, 8 | net::{SocketAddr, UdpSocket}, 9 | thread, 10 | time::Duration, 11 | }; 12 | use ttserver::cfg::Cfg; 13 | use ttserver_def::*; 14 | use ttutils::zlib; 15 | 16 | pub(super) const CPU_TOTAL: i32 = 48; 17 | pub(super) const MEM_TOTAL: i32 = 64 * 1024; 18 | pub(super) const DISK_TOTAL: i32 = 1000 * 1024; 19 | 20 | lazy_static! { 21 | static ref CLI_SOCK: UdpSocket = pnk!(gen_sock(1)); 22 | static ref SERV_ADDR: SocketAddr = 23 | pnk!("127.0.0.1:9527".parse::()); 24 | static ref OPS_MAP: HashMap<&'static str, u8> = map! { 25 | "register_client_id" => 0, 26 | "get_server_info" => 1, 27 | "get_env_list" => 2, 28 | "get_env_info" => 3, 29 | "add_env" => 4, 30 | "del_env" => 5, 31 | "update_env_lifetime" => 6, 32 | "update_env_kick_vm" => 7, 33 | }; 34 | } 35 | 36 | pub(super) fn get_uid() -> u32 { 37 | getuid().as_raw() 38 | } 39 | 40 | // 异步启动 Server 41 | pub(super) fn start_server() { 42 | assert_eq!( 43 | 0, 44 | get_uid(), 45 | "\x1b[31;1mMust be root to run this test!\x1b[0m" 46 | ); 47 | 48 | thread::spawn(|| { 49 | pnk!(ttserver::start(mock_cfg())); 50 | }); 51 | 52 | thread::sleep(Duration::from_secs(1)); 53 | } 54 | 55 | fn mock_cfg() -> Cfg { 56 | let cfgdb_path = format!("/tmp/{}", ts!()); 57 | pnk!(fs::create_dir_all(&cfgdb_path)); 58 | Cfg { 59 | log_path: None, 60 | serv_ip: "127.0.0.1".to_owned(), 61 | serv_at: "127.0.0.1:9527".to_owned(), 62 | image_path: "/mnt/".to_owned(), 63 | cfgdb_path, 64 | cpu_total: CPU_TOTAL, 65 | mem_total: MEM_TOTAL, 66 | disk_total: DISK_TOTAL, 67 | } 68 | } 69 | 70 | /// 发送请求信息 71 | pub(super) fn send_req(ops: &str, req: Req) -> Result { 72 | let ops_id = OPS_MAP 73 | .get(ops) 74 | .copied() 75 | .ok_or(eg!(format!("Unknown request: {}", ops)))?; 76 | 77 | let mut req_bytes = serde_json::to_vec(&req) 78 | .c(d!()) 79 | .and_then(|req| zlib::encode(&req[..]).c(d!()))?; 80 | let mut body = 81 | format!("{id:>0width$}", id = ops_id, width = OPS_ID_LEN).into_bytes(); 82 | body.append(&mut req_bytes); 83 | 84 | CLI_SOCK.send_to(&body, *SERV_ADDR).c(d!()).and_then(|_| { 85 | let mut buf = vct![0; 8 * 4096]; 86 | let size = CLI_SOCK.recv(&mut buf).c(d!())?; 87 | 88 | zlib::decode(&buf[..size]) 89 | .c(d!()) 90 | .and_then(|resp_decompressed| { 91 | serde_json::from_slice(&resp_decompressed).c(d!()) 92 | }) 93 | }) 94 | } 95 | 96 | fn gen_sock(timeout: u64) -> Result { 97 | let mut addr; 98 | for port in (40_000 + ts!() % 10_000)..60_000 { 99 | addr = SocketAddr::from(([127, 0, 0, 1], port as u16)); 100 | if let Ok(sock) = UdpSocket::bind(addr) { 101 | sock.set_read_timeout(Some(Duration::from_secs(timeout))) 102 | .c(d!())?; 103 | return Ok(sock); 104 | } 105 | } 106 | Err(eg!()) 107 | } 108 | -------------------------------------------------------------------------------- /src/server/tests/integration.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Integration Tests. 3 | //! 4 | 5 | #![cfg(feature = "testmock")] 6 | 7 | mod env; 8 | mod knead; 9 | mod standalone; 10 | 11 | #[test] 12 | fn i_ttserver() { 13 | if 0 == env::get_uid() { 14 | env::start_server(); 15 | standalone::test(); 16 | knead::test(); 17 | } else { 18 | println!("\x1b[31;01mNOT root, ignore...\x1b[00m"); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/server/tests/standalone/mod.rs: -------------------------------------------------------------------------------- 1 | use super::env::*; 2 | use ruc::*; 3 | use std::collections::HashMap; 4 | use ttserver_def::*; 5 | 6 | const CUSTOM_CLI_ID: &str = "ErHa"; 7 | 8 | // 孤立测试每一个接口 9 | pub(super) fn test() { 10 | t_register_client_id(); 11 | t_get_server_info(); 12 | t_get_env_list(); 13 | t_get_env_info(); 14 | t_add_env(); 15 | t_del_env(); 16 | t_update_env_lifetime(); 17 | t_update_env_kick_vm(); 18 | } 19 | 20 | fn t_register_client_id() { 21 | assert!( 22 | send_req( 23 | "register_client_id", 24 | Req::new(0, CUSTOM_CLI_ID.to_owned(), "") 25 | ) 26 | .is_ok() 27 | ); 28 | assert!( 29 | send_req( 30 | "register_client_id", 31 | Req::new(0, CUSTOM_CLI_ID.to_owned(), "") 32 | ) 33 | .is_ok() 34 | ); 35 | } 36 | 37 | fn t_get_server_info() { 38 | let uuid = 1111; 39 | let resp = pnk!(send_req( 40 | "get_server_info", 41 | Req::new(uuid, CUSTOM_CLI_ID.to_owned(), "") 42 | )); 43 | 44 | assert_eq!(resp.uuid, uuid); 45 | assert_eq!(resp.status, RetStatus::Success); 46 | 47 | let body = pnk!(serde_json::from_slice::< 48 | HashMap, 49 | >(&resp.msg)); 50 | assert_eq!(1, body.len()); 51 | 52 | let body = body.into_iter().next().unwrap().1; 53 | assert_eq!(body.vm_total, 0); 54 | assert_eq!(body.cpu_total, CPU_TOTAL); 55 | assert_eq!(body.mem_total, MEM_TOTAL); 56 | assert_eq!(body.disk_total, DISK_TOTAL); 57 | assert_eq!(body.cpu_used, 0); 58 | assert_eq!(body.mem_used, 0); 59 | assert_eq!(body.disk_used, 0); 60 | assert!(!body.supported_list.is_empty()); 61 | } 62 | 63 | // 在 add_env 之前调用 64 | fn t_get_env_list() { 65 | let uuid = 1111; 66 | 67 | let resp = pnk!(send_req( 68 | "get_env_list", 69 | Req::new(uuid, CUSTOM_CLI_ID.to_owned(), "") 70 | )); 71 | 72 | assert_eq!(resp.uuid, uuid); 73 | assert_eq!(resp.status, RetStatus::Success); 74 | 75 | let body = pnk!( 76 | serde_json::from_slice::>(&resp.msg) 77 | ); 78 | assert_eq!(1, body.len()); 79 | 80 | let body = body.into_iter().next().unwrap().1; 81 | assert!(body.is_empty()); 82 | } 83 | 84 | fn t_get_env_info() { 85 | let uuid = 1111; 86 | 87 | let msg = ReqGetEnvInfo { 88 | env_set: vct!["abcxxx".to_owned(), "xxxabc".to_owned()], 89 | }; 90 | 91 | let resp = pnk!(send_req( 92 | "get_env_info", 93 | Req::new(uuid, CUSTOM_CLI_ID.to_owned(), msg) 94 | )); 95 | 96 | assert_eq!(resp.uuid, uuid); 97 | assert_eq!(resp.status, RetStatus::Success); 98 | 99 | let body = pnk!( 100 | serde_json::from_slice::>(&resp.msg) 101 | ); 102 | assert_eq!(1, body.len()); 103 | 104 | let body = body.into_iter().next().unwrap().1; 105 | assert!(body.is_empty()); 106 | } 107 | 108 | fn t_add_env() { 109 | let uuid = 1111; 110 | 111 | let msg = ReqAddEnv { 112 | env_id: "UselessEnv".to_owned(), 113 | os_prefix: vct!["c".to_owned(), "u".to_owned()], 114 | life_time: None, 115 | cpu_num: None, 116 | mem_size: Some(512), 117 | disk_size: None, 118 | port_set: vct![], 119 | dup_each: None, 120 | deny_outgoing: false, 121 | rand_uuid: true, 122 | vmcfg: None, 123 | }; 124 | 125 | let resp = pnk!(send_req( 126 | "add_env", 127 | Req::new(uuid, CUSTOM_CLI_ID.to_owned(), msg) 128 | )); 129 | 130 | assert_eq!(resp.uuid, uuid); 131 | assert_eq!(resp.status, RetStatus::Success); 132 | 133 | let body = pnk!(serde_json::from_slice::(&resp.msg)); 134 | assert_eq!("Success!", &body); 135 | } 136 | 137 | fn t_del_env() { 138 | let uuid = 1111; 139 | 140 | let msg = ReqDelEnv { 141 | env_id: "UselessEnv".to_owned(), 142 | }; 143 | 144 | let resp = pnk!(send_req( 145 | "del_env", 146 | Req::new(uuid, CUSTOM_CLI_ID.to_owned(), msg) 147 | )); 148 | 149 | assert_eq!(resp.uuid, uuid); 150 | assert_eq!(resp.status, RetStatus::Success); 151 | 152 | let body = pnk!(serde_json::from_slice::(&resp.msg)); 153 | assert_eq!("Success!", &body); 154 | } 155 | 156 | fn t_update_env_lifetime() { 157 | let uuid = 1111; 158 | 159 | let msg = ReqUpdateEnvLife { 160 | env_id: "UselessEnv".to_owned(), 161 | life_time: 88888888888888, 162 | is_fucker: true, 163 | }; 164 | 165 | let resp = pnk!(send_req( 166 | "update_env_lifetime", 167 | Req::new(uuid, CUSTOM_CLI_ID.to_owned(), msg) 168 | )); 169 | 170 | assert_eq!(resp.uuid, uuid); 171 | assert_eq!(resp.status, RetStatus::Fail); 172 | } 173 | 174 | fn t_update_env_kick_vm() { 175 | let uuid = 1111; 176 | 177 | let msg = ReqUpdateEnvKickVm { 178 | env_id: "UselessEnv".to_owned(), 179 | vm_id: vct![], 180 | os_prefix: vct!["c".to_owned(), "u".to_owned()], 181 | }; 182 | 183 | let resp = pnk!(send_req( 184 | "update_env_kick_vm", 185 | Req::new(uuid, CUSTOM_CLI_ID.to_owned(), msg) 186 | )); 187 | 188 | assert_eq!(resp.uuid, uuid); 189 | assert_eq!(resp.status, RetStatus::Success); 190 | 191 | let body = pnk!(serde_json::from_slice::(&resp.msg)); 192 | assert_eq!("Success!", &body); 193 | } 194 | -------------------------------------------------------------------------------- /src/server_def/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /src/server_def/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ttserver_def" 3 | version = "0.3.2" 4 | 5 | edition = "2024" 6 | authors = ["FanHui ", "FanHui "] 7 | description = "Lightweight private cloud solution for SME scenarios." 8 | license = "MIT OR Apache-2.0" 9 | keywords = ["firecracker", "qemu", "kvm", "openstack", "k8s", "cloud"] 10 | categories = ["command-line-utilities"] 11 | 12 | [dependencies] 13 | serde = { version = "1.0.216", features = ["derive"] } 14 | ttcore_def = { path = "../core_def" } 15 | ruc = "8.1.2" 16 | -------------------------------------------------------------------------------- /src/server_def/src/included_ops_map.rs: -------------------------------------------------------------------------------- 1 | 2 | /// 对应 Client 端的请求 3 | pub(crate) const OPS_MAP: &[Ops] = &[ 4 | /*0*/ register_client_id, 5 | /*1*/ get_server_info, 6 | /*2*/ get_env_list, 7 | /*3*/ get_env_info, 8 | /*4*/ add_env, 9 | /*5*/ del_env, 10 | /*6*/ update_env_lifetime, 11 | /*7*/ update_env_kick_vm, 12 | /*8*/ get_env_list_all, 13 | /*9*/ stop_env, 14 | /*10*/ start_env, 15 | /*11*/ update_env_resource, 16 | ]; 17 | -------------------------------------------------------------------------------- /src/server_def/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # 基本类型定义 3 | //! 4 | 5 | use ruc::*; 6 | use serde::{Deserialize, Serialize}; 7 | use std::fmt; 8 | pub use ttcore_def::*; 9 | 10 | /// ops_id 的字符长度, eg: "1234" 11 | pub const OPS_ID_LEN: usize = 4; 12 | 13 | /// 无法获取 uuid 时使用此默认 id 14 | pub const DEFAULT_REQ_ID: u64 = u64::MAX; 15 | 16 | /// uuid of req/resp 17 | pub type UUID = u64; 18 | 19 | /// - format: ":" 20 | /// - eg: "10.10.10.22:9527" 21 | pub type ServerAddr = String; 22 | 23 | /// Client 发送的信息 24 | #[derive(Debug, Default, Deserialize, Serialize)] 25 | pub struct Req { 26 | /// rpc uuid 27 | pub uuid: u64, 28 | /// 客户端标识 29 | pub cli_id: CliId, 30 | /// 消息正文 31 | pub msg: T, 32 | } 33 | 34 | impl Req { 35 | /// create a new instance 36 | pub fn new(uuid: u64, cli_id: CliId, msg: T) -> Self { 37 | Req { uuid, cli_id, msg } 38 | } 39 | } 40 | 41 | /// 服务端的执行结果 42 | #[allow(missing_docs)] 43 | #[derive( 44 | Clone, Copy, Debug, Deserialize, Serialize, Eq, PartialEq, Ord, PartialOrd, 45 | )] 46 | pub enum RetStatus { 47 | Fail, 48 | Success, 49 | } 50 | 51 | impl fmt::Display for RetStatus { 52 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 53 | let msg = match self { 54 | RetStatus::Fail => "Fail", 55 | RetStatus::Success => "Success", 56 | }; 57 | 58 | write!(f, "{}", msg) 59 | } 60 | } 61 | 62 | /// 返回给 Client 的信息 63 | #[derive(Debug, Deserialize, Serialize)] 64 | pub struct Resp { 65 | /// rpc uuid 66 | pub uuid: u64, 67 | /// Fail? Success? 68 | pub status: RetStatus, 69 | /// 消息正文 70 | pub msg: Vec, 71 | } 72 | 73 | impl fmt::Display for Resp { 74 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 75 | write!( 76 | f, 77 | "status: {}, msg: {}", 78 | self.status, 79 | String::from_utf8_lossy(&self.msg) 80 | ) 81 | } 82 | } 83 | 84 | #[allow(missing_docs)] 85 | #[derive( 86 | Clone, 87 | Debug, 88 | Default, 89 | Deserialize, 90 | Serialize, 91 | Eq, 92 | PartialEq, 93 | Ord, 94 | PartialOrd, 95 | )] 96 | pub struct RespGetServerInfo { 97 | pub vm_total: i32, 98 | pub cpu_total: i32, 99 | pub cpu_used: i32, 100 | pub mem_total: i32, 101 | pub mem_used: i32, 102 | pub disk_total: i32, 103 | pub disk_used: i32, 104 | pub supported_list: Vec, 105 | } 106 | 107 | /// 直接使用 core 模块返回的结果 108 | pub type RespGetEnvList = Vec; 109 | 110 | /// 公开给 Cli 使用 111 | #[allow(missing_docs)] 112 | #[derive(Debug, Default, Deserialize, Serialize)] 113 | pub struct ReqGetEnvInfo { 114 | pub env_set: Vec, 115 | } 116 | 117 | /// 直接使用 core 模块返回的结果 118 | pub type RespGetEnvInfo = Vec; 119 | 120 | /// 公开给 Cli 使用 121 | #[allow(missing_docs)] 122 | #[derive(Clone, Debug, Default, Deserialize, Serialize)] 123 | pub struct ReqAddEnv { 124 | pub env_id: EnvId, 125 | pub life_time: Option, 126 | pub dup_each: Option, 127 | pub deny_outgoing: bool, 128 | 129 | /// 若此项为不空, 130 | /// 则优先使用此项的信息 131 | pub vmcfg: Option>, 132 | 133 | /// 若vmcfg字段为空值, 134 | /// 使用这些字段从头解析 135 | pub os_prefix: Vec, 136 | pub cpu_num: Option, 137 | pub mem_size: Option, 138 | pub disk_size: Option, 139 | pub port_set: Vec, 140 | pub rand_uuid: bool, 141 | } 142 | 143 | impl ReqAddEnv { 144 | /// auto add ssh port map 145 | pub fn set_ssh_port(&mut self) { 146 | let set = |data: &mut Vec| { 147 | data.push(SSH_PORT); 148 | data.push(TTREXEC_PORT); 149 | data.sort_unstable(); 150 | data.dedup(); 151 | }; 152 | 153 | if let Some(vc) = self.vmcfg.as_mut() { 154 | // requests from TTproxy 155 | vc.iter_mut().for_each(|cfg| { 156 | set(&mut cfg.port_list); 157 | }); 158 | } else { 159 | // requests from TTclient 160 | set(&mut self.port_set); 161 | } 162 | } 163 | 164 | /// OS prefix matching is not case sensitive 165 | pub fn set_os_lowercase(&mut self) { 166 | self.os_prefix 167 | .iter_mut() 168 | .for_each(|os| *os = os.to_lowercase()); 169 | } 170 | 171 | /// check if the number of `dup` is out of limit 172 | pub fn check_dup(&self) -> ruc::Result { 173 | const DUP_MAX: u32 = 500; 174 | let dup_each = self.dup_each.unwrap_or(0); 175 | if DUP_MAX < dup_each { 176 | Err(eg!(format!( 177 | "the number of `dup` too large: {}(max {})", 178 | dup_each, DUP_MAX 179 | ))) 180 | } else { 181 | Ok(dup_each) 182 | } 183 | } 184 | } 185 | 186 | /// 公开给 Cli 使用 187 | #[allow(missing_docs)] 188 | #[derive(Clone, Debug, Default, Deserialize, Serialize)] 189 | pub struct ReqStopEnv { 190 | pub env_id: EnvId, 191 | } 192 | 193 | /// 公开给 Cli 使用 194 | pub type ReqStartEnv = ReqStopEnv; 195 | 196 | /// 公开给 Cli 使用 197 | #[allow(missing_docs)] 198 | #[derive(Debug, Default, Deserialize, Serialize)] 199 | pub struct ReqUpdateEnvLife { 200 | pub env_id: EnvId, 201 | pub life_time: u64, 202 | pub is_fucker: bool, 203 | } 204 | 205 | /// 公开给 Cli 使用 206 | #[allow(missing_docs)] 207 | #[derive(Debug, Default, Deserialize, Serialize)] 208 | pub struct ReqUpdateEnvResource { 209 | pub env_id: EnvId, 210 | pub cpu_num: Option, 211 | pub mem_size: Option, 212 | pub disk_size: Option, 213 | pub vm_port: Vec, 214 | pub deny_outgoing: Option, 215 | } 216 | 217 | /// 公开给 Cli 使用 218 | #[allow(missing_docs)] 219 | #[derive(Debug, Default, Deserialize, Serialize)] 220 | pub struct ReqDelEnv { 221 | pub env_id: EnvId, 222 | } 223 | 224 | /// 公开给 Cli 使用 225 | #[allow(missing_docs)] 226 | #[derive(Debug, Default, Deserialize, Serialize)] 227 | pub struct ReqUpdateEnvKickVm { 228 | pub env_id: EnvId, 229 | pub vm_id: Vec, 230 | pub os_prefix: Vec, 231 | } 232 | -------------------------------------------------------------------------------- /src/utils/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ttutils" 3 | version = "0.3.2" 4 | 5 | authors = ["FanHui "] 6 | edition = "2024" 7 | description = "Lightweight private cloud solution for SME scenarios." 8 | license = "MIT OR Apache-2.0" 9 | keywords = ["firecracker", "qemu", "kvm", "openstack", "k8s", "cloud"] 10 | categories = ["command-line-utilities"] 11 | 12 | [dependencies] 13 | flate2 = "1.0.35" 14 | ruc = "8.1.2" 15 | 16 | [dev-dependencies] 17 | rand = "0.8.5" 18 | -------------------------------------------------------------------------------- /src/utils/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Common Utils 3 | //! 4 | 5 | #![warn(missing_docs, unused_import_braces, unused_extern_crates)] 6 | 7 | /// (de)compression 8 | pub mod zlib { 9 | use flate2::{ 10 | write::{ZlibDecoder, ZlibEncoder}, 11 | Compression, 12 | }; 13 | use ruc::*; 14 | use std::io::Write; 15 | 16 | /// compress 17 | pub fn encode(data: &[u8]) -> Result> { 18 | let mut en = ZlibEncoder::new(Vec::new(), Compression::default()); 19 | en.write_all(data).c(d!())?; 20 | en.finish().c(d!()) 21 | } 22 | 23 | /// decompress 24 | pub fn decode(data: &[u8]) -> Result> { 25 | let mut d = ZlibDecoder::new(Vec::new()); 26 | d.write_all(data).c(d!()).and_then(|_| d.finish().c(d!())) 27 | } 28 | 29 | #[cfg(test)] 30 | mod tests { 31 | use rand::random; 32 | 33 | #[test] 34 | fn it_works() { 35 | (0..(10 + random::() % 20)) 36 | .map(|i| (0..i).map(|_| random::()).collect::>()) 37 | .for_each(|sample| { 38 | assert_eq!(sample, super::decode(&super::encode(&sample).unwrap()).unwrap()); 39 | }); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /tools/firecracker_update_image.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | zfspath="zroot/tt" 4 | if [[ "" != $1 ]]; then 5 | zfspath=$1 6 | fi 7 | 8 | oslist=" 9 | fire:centos-7.x:3.10.0-693.el7.x86_64 10 | fire:centos-7.x:3.10.0-862.el7.x86_64 11 | fire:centos-7.x:3.10.0-957.el7.x86_64 12 | fire:centos-7.x:3.10.0-1062.el7.x86_64 13 | fire:centos-7.x:3.10.0-1127.el7.x86_64 14 | 15 | fire:centos-7.x:4.14.0-115.el7.0.1.x86_64 16 | 17 | fire:centos-8.x:4.18.0-80.el8.x86_64 18 | fire:centos-8.x:4.18.0-80.1.2.el8.x86_64 19 | fire:centos-8.x:4.18.0-80.4.2.el8.x86_64 20 | fire:centos-8.x:4.18.0-80.7.1.el8.x86_64 21 | fire:centos-8.x:4.18.0-80.7.2.el8.x86_64 22 | fire:centos-8.x:4.18.0-80.11.1.el8.x86_64 23 | fire:centos-8.x:4.18.0-80.11.2.el8.x86_64 24 | 25 | fire:centos-8.x:4.18.0-147.el8.x86_64 26 | fire:centos-8.x:4.18.0-147.0.3.el8.x86_64 27 | fire:centos-8.x:4.18.0-147.3.1.el8.x86_64 28 | fire:centos-8.x:4.18.0-147.5.1.el8.x86_64 29 | fire:centos-8.x:4.18.0-147.8.1.el8.x86_64 30 | 31 | fire:centos-8.x:4.18.0-193.el8.x86_64 32 | fire:centos-8.x:4.18.0-193.1.2.el8.x86_64 33 | fire:centos-8.x:4.18.0-193.6.3.el8.x86_64 34 | fire:centos-8.x:4.18.0-193.14.2.el8.x86_64 35 | fire:centos-8.x:4.18.0-193.19.1.el8.x86_64 36 | " 37 | 38 | oslist_dev=" 39 | fire:dev:4.19.148.x86_64 40 | " 41 | 42 | oslist_bad=" 43 | fire:centos-7.x:3.10.0-123.el7.x86_64 44 | fire:centos-7.x:3.10.0-229.el7.x86_64 45 | fire:centos-7.x:3.10.0-327.el7.x86_64 46 | fire:centos-7.x:3.10.0-514.el7.x86_64 47 | " 48 | 49 | destroy_old() { 50 | for x in $(echo ${oslist}); do 51 | zfs destroy -R ${zfspath}/${x}@base 52 | done 53 | 54 | for x in $(echo ${oslist_dev}); do 55 | zfs destroy -R ${zfspath}/${x}@base 56 | done 57 | 58 | for x in $(echo ${oslist_bad}); do 59 | zfs destroy -R ${zfspath}/${x}@base 2>/dev/null 60 | done 61 | 62 | zfs destroy -R ${zfspath}/firecracker@base 63 | zfs destroy -R ${zfspath}/firecracker-dev@base 64 | } 65 | 66 | create_new() { 67 | zfs snapshot ${zfspath}/firecracker@base || exit 1 68 | for x in $(echo ${oslist}); do 69 | zfs clone ${zfspath}/firecracker@base ${zfspath}/$x || exit 1 70 | zfs snapshot ${zfspath}/${x}@base || exit 1 71 | done 72 | 73 | zfs snapshot ${zfspath}/firecracker-dev@base || exit 1 74 | for x in $(echo ${oslist_dev}); do 75 | zfs clone ${zfspath}/firecracker-dev@base ${zfspath}/${x} || exit 1 76 | zfs snapshot ${zfspath}/${x}@base || exit 1 77 | done 78 | } 79 | 80 | destroy 81 | destroy_old 82 | create_new 83 | -------------------------------------------------------------------------------- /tools/fmt.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ################################################# 4 | #### Ensure we are in the right path. ########### 5 | ################################################# 6 | if [[ 0 -eq $(echo $0 | grep -c '^/') ]]; then 7 | # relative path 8 | EXEC_PATH=$(dirname "`pwd`/$0") 9 | else 10 | # absolute path 11 | EXEC_PATH=$(dirname "$0") 12 | fi 13 | 14 | EXEC_PATH=$(echo ${EXEC_PATH} | sed 's@/\./@/@g' | sed 's@/\.*$@@') 15 | cd $EXEC_PATH || exit 1 16 | ################################################# 17 | 18 | export LC_ALL=en_US.UTF-8 # perl 19 | 20 | for file in $(find .. -type f \ 21 | -name "*.rs" \ 22 | -o -name "*.c" \ 23 | -o -name "*.h" \ 24 | -o -name "*.sh" \ 25 | -o -name "*.toml" \ 26 | -o -name "*.json" \ 27 | -o -name "*.md"\ 28 | -o -name "rc.local"\ 29 | | grep -v "$(basename $0)" \ 30 | | grep -v 'target/' \ 31 | | grep -v 'postgres'); do 32 | 33 | perl -p -i -e 's/　/ /g' $file 34 | perl -p -i -e 's/！/!/g' $file 35 | perl -p -i -e 's/（/(/g' $file 36 | perl -p -i -e 's/）/)/g' $file 37 | 38 | perl -p -i -e 's/：/: /g' $file 39 | perl -p -i -e 's/， */, /g' $file 40 | perl -p -i -e 's/。 */. /g' $file 41 | perl -p -i -e 's/、 +/、/g' $file 42 | 43 | perl -p -i -e 's/, +/, /g' $file 44 | perl -p -i -e 's/\. +/. /g' $file 45 | 46 | perl -p -i -e 's/\t/ /g' $file 47 | perl -p -i -e 's/ +$//g' $file 48 | done 49 | 50 | ~/.cargo/bin/cargo +nightly fmt --all 51 | -------------------------------------------------------------------------------- /tools/githooks/pre-commit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | make lint || exit 1 4 | 5 | make test || exit 1 6 | -------------------------------------------------------------------------------- /tools/githooks/pre-commit.sample: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 具体的行为自定义, 4 | # 通常是执行 `go vet` 或 `cargo clippy` 等规范性较验流程 5 | make lint || exit 1 6 | 7 | # 具体的行为自定义, 8 | # 通常是执行自动化测试用例 9 | make test || exit 1 10 | 11 | # 连接到 TT 私有云系统, 12 | # 执行多系统兼容性测试 13 | tt_auto_test() { 14 | # 将在 TT 系统中创建的环境名称 15 | env="$(hostname)$(date +%s)$RANDOM" 16 | 17 | # 需要参与测试的系统列表; 18 | # 需要提前在 TT 中准备好基础镜像, 按需更改 19 | os_list="fire:centos-7.x:3.10.0,fire:centos-7.x:4.14,fire:centos-8.x:4.18" 20 | 21 | if [[ "Linux" = $(uname -s) ]]; then 22 | cp --reflink=auto -rp $(pwd) /tmp/${env} || exit 1 23 | else 24 | cp -rp $(pwd) /tmp/${env} || exit 1 25 | fi 26 | 27 | # 清理可能存在的无关文件 28 | cwd=$(pwd) 29 | cd /tmp/${env} || exit 1 30 | mv -f .git/hooks/* /tmp/ || exit 1 31 | git commit -m 'test' || exit 1 32 | git reset --hard HEAD || exit 1 33 | git clean -fdx || exit 1 34 | rm -rf .git || exit 1 35 | cd ${cwd} || exit 1 36 | 37 | tar -C /tmp -cpf /tmp/${env}.tar ${env} || exit 1 38 | rm -rf /tmp/${env} 2>/dev/null 39 | 40 | # 创建临时测试环境; 41 | # 生命周期 180 秒, 可根据项目实际情况按需更改 42 | tt env add ${env} -s ${os_list} -C 2 -M 2048 -l 180 || exit 1 43 | 44 | # 等待全部虚拟机启动完毕 45 | sleep 10 46 | 47 | # 将代码快照的 tar 包传送到测试环境中; 48 | # 推送超时时间 50 秒, 可根据代码包的大小及网络速度按需更改 49 | tt env push ${env} -f /tmp/${env}.tar -t 50 || exit 1 50 | 51 | # 在所有虚拟机中批量执行自动化测试; 52 | # 测试超时时间 60 秒, 可根据项目实际情况按需更改 53 | tt env run ${env} -c "tar -xpf /tmp/${env}.tar && make test -C ${env}" -t 60 54 | 55 | # 销毁测试环境, 释放资源; 56 | # 不手动销毁也可以, 到达设定的生命周期时, 会自动销毁 57 | tt env del ${env} 58 | } 59 | 60 | # 按需决定是否启用 TT 运程多系统测试 61 | # 若本地测试能满足需求, 可注释掉此行 62 | tt_auto_test 63 | 64 | -------------------------------------------------------------------------------- /tools/images/README.md: -------------------------------------------------------------------------------- 1 | # images 2 | 3 | 运行时镜像相关的说明. 4 | 5 | ## rc.local 6 | 7 | 各镜像内部均已预置本路径下的`rc.local`开机启动项. 8 | 9 | #### 示例 10 | 11 | 客户机`MAC`地址的末尾两段数字转化为十进制后, 作为`IP`地址的后两段使用. 12 | 13 | 因此为虚拟机设置`MAC`地址的同时, 会自动设置其`IP`地址. 14 | 15 | ```shell 16 | # MAC 地址 17 | f4:b5:20:1b:3a:83 18 | 19 | # IP 地址 20 | 10.10.58.131 21 | ``` 22 | 23 | ## Linux Images 24 | 25 | ```shell 26 | # tree -F 27 | . 28 | ├── centos/ 29 | │   ├── qemu:CentOS-7.0.qcow2:default 30 | │   ├── qemu:CentOS-7.1.qcow2:default 31 | │   ├── qemu:CentOS-7.2.qcow2:default 32 | │   ├── qemu:CentOS-7.3.qcow2:default 33 | │   ├── qemu:CentOS-7.4.qcow2:default 34 | │   ├── qemu:CentOS-7.5.qcow2:default 35 | │   ├── qemu:CentOS-7.6.qcow2:default 36 | │   ├── qemu:CentOS-7.7.qcow2:default 37 | │   ├── qemu:CentOS-7.8.qcow2:default 38 | │   └── qemu:CentOS-8.2.qcow2:default 39 | └── ubuntu/ 40 | ├── qemu:ubuntu-14.04.qcow2:default 41 | ├── qemu:ubuntu-16.04.qcow2:default 42 | ├── qemu:ubuntu-18.04.qcow2:default 43 | └── qemu:ubuntu-20.04.qcow2:default 44 | ``` 45 | -------------------------------------------------------------------------------- /tools/images/freebsd_vm/rc.local: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set_ip() { 4 | if=$(ifconfig | grep -v '^lo' | grep -o '^[^[:blank:]]*:' | grep -o '[^:]*') 5 | mac=$(ifconfig | grep -A 2 "${if}:" | grep "ether" | grep -Eo '(\w{2}:){5}\w{2}') 6 | field1=$(echo $(( 0x$(echo $mac | grep -Eo '\w{2}:\w{2}$' | grep -Eo '^\w+{2}') ))) 7 | field2=$(echo $(( 0x$(echo $mac | grep -Eo '\w{2}$') ))) 8 | addr="10.10.${field1}.${field2}/8" 9 | 10 | ifconfig $if flush 11 | ifconfig $if inet $addr up 12 | route flush 13 | route add default 10.0.0.1 14 | } 15 | 16 | ssh_login() { 17 | user="root" 18 | # passwd -d $user 19 | sed -ri."" 's/^ *#* *(PermitEmptyPasswords).*/\1 yes/' /etc/ssh/sshd_config 20 | sed -ri."" 's/^ *#* *(PermitRootLogin).*/\1 yes/' /etc/ssh/sshd_config 21 | sed -ri."" 's/^ *#* *(UsePAM).*/\1 yes/' /etc/ssh/sshd_config 22 | sed -ri."" 's/^ *#* *(UseDNS).*/\1 no/' /etc/ssh/sshd_config 23 | 24 | service sshd restart 25 | } 26 | 27 | set_pub_key() { 28 | PUBKEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDk/CSaXip7wwa2xsDaZm6eKmyET91fI28ufdVMZvsym7s8y0QVxdO16t9IlbH9ognp/zLETJSl1MxoEz1R2I9rE+R0eraeBT/LIsF5nIyMBZ/Zk5SRNrociW+9w4AxWZHcugGznVFIxbeC2Ja/q4xxDWRbtPB5nv2R92eTwZZ6TovxjmSGXUcBVRVFgvE0k7KgdRCGK9q0JOUZMT2d3fs8FkOJPeSznnawaUiLlLYFsY1R4F/+0r+zlbDgKrBmnjv1GYKohBcFMsw10m08cU3UjJym67R0I2NKMAFhWBZAh4IGMIfg7w3Ua2K4nHQwlcnOWuZ7jzll/nYRmrS9T3LF root@718f9c9841a8" 29 | sshpath="/root/.ssh" 30 | mkdir -p $sshpath 31 | echo $PUBKEY > ${sshpath}/authorized_keys 32 | chmod -R 0600 $sshpath 33 | } 34 | 35 | start_ttrexec_daemon() { 36 | cmd_name="ttrexec-daemon" 37 | cmd_path="/usr/local/bin/${cmd_name}" 38 | $cmd_path -a 0.0.0.0 -p 22000 2>/tmp/${cmd_name}_stderr.log >/tmp/${cmd_name}_stdout.log & 39 | } 40 | 41 | set_ip 42 | ssh_login 43 | set_pub_key 44 | 45 | start_ttrexec_daemon 46 | -------------------------------------------------------------------------------- /tools/images/freebsd_vm/rc.local_hostfwd: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # use QEMU's `hostfwd` to set network. 5 | # 6 | 7 | ssh_login() { 8 | user="root" 9 | # passwd -d $user 10 | sed -ri."" 's/^ *#* *(PermitEmptyPasswords).*/\1 yes/' /etc/ssh/sshd_config 11 | sed -ri."" 's/^ *#* *(PermitRootLogin).*/\1 yes/' /etc/ssh/sshd_config 12 | sed -ri."" 's/^ *#* *(UsePAM).*/\1 yes/' /etc/ssh/sshd_config 13 | sed -ri."" 's/^ *#* *(UseDNS).*/\1 no/' /etc/ssh/sshd_config 14 | 15 | service sshd restart 16 | } 17 | 18 | set_pub_key() { 19 | PUBKEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDk/CSaXip7wwa2xsDaZm6eKmyET91fI28ufdVMZvsym7s8y0QVxdO16t9IlbH9ognp/zLETJSl1MxoEz1R2I9rE+R0eraeBT/LIsF5nIyMBZ/Zk5SRNrociW+9w4AxWZHcugGznVFIxbeC2Ja/q4xxDWRbtPB5nv2R92eTwZZ6TovxjmSGXUcBVRVFgvE0k7KgdRCGK9q0JOUZMT2d3fs8FkOJPeSznnawaUiLlLYFsY1R4F/+0r+zlbDgKrBmnjv1GYKohBcFMsw10m08cU3UjJym67R0I2NKMAFhWBZAh4IGMIfg7w3Ua2K4nHQwlcnOWuZ7jzll/nYRmrS9T3LF root@718f9c9841a8" 20 | sshpath="/root/.ssh" 21 | mkdir -p $sshpath 22 | echo $PUBKEY > ${sshpath}/authorized_keys 23 | chmod -R 0600 $sshpath 24 | } 25 | 26 | start_ttrexec_daemon() { 27 | cmd_name="ttrexec-daemon" 28 | cmd_path="/usr/local/bin/${cmd_name}" 29 | $cmd_path -a 0.0.0.0 -p 22000 2>/tmp/${cmd_name}_stderr.log >/tmp/${cmd_name}_stdout.log & 30 | } 31 | 32 | ssh_login 33 | set_pub_key 34 | start_ttrexec_daemon 35 | -------------------------------------------------------------------------------- /tools/images/id_rsa: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEA5Pwkml4qe8MGtsbA2mZunipshE/dXyNvLn3VTGb7Mpu7PMtE 3 | FcXTterfSJWx/aIJ6f8yxEyUpdTMaBM9UdiPaxPkdHq2ngU/yyLBeZyMjAWf2ZOU 4 | kTa6HIlvvcOAMVmR3LoBs51RSMW3gtiWv6uMcQ1kW7TweZ79kfdnk8GWek6L8Y5k 5 | hl1HAVUVRYLxNJOyoHUQhivatCTlGTE9nd37PBZDiT3ks552sGlIi5S2BbGNUeBf 6 | /tK/s5Ww4CqwZp479RmCqIQXBTLMNdJtPHFN1Iycpuu0dCNjSjABYVgWQIeCBjCH 7 | 4O8N1GtiuJx0MJXJzlrme485Zf52EZq0vU9yxQIDAQABAoIBAQDT/mPczoVCY0Ja 8 | ARQWnnKW1+vzawUlyWZrgm/w9f5l0iu8kusLxUTFzRa+2mgYyuWmz28usT+Fb8d2 9 | KynAFmBg39/HvrxG+9EdvaWlczvjfmmJQ8ptzl7rgIoFA3QxPB2AXmyo32KbnwDQ 10 | kLiv5qB1IdLh3FguIPXdJ1GrR7SKsWDVooOEWJvJfbNxt/ak5gxow9LIN/2Wv1PM 11 | xh/+C7Evx1auOkVuC9+2F7rJbb5M/B6B+YLqjOMtjIe1ME9VUqkSuf3bsYHpMVv5 12 | LLVylEpfdtYG6fi1t9SA+P6e4DyH13zQLSN3QDjUdeeyIbXqe1FruiYpQtLo3A4r 13 | 6+rKaFjZAoGBAPurbsDvtAbxfR0YFKNov3nhE4QE71rr+B1VLGinnHX3I60tpY07 14 | jhf+v1ntmCsXY64W8HFdn0rOpUXPhwsdyXi9lVBA9JFAICPyVu/o6mJXfJ8a1BMi 15 | E96veImNDwuK/0Ae5VK1jcdouZgoG9co6AWmgueHx3D92fFs/Oirh+1nAoGBAOjs 16 | yZpt1+gH1m3/skzUoKGgc+vk13LM42EtjTQy1mnWfks7Nl8lkDKrlyTqDJxeycTP 17 | Cr7+bkqQkmQFHtzuBiSq7yt4KXOdlWYrsx9KSbYMXFPdugxfiOFQsidNAOLpobjh 18 | QIJWmFYSQFWdVNv/YOQ8DCNbznt5VhlBF4UkY9bzAoGAd+862LdjE+wBs9vF+hnx 19 | JiQdKM0xRCMwGsp8X2OBLLaaSe1299dp4AWHK1QPMHn1BwHnlB8JypywJpS/xoxr 20 | dx7iCVzrME1fA8J5q9tT14nZ2fjvGC8lSPpWdzbB9L5I5kXTA5eB+YXu7JQwsFjO 21 | OeMgfzY11aMkOem2nSshnAECgYBs1WcFz1lYw4C/+P+4wokjvDMt/8ljjLSZzYzy 22 | 3OYuodh1En+/SW/tHRwMVYf68JdabFtbDss97/tW3MWk+VrJe00xhH3p1bHfAYA6 23 | mJ2EgJYLYcjyyxjMHsZ/co19eSjll+pqfEfFv9Vrq43hFZySSDRruRPrwbAnMLDq 24 | tywnXQKBgG7MosZPAKKGUUpBqw6ZbkHy/K+qSUtKvyUVIXVBl8hQGeeKltcOVesG 25 | UIkpfJfmyFlw52THX7jwc0RmHfIjjeit4vRW2Id2wUNRlgFW5OQu5PCncRhKpdJM 26 | wUVL9Lu+n7huI+GtsmATdhUYlzE/K8nNmhJuw3abrGPPUgrtx5OE 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /tools/images/id_rsa.pub: -------------------------------------------------------------------------------- 1 | issh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDk/CSaXip7wwa2xsDaZm6eKmyET91fI28ufdVMZvsym7s8y0QVxdO16t9IlbH9ognp/zLETJSl1MxoEz1R2I9rE+R0eraeBT/LIsF5nIyMBZ/Zk5SRNrociW+9w4AxWZHcugGznVFIxbeC2Ja/q4xxDWRbtPB5nv2R92eTwZZ6TovxjmSGXUcBVRVFgvE0k7KgdRCGK9q0JOUZMT2d3fs8FkOJPeSznnawaUiLlLYFsY1R4F/+0r+zlbDgKrBmnjv1GYKohBcFMsw10m08cU3UjJym67R0I2NKMAFhWBZAh4IGMIfg7w3Ua2K4nHQwlcnOWuZ7jzll/nYRmrS9T3LF root@718f9c9841a8 2 | -------------------------------------------------------------------------------- /tools/images/linux_vm/README.md: -------------------------------------------------------------------------------- 1 | # notes 2 | 3 | ttrexec-daemon MUST run in frondend on CentOS8.x platform. 4 | -------------------------------------------------------------------------------- /tools/images/linux_vm/rc.local: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | clean_env() { 4 | systemctl stop NetworkManager 2>/dev/null 5 | systemctl stop firewalld 2>/dev/null 6 | iptables -F 2>/dev/null 7 | nft flush ruleset 2>/dev/null 8 | # setenforce 0 2>/dev/null 9 | } 10 | 11 | set_ip() { 12 | if=$(ip link | grep -v 'lo:' | grep -oE '^ *[0-9]+:[[:blank:]]+[0-9a-zA-Z]+' | head -1 | grep -oE '[^[:blank:]]+$') 13 | mac=$(ip link | grep -A 1 "${if}:" | grep -oE 'ether [^[:blank:]]+' | grep -oE '[^[:blank:]]+$') 14 | field1=$(echo $(( 0x$(echo $mac | grep -Eo '[0-9a-zA-Z]{2}:[0-9a-zA-Z]{2}$' | grep -Eo '^[0-9a-zA-Z]{2}') ))) 15 | field2=$(echo $(( 0x$(echo $mac | grep -Eo '[0-9a-zA-Z]{2}$') ))) 16 | addr="10.10.${field1}.${field2}/8" 17 | 18 | ip addr flush dev $if 19 | ip addr add $addr dev $if 20 | ip link set $if up 21 | ip route replace default via 10.0.0.1 dev $if 22 | 23 | ip link set lo up 24 | } 25 | 26 | ssh_login() { 27 | user="root" 28 | # passwd -d $user 29 | sed -ri."" 's/^ *#* *(PermitEmptyPasswords).*/\1 yes/' /etc/ssh/sshd_config 30 | sed -ri."" 's/^ *#* *(PermitRootLogin).*/\1 yes/' /etc/ssh/sshd_config 31 | sed -ri."" 's/^ *#* *(UseDNS).*/\1 no/' /etc/ssh/sshd_config 32 | systemctl restart sshd \ 33 | || service sshd restart \ 34 | || rc-service sshd restart \ 35 | || systemctl restart ssh \ 36 | || service ssh restart \ 37 | || rc-service sshd restart \ 38 | || rc-service ssh restart 39 | } 40 | 41 | set_pub_key() { 42 | PUBKEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDk/CSaXip7wwa2xsDaZm6eKmyET91fI28ufdVMZvsym7s8y0QVxdO16t9IlbH9ognp/zLETJSl1MxoEz1R2I9rE+R0eraeBT/LIsF5nIyMBZ/Zk5SRNrociW+9w4AxWZHcugGznVFIxbeC2Ja/q4xxDWRbtPB5nv2R92eTwZZ6TovxjmSGXUcBVRVFgvE0k7KgdRCGK9q0JOUZMT2d3fs8FkOJPeSznnawaUiLlLYFsY1R4F/+0r+zlbDgKrBmnjv1GYKohBcFMsw10m08cU3UjJym67R0I2NKMAFhWBZAh4IGMIfg7w3Ua2K4nHQwlcnOWuZ7jzll/nYRmrS9T3LF root@718f9c9841a8" 43 | sshpath="/root/.ssh" 44 | mkdir -p $sshpath 45 | echo $PUBKEY > ${sshpath}/authorized_keys 46 | chmod -R 0600 $sshpath 47 | } 48 | 49 | start_ttrexec_daemon() { 50 | cmd="ttrexec-daemon" 51 | cmd_path="/usr/local/bin/${cmd}" 52 | ($cmd_path -a 0.0.0.0 -p 22000 2>/tmp/${cmd}_stderr.log >/tmp/${cmd}_stdout.log) & 53 | } 54 | 55 | start_hawk_agent() { 56 | cmd="hawk-agent" 57 | (bash -x /usr/local/bin/${cmd}/tool/start.sh 2>/tmp/${cmd}_stderr.log >/tmp/${cmd}_stdout.log) & 58 | } 59 | 60 | clean_env 61 | set_ip 62 | ssh_login 63 | set_pub_key 64 | start_ttrexec_daemon 65 | start_hawk_agent 66 | -------------------------------------------------------------------------------- /tools/images/linux_vm/rc.local_hostfwd: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # use QEMU's `hostfwd` to set network. 5 | # 6 | 7 | clean_env() { 8 | systemctl stop NetworkManager 2>/dev/null 9 | systemctl stop firewalld 2>/dev/null 10 | iptables -F 2>/dev/null 11 | nft flush ruleset 2>/dev/null 12 | # setenforce 0 2>/dev/null 13 | } 14 | 15 | ssh_login() { 16 | user="root" 17 | # passwd -d $user 18 | sed -ri."" 's/^ *#* *(PermitEmptyPasswords).*/\1 yes/' /etc/ssh/sshd_config 19 | sed -ri."" 's/^ *#* *(PermitRootLogin).*/\1 yes/' /etc/ssh/sshd_config 20 | sed -ri."" 's/^ *#* *(UsePAM).*/\1 yes/' /etc/ssh/sshd_config 21 | sed -ri."" 's/^ *#* *(UseDNS).*/\1 no/' /etc/ssh/sshd_config 22 | systemctl restart sshd \ 23 | || service sshd restart \ 24 | || systemctl restart ssh \ 25 | || service ssh restart \ 26 | || rc-service sshd restart \ 27 | || rc-service ssh restart 28 | } 29 | 30 | set_pub_key() { 31 | PUBKEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDk/CSaXip7wwa2xsDaZm6eKmyET91fI28ufdVMZvsym7s8y0QVxdO16t9IlbH9ognp/zLETJSl1MxoEz1R2I9rE+R0eraeBT/LIsF5nIyMBZ/Zk5SRNrociW+9w4AxWZHcugGznVFIxbeC2Ja/q4xxDWRbtPB5nv2R92eTwZZ6TovxjmSGXUcBVRVFgvE0k7KgdRCGK9q0JOUZMT2d3fs8FkOJPeSznnawaUiLlLYFsY1R4F/+0r+zlbDgKrBmnjv1GYKohBcFMsw10m08cU3UjJym67R0I2NKMAFhWBZAh4IGMIfg7w3Ua2K4nHQwlcnOWuZ7jzll/nYRmrS9T3LF root@718f9c9841a8" 32 | sshpath="/root/.ssh" 33 | mkdir -p $sshpath 34 | echo $PUBKEY > ${sshpath}/authorized_keys 35 | chmod -R 0600 $sshpath 36 | } 37 | 38 | start_ttrexec_daemon() { 39 | cmd="ttrexec-daemon" 40 | cmd_path="/usr/local/bin/${cmd}" 41 | $cmd_path -a 0.0.0.0 -p 22000 2>/tmp/${cmd}_stderr.log >/tmp/${cmd}_stdout.log & 42 | } 43 | 44 | start_hawk_agent() { 45 | cmd="hawk-agent" 46 | (bash -x /usr/local/bin/${cmd}/tool/start.sh 2>/tmp/${cmd}_stderr.log >/tmp/${cmd}_stdout.log) & 47 | } 48 | 49 | clean_env 50 | ssh_login 51 | set_pub_key 52 | start_ttrexec_daemon 53 | start_hawk_agent 54 | -------------------------------------------------------------------------------- /tools/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ################################################# 4 | #### Ensure we are in the right path. ########### 5 | ################################################# 6 | if [[ 0 -eq $(echo $0 | grep -c '^/') ]]; then 7 | # relative path 8 | EXEC_PATH=$(dirname "`pwd`/$0") 9 | else 10 | # absolute path 11 | EXEC_PATH=$(dirname "$0") 12 | fi 13 | 14 | EXEC_PATH=$(echo ${EXEC_PATH} | sed 's@/\./@/@g' | sed 's@/\.*$@@') 15 | cd $EXEC_PATH || exit 1 16 | ################################################# 17 | 18 | if [[ 0 -ne `id -u` ]]; then 19 | echo -e "\x1b[31;01mSorry, only root can do this operation!\x1b[00m" 20 | fi 21 | 22 | cp * /usr/sbin/ 23 | -------------------------------------------------------------------------------- /tools/qemu_update_image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | zvol_prefix="/dev/zvol/zroot/tt-nftables/" 4 | tap_path="/tmp/.xxsssxxxyyyyyyxx__xxx" 5 | 6 | echo '#!/bin/sh 7 | ip link set $1 up; sleep 1; ip link set $1 master ttcore-bridge' > $tap_path 8 | chmod +x $tap_path 9 | 10 | let i=0 11 | for img in Alpine-3.12 CentOS-6.10 CentOS-7.0 CentOS-7.1 CentOS-7.2 CentOS-7.3 \ 12 | CentOS-7.4 CentOS-7.5 CentOS-7.6 CentOS-7.7 CentOS-7.8 CentOS-8.2 \ 13 | Ubuntu-1410 Ubuntu-1504 Ubuntu-1510 Ubuntu-1604 Ubuntu-1610 Ubuntu-1704 Ubuntu-1710 \ 14 | Ubuntu-1804 Ubuntu-1810 Ubuntu-1904 Ubuntu-1910 Ubuntu-2004 slow-Ubuntu-1404; 15 | do 16 | let i+=1 17 | id=$(printf "%02x" ${i}) 18 | 19 | qemu-system-x86_64 -enable-kvm -cpu host -smp 2 -m 800 \ 20 | -netdev tap,ifname=TMP_${i}-if,script=${tap_path},downscript=no,id=TMP_${i}-NIC \ 21 | -device virtio-net-pci,mac=00:be:fa:76:09:${id},netdev=TMP_${i}-NIC \ 22 | -drive file=${zvol_prefix}${img},if=none,format=raw,cache=none,id=TMP_${i}-DISK \ 23 | -device virtio-blk-pci,drive=TMP_${i}-DISK -boot order=cd -daemonize -vnc :$i 24 | done 25 | 26 | sleep 120 27 | 28 | data1="/data/ftp_home/tt_releases/linux/ttrexec-daemon" 29 | data2="/data/ftp_home/tt_releases/linux/ttrexec-daemon" 30 | 31 | for ((;i>0;i--)); do 32 | ssh -i ~/.ssh/tt_rsa root@10.10.9.${i} pkill -9 ttrexec 33 | 34 | scp -i ~/.ssh/tt_rsa $data1 root@10.10.9.${i}:/usr/local/bin/ 35 | scp -i ~/.ssh/tt_rsa $data2 root@10.10.9.${i}:/usr/local/bin/ 36 | 37 | ssh -i ~/.ssh/tt_rsa root@10.10.9.${i} poweroff 38 | done 39 | -------------------------------------------------------------------------------- /tools/trash/baseimage_update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | qemu-system-x86_64 \ 4 | -enable-kvm \ 5 | -cpu host -smp 4,sockets=4,cores=1,threads=1 \ 6 | -m 8192 \ 7 | -net nic -net user,hostfwd=tcp::2222-:22 \ 8 | -drive file=$1,if=none,format=raw,cache=none,id=DISK_111 \ 9 | -device virtio-blk-pci,drive=DISK_111 \ 10 | -vnc :9 \ 11 | -daemonize \ 12 | -boot order=cd 13 | 14 | # -drive file=$2,readonly=on,media=cdrom \ 15 | -------------------------------------------------------------------------------- /tools/trash/muslenv.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | name="muslenv" 4 | 5 | if [[ 0 == $(docker ps --filter="name=^${name}$" | grep -c "${name}") ]]; then 6 | docker run -v $(pwd):/volume --name $name --privileged --rm -dt clux/muslrust 7 | fi 8 | -------------------------------------------------------------------------------- /tools/trash/net.start.gentoo: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | dhcpcd 4 | 5 | if="enp5s0" 6 | netns="ttserver" 7 | 8 | (ip netns pids $etns | xargs kill -9) 2>/dev/null 9 | (ip netns del $netns && sleep 1) 2>/dev/null 10 | 11 | ip netns add $netns || exit 1 12 | ip netns exec $netns ip link set lo up || exit 1 13 | 14 | ip link set dev $if netns $netns || exit 1 15 | ip netns exec $netns ip link set $if up || exit 1 16 | ip netns exec $netns ip addr add 192.168.3.81/24 dev $if || exit 1 17 | ip netns exec $netns ip route replace default via 192.168.3.1 dev $if || exit 1 18 | ip netns exec $netns /usr/local/bin/ttserver \ 19 | --serv-addr=192.168.3.81 \ 20 | --serv-port=9527 \ 21 | --image-path=/dev/zvol/zroot/tt \ 22 | --cpu-total=24 \ 23 | --mem-total=$[48 * 1024] \ 24 | --disk-total=81920000 || exit 1 25 | -------------------------------------------------------------------------------- /tools/trash/start.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ################################################# 4 | #### Ensure we are in the right path. ########### 5 | ################################################# 6 | if [[ 0 -eq `echo $0 | grep -c '^/'` ]]; then 7 | # relative path 8 | EXEC_PATH=$(dirname "`pwd`/$0") 9 | else 10 | # absolute path 11 | EXEC_PATH=$(dirname "$0") 12 | fi 13 | 14 | EXEC_PATH=$(echo ${EXEC_PATH} | sed 's@/\./@/@g' | sed 's@/\.*$@@') 15 | cd $EXEC_PATH || exit 1 16 | ################################################# 17 | 18 | export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' 19 | export PATH=/usr/local/bin:$PATH 20 | 21 | ip=$1 22 | port=$2 23 | imgpath=$3 24 | 25 | if [[ "" == $imgpath || "" == $ip ]]; then 26 | syskind=$(uname -s) 27 | if [[ "Linux" == $syskind ]]; then 28 | ip=$(ip addr | grep -Eo '192\.168\.[0-9]{1,3}\.[0-9]{1,3}' | head -1) 29 | imgpath="/data/images" 30 | elif [[ "FreeBSD" == $syskind ]]; then 31 | ip=$(ifconfig | grep -Eo '192\.168\.[0-9]{1,3}\.[0-9]{1,3}' | head -1) 32 | imgpath="/dev/zvol/zroot/bhyve" 33 | fi 34 | fi 35 | 36 | if [[ "" == $port ]]; then 37 | port=9527 38 | fi 39 | 40 | nohup ttserver \ 41 | --serv-addr=${ip} \ 42 | --serv-port=${port} \ 43 | --image-path=${imgpath} \ 44 | --cpu-total=144 \ 45 | --mem-total=54244 \ 46 | --disk-total=8192000 & 47 | -------------------------------------------------------------------------------- /tools/trash/vfio_bind.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ################################### 4 | # TESTing only, can NOT work now! # 5 | ################################### 6 | 7 | modprobe vfio-pci 8 | 9 | if_name="enp5s0" 10 | if_id="0000:05:00.0" 11 | 12 | vendor=$(cat /sys/bus/pci/devices/${if_id}/vendor) 13 | device=$(cat /sys/bus/pci/devices/${if_id}/device) 14 | 15 | echo "${if_id}" > /sys/bus/pci/devices/${if_id}/driver/unbind 16 | echo "vfio-pci" > /sys/bus/pci/devices/${if_id}/driver_override 17 | echo "${vendor} ${device}" > /sys/bus/pci/drivers/vfio-pci/new_id 18 | 19 | echo "${if_id}" > /sys/bus/pci/drivers/vfio-pci/bind 20 | --------------------------------------------------------------------------------