├── .github └── workflows │ ├── build.yml │ ├── golangci-lint.yml │ └── test.yml ├── .gitignore ├── .golangci.yml ├── LICENSE ├── Makefile ├── README.md ├── addlicense.sh ├── cluster.yml.sample ├── cmd └── m3fs │ ├── architecture.go │ ├── architecture_test.go │ ├── artifact.go │ ├── base_test.go │ ├── cluster.go │ ├── config.go │ ├── diagram_renderer.go │ ├── main.go │ ├── os.go │ └── template.go ├── dockerfile └── grafana.dockerfile ├── go.mod ├── go.sum ├── pkg ├── 3fs_client │ ├── steps.go │ ├── steps_test.go │ ├── tasks.go │ └── templates │ │ ├── hf3fs_fuse_main.toml.tmpl │ │ └── hf3fs_fuse_main_launcher.toml.tmpl ├── artifact │ ├── steps.go │ ├── steps_test.go │ └── tasks.go ├── clickhouse │ ├── steps.go │ ├── steps_test.go │ ├── tasks.go │ └── templates │ │ ├── config.tmpl │ │ └── sql.tmpl ├── common │ ├── copy.go │ ├── pprint.go │ ├── string.go │ ├── string_camelcase.go │ ├── utils.go │ ├── version.go │ └── worker.go ├── config │ ├── config.go │ ├── config_test.go │ ├── image.go │ ├── image_test.go │ └── types.go ├── errors │ ├── base_test.go │ ├── error.go │ ├── error_test.go │ └── path.go ├── external │ ├── base_mock_test.go │ ├── base_test.go │ ├── command.go │ ├── disk.go │ ├── docker.go │ ├── docker_test.go │ ├── fs.go │ ├── local_runner.go │ ├── manager.go │ ├── network.go │ └── runner.go ├── fdb │ ├── steps.go │ ├── steps_test.go │ └── tasks.go ├── grafana │ ├── steps.go │ ├── steps_test.go │ ├── tasks.go │ └── templates │ │ ├── dashboard.json.tmpl │ │ ├── dashboard_provision.yaml.tmpl │ │ └── datasource_provision.yaml.tmpl ├── log │ └── logger.go ├── meta │ ├── steps.go │ ├── tasks.go │ └── templates │ │ ├── meta_main.toml.tmpl │ │ ├── meta_main_app.toml.tmpl │ │ └── meta_main_launcher.toml.tmpl ├── mgmtd │ ├── steps.go │ ├── steps_test.go │ ├── tasks.go │ └── templates │ │ ├── admin_cli.sh.tmpl │ │ ├── admin_cli.toml.tmpl │ │ ├── mgmtd_main.toml.tmpl │ │ ├── mgmtd_main_app.toml.tmpl │ │ └── mgmtd_main_launcher.toml.tmpl ├── monitor │ ├── steps.go │ ├── steps_test.go │ ├── tasks.go │ └── templates │ │ └── monitor_collector_main.tmpl ├── network │ ├── speed.go │ ├── steps.go │ ├── steps_test.go │ └── tasks.go ├── storage │ ├── tasks.go │ └── templates │ │ ├── disk_tool.sh.tmpl │ │ ├── storage_main.toml.tmpl │ │ ├── storage_main_app.toml.tmpl │ │ └── storage_main_launcher.toml.tmpl ├── task │ ├── base_test.go │ ├── runner.go │ ├── runner_test.go │ ├── steps │ │ ├── 3fs_steps.go │ │ ├── 3fs_steps_test.go │ │ ├── local_steps.go │ │ └── local_steps_test.go │ └── task.go └── utils │ ├── net.go │ └── set.go └── tests ├── base └── suite.go ├── external ├── docker.go ├── fs.go └── runner.go └── task └── step.go /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | on: 3 | push: 4 | branches: 5 | - main 6 | - master 7 | - dev 8 | 9 | permissions: 10 | contents: read 11 | # Optional: allow read access to pull request. Use with `only-new-issues` option. 12 | # pull-requests: read 13 | 14 | jobs: 15 | build: 16 | name: Run on Ubuntu 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v4 20 | - uses: actions/setup-go@v5 21 | with: 22 | go-version-file: go.mod 23 | - name: Build Binary 24 | run: | 25 | make build 26 | -------------------------------------------------------------------------------- /.github/workflows/golangci-lint.yml: -------------------------------------------------------------------------------- 1 | name: golangci-lint 2 | on: 3 | push: 4 | branches: 5 | - main 6 | - master 7 | - dev 8 | pull_request: 9 | types: [opened, edited, synchronize, reopened] 10 | 11 | permissions: 12 | contents: read 13 | # Optional: allow read access to pull request. Use with `only-new-issues` option. 14 | # pull-requests: read 15 | 16 | jobs: 17 | golangci: 18 | name: lint 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@v4 22 | - uses: actions/setup-go@v5 23 | with: 24 | go-version: stable 25 | - name: golangci-lint 26 | uses: golangci/golangci-lint-action@v6 27 | with: 28 | version: v1.64 29 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | on: 3 | push: 4 | branches: 5 | - main 6 | - master 7 | - dev 8 | pull_request: 9 | types: [opened, edited, synchronize, reopened] 10 | 11 | permissions: 12 | contents: read 13 | # Optional: allow read access to pull request. Use with `only-new-issues` option. 14 | # pull-requests: read 15 | 16 | jobs: 17 | test: 18 | name: Run on Ubuntu 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@v4 22 | - uses: actions/setup-go@v5 23 | with: 24 | go-version-file: go.mod 25 | - name: Running Tests 26 | run: | 27 | make test 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # If you prefer the allow list template instead of the deny list, see community template: 2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 3 | # 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Dependency directories (remove the comment below to include it) 18 | # vendor/ 19 | 20 | # Go workspace file 21 | go.work 22 | go.work.sum 23 | 24 | # env file 25 | .env 26 | 27 | # build output 28 | bin/ 29 | 30 | .vscode/ 31 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | timeout: 5m 3 | allow-parallel-runners: true 4 | 5 | issues: 6 | # don't skip warning about doc comments 7 | # don't exclude the default set of lint 8 | exclude-use-default: false 9 | 10 | linters: 11 | disable-all: true 12 | enable: 13 | - dupl 14 | - errcheck 15 | - copyloopvar 16 | - ginkgolinter 17 | - goconst 18 | - gocyclo 19 | - gofmt 20 | - goimports 21 | - gosimple 22 | - govet 23 | - ineffassign 24 | - lll 25 | - misspell 26 | - nakedret 27 | - prealloc 28 | - revive 29 | - staticcheck 30 | - typecheck 31 | - unconvert 32 | - unparam 33 | - unused 34 | - goheader 35 | linters-settings: 36 | revive: 37 | ignore-generated-header: true 38 | severity: warning 39 | confidence: 0.8 40 | rules: 41 | - name: blank-imports 42 | - name: context-as-argument 43 | - name: context-keys-type 44 | - name: dot-imports 45 | - name: error-return 46 | - name: error-strings 47 | - name: error-naming 48 | - name: exported 49 | - name: if-return 50 | - name: increment-decrement 51 | - name: var-naming 52 | arguments: 53 | - ["IP", "URI", "URL", "API", "HTTP", "JSON", "SSH", "DNS"] 54 | - [] 55 | - name: var-declaration 56 | - name: range 57 | - name: receiver-naming 58 | - name: time-naming 59 | - name: unexported-return 60 | - name: indent-error-flow 61 | - name: errorf 62 | - name: bare-return 63 | - name: comment-spacings 64 | - name: use-any 65 | goheader: 66 | template: |- 67 | Copyright 2025 Open3FS Authors 68 | 69 | Licensed under the Apache License, Version 2.0 (the "License"); 70 | you may not use this file except in compliance with the License. 71 | You may obtain a copy of the License at 72 | 73 | http://www.apache.org/licenses/LICENSE-2.0 74 | 75 | Unless required by applicable law or agreed to in writing, software 76 | distributed under the License is distributed on an "AS IS" BASIS, 77 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 78 | See the License for the specific language governing permissions and 79 | limitations under the License. 80 | 81 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL = bash 2 | 3 | export PWD := $(shell pwd) 4 | 5 | export ARCH := $(shell uname -m) 6 | export ARCH_DIST := $(ARCH) 7 | ifeq ($(ARCH), x86_64) 8 | ARCH_DISK := amd64 9 | endif 10 | 11 | export OS_NAME = $(shell uname -s) 12 | export OS_TYPE := linux 13 | ifeq ($(OS_NAME), Darwin) 14 | OS_TYPE := darwin 15 | endif 16 | 17 | export BUILD_AT := $(shell date -u +'%Y-%m-%dT%T%Z') 18 | 19 | # Tag of the current commit, if any. If this is not "" then we are building a release 20 | export RELEASE_TAG := $(shell git tag -l --points-at HEAD| sort | head -n 1) 21 | # Last tag on this branch 22 | export LAST_TAG := $(shell git describe --tags --abbrev=0) 23 | export BUILD_VERSION := $(or $(RELEASE_TAG), $(LAST_TAG)) 24 | export TAG_BRANCH := .$(BRANCH) 25 | # If building HEAD or main then unset TAG_BRANCH 26 | ifeq ($(subst HEAD,,$(subst main,,$(BRANCH))),) 27 | TAG_BRANCH := 28 | endif 29 | 30 | # COMMIT is the commit hash 31 | export COMMIT := $(shell git log -1 --format="%H" | head -1) 32 | # COMMIT_NUMBER is the number commits since last tag. 33 | export COMMIT_NUMBER := $(shell git rev-list --count $(RELEASE_TAG)...HEAD) 34 | 35 | # Make version suffix -NNNN.CCCCCCCC (N=Commit number, C=Commit) 36 | export VERSION_SUFFIX := $(COMMIT_NUMBER).$(shell git show --no-patch --no-notes --pretty='%h' HEAD) 37 | export VERSION := $(RELEASE_TAG)-$(VERSION_SUFFIX)$(TAG_BRANCH) 38 | 39 | # Pass in GOTAGS=xyz on the make command line to set build tags 40 | ifdef GOTAGS 41 | BUILDTAGS=-tags "$(GOTAGS)" 42 | LINTTAGS=--build-tags "$(GOTAGS)" 43 | endif 44 | 45 | export BIN := $(PWD)/bin 46 | 47 | buildVersionLDFlag := -X github.com/open3fs/m3fs/pkg/common.Version=$(BUILD_VERSION) -X github.com/open3fs/m3fs/pkg/common.GitSha=$(COMMIT) \ 48 | -X github.com/open3fs/m3fs/pkg/common.BuildTime=$(BUILD_AT) 49 | 50 | .PHONY: build 51 | build: 52 | CGO_ENABLED=0 go build -ldflags "$(buildVersionLDFlag)" $(BUILDTAGS) -o $(BIN)/m3fs github.com/open3fs/m3fs/cmd/m3fs 53 | 54 | .PHONY: test 55 | test: 56 | go test -timeout 1h `go list ./...` 57 | 58 | .PHONY: validate 59 | validate: 60 | go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.6 61 | @echo "Lint code with golangci-lint" 62 | PATH=$(shell go env GOPATH)/bin:$(PATH) golangci-lint run 63 | 64 | .PHONY: lint 65 | lint: 66 | go install github.com/mgechev/revive@v1.7.0 67 | @echo "Lint code with revive" 68 | PATH=$(shell go env GOPATH)/bin:$(PATH) revive -config revive.toml --formatter default ./... 69 | 70 | .PHONY: checkfmt 71 | # Use lazy assignment until called by SET_GOFILES to fetch file list. 72 | GENERATE_GOFILES = $(shell find . -type f \( -iname "*.go" \)) 73 | SET_GOFILES = $(eval GOFILES=$(GENERATE_GOFILES)) 74 | checkfmt: 75 | @echo "Check if code are formatted by gofmt" 76 | $(SET_GOFILES) 77 | @for file in $(GOFILES); do \ 78 | diff -u <(echo -n) <(gofmt -d $$file) || exit 1; \ 79 | done 80 | -------------------------------------------------------------------------------- /addlicense.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | addlicense -c "Open3FS Authors" -l apache "$@" 4 | -------------------------------------------------------------------------------- /cmd/m3fs/architecture_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/open3fs/m3fs/pkg/config" 21 | ) 22 | 23 | func TestArchDiagramSuite(t *testing.T) { 24 | suiteRun(t, &archDiagramSuite{}) 25 | } 26 | 27 | type archDiagramSuite struct { 28 | Suite 29 | cfg *config.Config 30 | diagram *ArchDiagram 31 | } 32 | 33 | func (s *archDiagramSuite) SetupTest() { 34 | s.Suite.SetupTest() 35 | 36 | s.cfg = s.newTestConfig() 37 | diagram, err := NewArchDiagram(s.cfg, false) 38 | s.NoError(err) 39 | s.diagram = diagram 40 | } 41 | 42 | func (s *archDiagramSuite) newTestConfig() *config.Config { 43 | return &config.Config{ 44 | Name: "test-cluster", 45 | NetworkType: "RXE", 46 | Nodes: []config.Node{ 47 | {Name: "192.168.1.1", Host: "192.168.1.1"}, 48 | {Name: "192.168.1.2", Host: "192.168.1.2"}, 49 | {Name: "192.168.1.3", Host: "192.168.1.3"}, 50 | {Name: "192.168.1.4", Host: "192.168.1.4"}, 51 | }, 52 | Services: config.Services{ 53 | Mgmtd: config.Mgmtd{ 54 | Nodes: []string{"192.168.1.1"}, 55 | }, 56 | Meta: config.Meta{ 57 | Nodes: []string{"192.168.1.1", "192.168.1.2"}, 58 | }, 59 | Storage: config.Storage{ 60 | Nodes: []string{"192.168.1.2", "192.168.1.3"}, 61 | }, 62 | Client: config.Client{ 63 | Nodes: []string{"192.168.1.3", "192.168.1.4"}, 64 | HostMountpoint: "/mnt/m3fs", 65 | }, 66 | Fdb: config.Fdb{ 67 | Nodes: []string{"192.168.1.1"}, 68 | }, 69 | Clickhouse: config.Clickhouse{ 70 | Nodes: []string{"192.168.1.2"}, 71 | }, 72 | Monitor: config.Monitor{ 73 | Nodes: []string{"192.168.1.3"}, 74 | }, 75 | }, 76 | } 77 | } 78 | 79 | func (s *archDiagramSuite) TestArchDiagram() { 80 | diagram := s.diagram.Render() 81 | 82 | s.NotEmpty(diagram, "Generated diagram should not be empty") 83 | s.Contains(diagram, "Cluster: test-cluster", "Diagram should contain cluster name") 84 | 85 | // Check node sections 86 | s.Contains(diagram, "CLIENT NODES", "Diagram should have CLIENT NODES section") 87 | s.Contains(diagram, "STORAGE NODES", "Diagram should have STORAGE NODES section") 88 | 89 | // Check IP addresses are present 90 | s.Contains(diagram, "192.168.1.1", "Diagram should show 192.168.1.1") 91 | s.Contains(diagram, "192.168.1.2", "Diagram should show 192.168.1.2") 92 | s.Contains(diagram, "192.168.1.3", "Diagram should show 192.168.1.3") 93 | s.Contains(diagram, "192.168.1.4", "Diagram should show 192.168.1.4") 94 | 95 | // Check service labels are present 96 | s.Contains(diagram, "[mgmtd]", "Diagram should show mgmtd service") 97 | s.Contains(diagram, "[meta]", "Diagram should show meta service") 98 | s.Contains(diagram, "[storage]", "Diagram should show storage service") 99 | s.Contains(diagram, "[hf3fs_fuse]", "Diagram should show hf3fs_fuse service") 100 | s.Contains(diagram, "[foundationdb]", "Diagram should show foundationdb service") 101 | s.Contains(diagram, "[clickhouse]", "Diagram should show clickhouse service") 102 | s.Contains(diagram, "[monitor]", "Diagram should show monitor service") 103 | } 104 | 105 | func (s *archDiagramSuite) TestNoColorOption() { 106 | s.diagram.noColor = true 107 | 108 | diagram := s.diagram.Render() 109 | 110 | s.NotContains(diagram, "\033[", "Diagram should not contain color codes when colors are disabled") 111 | 112 | // Check if the diagram content is still complete 113 | s.Contains(diagram, "Cluster: test-cluster", "Diagram should still contain cluster name") 114 | s.Contains(diagram, "CLIENT NODES", "Diagram should still have CLIENT NODES section") 115 | s.Contains(diagram, "STORAGE NODES", "Diagram should still have STORAGE NODES section") 116 | s.Contains(diagram, "[storage]", "Diagram should still show storage service label") 117 | s.Contains(diagram, "[hf3fs_fuse]", "Diagram should still show hf3fs_fuse service label") 118 | } 119 | -------------------------------------------------------------------------------- /cmd/m3fs/artifact.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "os" 19 | 20 | "github.com/urfave/cli/v2" 21 | 22 | "github.com/open3fs/m3fs/pkg/artifact" 23 | "github.com/open3fs/m3fs/pkg/errors" 24 | "github.com/open3fs/m3fs/pkg/task" 25 | ) 26 | 27 | var artifactCmd = &cli.Command{ 28 | Name: "artifact", 29 | Aliases: []string{"a"}, 30 | Usage: "Manage 3fs artifact", 31 | Subcommands: []*cli.Command{ 32 | { 33 | Name: "export", 34 | Aliases: []string{"download", "d", "e"}, 35 | Usage: "Export a 3fs offline artifact", 36 | Action: exportArtifact, 37 | Flags: []cli.Flag{ 38 | &cli.StringFlag{ 39 | Name: "config", 40 | Aliases: []string{"c"}, 41 | Usage: "Path to the cluster configuration file", 42 | Destination: &configFilePath, 43 | Required: true, 44 | }, 45 | &cli.StringFlag{ 46 | Name: "tmp-dir", 47 | Aliases: []string{"t"}, 48 | Usage: "Temporary dir used to save downloaded packages (default: \"/tmp/3fs\")", 49 | Destination: &tmpDir, 50 | Required: false, 51 | }, 52 | &cli.BoolFlag{ 53 | Name: "gzip", 54 | Aliases: []string{"z"}, 55 | Usage: "Archive the artifact through gzip", 56 | Destination: &artifactGzip, 57 | Required: false, 58 | }, 59 | &cli.StringFlag{ 60 | Name: "output", 61 | Aliases: []string{"o"}, 62 | Usage: "Output path", 63 | Destination: &outputPath, 64 | Required: true, 65 | }, 66 | }, 67 | }, 68 | }, 69 | } 70 | 71 | func exportArtifact(ctx *cli.Context) error { 72 | cfg, err := loadClusterConfig() 73 | if err != nil { 74 | return errors.Trace(err) 75 | } 76 | if tmpDir == "" { 77 | tmpDir = "/tmp/3fs" 78 | } 79 | 80 | if _, err := os.Stat(outputPath); err == nil { 81 | return errors.Errorf("output path %s already exists", outputPath) 82 | } else if !os.IsNotExist(err) { 83 | return errors.Trace(err) 84 | } 85 | 86 | runner, err := task.NewRunner(cfg, new(artifact.ExportArtifactTask)) 87 | if err != nil { 88 | return errors.Trace(err) 89 | } 90 | runner.Init() 91 | if err = runner.Store(task.RuntimeArtifactTmpDirKey, tmpDir); err != nil { 92 | return errors.Trace(err) 93 | } 94 | if err = runner.Store(task.RuntimeArtifactPathKey, outputPath); err != nil { 95 | return errors.Trace(err) 96 | } 97 | if err = runner.Store(task.RuntimeArtifactGzipKey, artifactGzip); err != nil { 98 | return errors.Trace(err) 99 | } 100 | if err = runner.Run(ctx.Context); err != nil { 101 | return errors.Annotate(err, "import artifact") 102 | } 103 | 104 | return nil 105 | } 106 | -------------------------------------------------------------------------------- /cmd/m3fs/base_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "github.com/stretchr/testify/suite" 19 | 20 | tbase "github.com/open3fs/m3fs/tests/base" 21 | ) 22 | 23 | var suiteRun = suite.Run 24 | 25 | type Suite struct { 26 | tbase.Suite 27 | } 28 | 29 | func (s *Suite) SetupTest() { 30 | s.Suite.SetupTest() 31 | } 32 | -------------------------------------------------------------------------------- /cmd/m3fs/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "os" 19 | "text/template" 20 | 21 | "github.com/urfave/cli/v2" 22 | 23 | "github.com/open3fs/m3fs/pkg/errors" 24 | ) 25 | 26 | var ( 27 | clusterName string 28 | sampleConfigPath string 29 | ) 30 | 31 | var configCmd = &cli.Command{ 32 | Name: "config", 33 | Aliases: []string{"cfg"}, 34 | Usage: "Manage 3fs config", 35 | Subcommands: []*cli.Command{ 36 | { 37 | Name: "create", 38 | Usage: "Create a sample 3fs config", 39 | Action: createSampleConfig, 40 | Flags: []cli.Flag{ 41 | &cli.StringFlag{ 42 | Name: "name", 43 | Aliases: []string{"n"}, 44 | Usage: "3FS cluster name (default: \"open3fs\")", 45 | Value: "open3fs", 46 | Destination: &clusterName, 47 | }, 48 | &cli.StringFlag{ 49 | Name: "registry", 50 | Aliases: []string{"r"}, 51 | Usage: "Image registry (default is empty)", 52 | Destination: ®istry, 53 | }, 54 | &cli.StringFlag{ 55 | Name: "file", 56 | Aliases: []string{"f"}, 57 | Usage: "Specify a configuration file path (default: \"cluster.yml\")", 58 | Destination: &sampleConfigPath, 59 | Value: "cluster.yml", 60 | }, 61 | }, 62 | }, 63 | }, 64 | } 65 | 66 | var sampleConfigTemplate = `name: "{{.name}}" 67 | workDir: "/opt/3fs" 68 | # networkType configure the network type of the cluster, can be one of the following: 69 | # - IB: use InfiniBand network protocol 70 | # - RDMA: use RDMA network protocol 71 | # - ERDMA: use aliyun ERDMA as RDMA network protocol 72 | # - RXE: use Linux rxe kernel module to mock RDMA network protocol 73 | networkType: "RDMA" 74 | nodes: 75 | - name: node1 76 | host: "192.168.1.1" 77 | username: "root" 78 | password: "password" 79 | - name: node2 80 | host: "192.168.1.2" 81 | username: "root" 82 | password: "password" 83 | services: 84 | client: 85 | nodes: 86 | - node1 87 | hostMountpoint: /mnt/3fs 88 | storage: 89 | nodes: 90 | - node1 91 | - node2 92 | # diskType configure the disk type of the storage node to use, can be one of the following: 93 | # - nvme: NVMe SSD 94 | # - dir: use a directory on the filesystem 95 | diskType: "nvme" 96 | mgmtd: 97 | nodes: 98 | - node1 99 | meta: 100 | nodes: 101 | - node1 102 | monitor: 103 | nodes: 104 | - node1 105 | fdb: 106 | nodes: 107 | - node1 108 | clickhouse: 109 | nodes: 110 | - node1 111 | # Database name for Clickhouse 112 | db: "3fs" 113 | # User for Clickhouse authentication 114 | user: "default" 115 | # Password for Clickhouse authentication 116 | password: "password" 117 | # TCP port for Clickhouse 118 | tcpPort: 8999 119 | grafana: 120 | nodes: 121 | - node1 122 | # TCP port for Grafana 123 | port: 3000 124 | images: 125 | registry: "{{ .registry }}" 126 | 3fs: 127 | # If you want to run on environment not support avx512, add -avx2 to the end of the image tag, e.g. 20250410-avx2. 128 | repo: "open3fs/3fs" 129 | tag: "20250410" 130 | fdb: 131 | repo: "open3fs/foundationdb" 132 | tag: "7.3.63" 133 | clickhouse: 134 | repo: "open3fs/clickhouse" 135 | tag: "25.1-jammy" 136 | grafana: 137 | repo: "open3fs/grafana" 138 | tag: "12.0.0" 139 | ` 140 | 141 | func createSampleConfig(ctx *cli.Context) error { 142 | tmpl, err := template.New("sampleConfig").Parse(sampleConfigTemplate) 143 | if err != nil { 144 | return errors.Annotate(err, "parse sample config template") 145 | } 146 | if clusterName == "" { 147 | return errors.New("cluster name is required") 148 | } 149 | if sampleConfigPath == "" { 150 | sampleConfigPath = "cluster.yml" 151 | } 152 | 153 | file, err := os.OpenFile(sampleConfigPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644) 154 | if err != nil { 155 | return errors.Annotate(err, "create sample config file") 156 | } 157 | err = tmpl.Execute(file, map[string]string{ 158 | "name": clusterName, 159 | "registry": registry, 160 | }) 161 | if err != nil { 162 | return errors.Annotate(err, "write sample config file") 163 | } 164 | 165 | return nil 166 | } 167 | -------------------------------------------------------------------------------- /cmd/m3fs/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "fmt" 19 | "log" 20 | "os" 21 | "runtime" 22 | 23 | "github.com/sirupsen/logrus" 24 | "github.com/urfave/cli/v2" 25 | 26 | "github.com/open3fs/m3fs/pkg/common" 27 | "github.com/open3fs/m3fs/pkg/errors" 28 | mlog "github.com/open3fs/m3fs/pkg/log" 29 | ) 30 | 31 | var ( 32 | debug bool 33 | configFilePath string 34 | artifactPath string 35 | artifactGzip bool 36 | outputPath string 37 | tmpDir string 38 | workDir string 39 | registry string 40 | clusterDeleteAll bool 41 | noColorOutput bool 42 | ) 43 | 44 | func main() { 45 | app := &cli.App{ 46 | Name: "m3fs", 47 | Usage: "3FS Deploy Tool", 48 | Before: func(ctx *cli.Context) error { 49 | level := logrus.InfoLevel 50 | if debug { 51 | level = logrus.DebugLevel 52 | } 53 | mlog.InitLogger(level) 54 | return nil 55 | }, 56 | Commands: []*cli.Command{ 57 | artifactCmd, 58 | clusterCmd, 59 | configCmd, 60 | osCmd, 61 | tmplCmd, 62 | }, 63 | Action: func(ctx *cli.Context) error { 64 | return cli.ShowAppHelp(ctx) 65 | }, 66 | ExitErrHandler: func(cCtx *cli.Context, err error) { 67 | if err != nil { 68 | logrus.Debugf("Command failed stacktrace: %s", errors.StackTrace(err)) 69 | } 70 | cli.HandleExitCoder(err) 71 | }, 72 | Flags: []cli.Flag{ 73 | &cli.BoolFlag{ 74 | Name: "debug", 75 | Usage: "Enable debug mode", 76 | Destination: &debug, 77 | }, 78 | }, 79 | Version: fmt.Sprintf(`%s 80 | Git SHA: %s 81 | Build At: %s 82 | Go Version: %s 83 | Go OS/Arch: %s/%s`, 84 | common.Version, 85 | common.GitSha[:7], 86 | common.BuildTime, 87 | runtime.Version(), 88 | runtime.GOOS, 89 | runtime.GOARCH), 90 | } 91 | 92 | if err := app.Run(os.Args); err != nil { 93 | log.Fatal(err) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /cmd/m3fs/os.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import "github.com/urfave/cli/v2" 18 | 19 | var osCmd = &cli.Command{ 20 | Name: "os", 21 | Usage: "Manage os environment", 22 | Subcommands: []*cli.Command{ 23 | { 24 | Name: "init", 25 | Usage: "Initialize os environment", 26 | }, 27 | }, 28 | } 29 | -------------------------------------------------------------------------------- /cmd/m3fs/template.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import "github.com/urfave/cli/v2" 18 | 19 | var tmplCmd = &cli.Command{ 20 | Name: "template", 21 | Aliases: []string{"t"}, 22 | Usage: "Service config template operate", 23 | Subcommands: []*cli.Command{ 24 | { 25 | Name: "create", 26 | Usage: "Create 3fs service config template", 27 | Flags: []cli.Flag{ 28 | &cli.StringFlag{ 29 | Name: "service", 30 | Usage: "service name", 31 | Required: true, 32 | }, 33 | }, 34 | }, 35 | }, 36 | } 37 | -------------------------------------------------------------------------------- /dockerfile/grafana.dockerfile: -------------------------------------------------------------------------------- 1 | FROM grafana/grafana:12.0.0 2 | 3 | RUN grafana cli plugins install grafana-clickhouse-datasource 4 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/open3fs/m3fs 2 | 3 | go 1.23.7 4 | 5 | require ( 6 | github.com/bitly/go-simplejson v0.5.1 7 | github.com/davecgh/go-spew v1.1.1 8 | github.com/fatih/color v1.18.0 9 | github.com/google/uuid v1.6.0 10 | github.com/pkg/sftp v1.13.7 11 | github.com/sirupsen/logrus v1.9.3 12 | github.com/stretchr/testify v1.10.0 13 | github.com/urfave/cli/v2 v2.27.6 14 | golang.org/x/crypto v0.36.0 15 | golang.org/x/text v0.23.0 16 | gopkg.in/yaml.v3 v3.0.1 17 | ) 18 | 19 | require ( 20 | github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect 21 | github.com/kr/fs v0.1.0 // indirect 22 | github.com/mattn/go-colorable v0.1.13 // indirect 23 | github.com/mattn/go-isatty v0.0.20 // indirect 24 | github.com/pmezard/go-difflib v1.0.0 // indirect 25 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 26 | github.com/stretchr/objx v0.5.2 // indirect 27 | github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect 28 | golang.org/x/sys v0.31.0 // indirect 29 | ) 30 | -------------------------------------------------------------------------------- /pkg/3fs_client/steps.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package fsclient 16 | 17 | import ( 18 | "context" 19 | "strings" 20 | 21 | "github.com/open3fs/m3fs/pkg/errors" 22 | "github.com/open3fs/m3fs/pkg/task" 23 | ) 24 | 25 | type umountHostMountponitStep struct { 26 | task.BaseStep 27 | } 28 | 29 | func (s *umountHostMountponitStep) Execute(ctx context.Context) error { 30 | mp := s.Runtime.Services.Client.HostMountpoint 31 | 32 | out, err := s.Em.Runner.Exec(ctx, "mount") 33 | if err != nil { 34 | return errors.Annotate(err, "get mountponits") 35 | } 36 | if !strings.Contains(out, mp) { 37 | s.Logger.Infof("%s is not mounted, skip umount it", mp) 38 | return nil 39 | } 40 | 41 | _, err = s.Em.Runner.Exec(ctx, "umount", s.Runtime.Services.Client.HostMountpoint) 42 | if err != nil { 43 | return errors.Annotatef(err, "umount %s", mp) 44 | } 45 | s.Logger.Infof("Successfully umount %s", mp) 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /pkg/3fs_client/steps_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package fsclient 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/stretchr/testify/suite" 21 | 22 | "github.com/open3fs/m3fs/pkg/config" 23 | ttask "github.com/open3fs/m3fs/tests/task" 24 | ) 25 | 26 | var suiteRun = suite.Run 27 | 28 | func TestUmountHostMountpointSuite(t *testing.T) { 29 | suiteRun(t, &umountHostMountpointSuite{}) 30 | } 31 | 32 | type umountHostMountpointSuite struct { 33 | ttask.StepSuite 34 | 35 | step *umountHostMountponitStep 36 | } 37 | 38 | func (s *umountHostMountpointSuite) SetupTest() { 39 | s.StepSuite.SetupTest() 40 | 41 | s.Cfg.Services.Client.HostMountpoint = "/mnt/3fs" 42 | s.SetupRuntime() 43 | s.step = &umountHostMountponitStep{} 44 | s.step.Init(s.Runtime, s.MockEm, config.Node{}, s.Logger) 45 | } 46 | 47 | func (s *umountHostMountpointSuite) Test() { 48 | s.MockRunner.On("Exec", "mount", []string(nil)).Return("/mnt/3fs", nil) 49 | s.MockRunner.On("Exec", "umount", []string{"/mnt/3fs"}).Return("", nil) 50 | 51 | s.NoError(s.step.Execute(s.Ctx())) 52 | 53 | s.MockRunner.AssertExpectations(s.T()) 54 | } 55 | 56 | func (s *umountHostMountpointSuite) TestWithNotMount() { 57 | s.MockRunner.On("Exec", "mount", []string(nil)).Return("", nil) 58 | 59 | s.NoError(s.step.Execute(s.Ctx())) 60 | 61 | s.MockRunner.AssertExpectations(s.T()) 62 | } 63 | -------------------------------------------------------------------------------- /pkg/3fs_client/tasks.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package fsclient 16 | 17 | import ( 18 | "embed" 19 | "path" 20 | 21 | "github.com/open3fs/m3fs/pkg/common" 22 | "github.com/open3fs/m3fs/pkg/config" 23 | "github.com/open3fs/m3fs/pkg/external" 24 | "github.com/open3fs/m3fs/pkg/log" 25 | "github.com/open3fs/m3fs/pkg/task" 26 | "github.com/open3fs/m3fs/pkg/task/steps" 27 | ) 28 | 29 | var ( 30 | //go:embed templates/*.tmpl 31 | templatesFs embed.FS 32 | 33 | // ClientFuseMainLauncherTomlTmpl is the template content of hf3fs_fuse_main_launcher.toml 34 | ClientFuseMainLauncherTomlTmpl []byte 35 | // ClientMainTomlTmpl is the template content of hf3fs_fuse_main.toml 36 | ClientMainTomlTmpl []byte 37 | ) 38 | 39 | func init() { 40 | var err error 41 | ClientFuseMainLauncherTomlTmpl, err = templatesFs.ReadFile("templates/hf3fs_fuse_main_launcher.toml.tmpl") 42 | if err != nil { 43 | panic(err) 44 | } 45 | 46 | ClientMainTomlTmpl, err = templatesFs.ReadFile("templates/hf3fs_fuse_main.toml.tmpl") 47 | if err != nil { 48 | panic(err) 49 | } 50 | } 51 | 52 | const ( 53 | // ServiceName is the name of the 3fs client service. 54 | ServiceName = "hf3fs_fuse_main" 55 | serviceType = "FUSE" 56 | ) 57 | 58 | func getServiceWorkDir(workDir string) string { 59 | return path.Join(workDir, "client") 60 | } 61 | 62 | // Create3FSClientServiceTask is a task for creating 3fs client services. 63 | type Create3FSClientServiceTask struct { 64 | task.BaseTask 65 | } 66 | 67 | // Init initializes the task. 68 | func (t *Create3FSClientServiceTask) Init(r *task.Runtime, logger log.Interface) { 69 | t.BaseTask.SetName("Create3FSClientServiceTask") 70 | t.BaseTask.Init(r, logger) 71 | nodes := make([]config.Node, len(r.Cfg.Services.Client.Nodes)) 72 | client := r.Cfg.Services.Client 73 | for i, node := range client.Nodes { 74 | nodes[i] = r.Nodes[node] 75 | } 76 | runContainerVolumes := []*external.VolumeArgs{} 77 | if client.HostMountpoint != "" { 78 | runContainerVolumes = append(runContainerVolumes, &external.VolumeArgs{ 79 | Source: client.HostMountpoint, 80 | Target: client.HostMountpoint, 81 | Rshare: common.Pointer(true), 82 | }) 83 | } 84 | workDir := getServiceWorkDir(r.WorkDir) 85 | t.SetSteps([]task.StepConfig{ 86 | { 87 | Nodes: nodes, 88 | Parallel: true, 89 | NewStep: steps.NewPrepare3FSConfigStepFunc(&steps.Prepare3FSConfigStepSetup{ 90 | Service: ServiceName, 91 | ServiceWorkDir: workDir, 92 | MainAppTomlTmpl: []byte(""), 93 | MainLauncherTomlTmpl: ClientFuseMainLauncherTomlTmpl, 94 | MainTomlTmpl: ClientMainTomlTmpl, 95 | Extra3FSConfigFilesFunc: func(runtime *task.Runtime) []*steps.Extra3FSConfigFile { 96 | token, _ := r.LoadString(task.RuntimeUserTokenKey) 97 | return []*steps.Extra3FSConfigFile{ 98 | { 99 | FileName: "token.txt", 100 | Data: []byte(token), 101 | }, 102 | } 103 | }, 104 | }, 105 | ), 106 | }, 107 | { 108 | Nodes: []config.Node{nodes[0]}, 109 | NewStep: steps.NewUpload3FSMainConfigStepFunc( 110 | config.ImageName3FS, 111 | client.ContainerName, 112 | ServiceName, 113 | workDir, 114 | serviceType, 115 | ), 116 | }, 117 | { 118 | Nodes: nodes, 119 | Parallel: true, 120 | NewStep: steps.NewRun3FSContainerStepFunc( 121 | &steps.Run3FSContainerStepSetup{ 122 | ImgName: config.ImageName3FS, 123 | ContainerName: client.ContainerName, 124 | Service: ServiceName, 125 | WorkDir: workDir, 126 | ExtraVolumes: runContainerVolumes, 127 | UseRdmaNetwork: true, 128 | }), 129 | }, 130 | }) 131 | } 132 | 133 | // Delete3FSClientServiceTask is a task for deleting a 3fs client services. 134 | type Delete3FSClientServiceTask struct { 135 | task.BaseTask 136 | } 137 | 138 | // Init initializes the task. 139 | func (t *Delete3FSClientServiceTask) Init(r *task.Runtime, logger log.Interface) { 140 | t.BaseTask.SetName("Delete3FSClientServiceTask") 141 | t.BaseTask.Init(r, logger) 142 | client := r.Services.Client 143 | nodes := make([]config.Node, len(client.Nodes)) 144 | for i, node := range client.Nodes { 145 | nodes[i] = r.Nodes[node] 146 | } 147 | workDir := getServiceWorkDir(r.WorkDir) 148 | t.SetSteps([]task.StepConfig{ 149 | { 150 | Nodes: nodes, 151 | Parallel: true, 152 | NewStep: steps.NewRm3FSContainerStepFunc( 153 | client.ContainerName, 154 | ServiceName, 155 | workDir), 156 | }, 157 | { 158 | Nodes: nodes, 159 | Parallel: true, 160 | NewStep: func() task.Step { return new(umountHostMountponitStep) }, 161 | }, 162 | }) 163 | } 164 | -------------------------------------------------------------------------------- /pkg/3fs_client/templates/hf3fs_fuse_main_launcher.toml.tmpl: -------------------------------------------------------------------------------- 1 | allow_other = true 2 | cluster_id = '{{ .ClusterID }}' 3 | mountpoint = '{{ .HostMountpoint }}' 4 | token_file = '/opt/3fs/etc/token.txt' 5 | 6 | [client] 7 | default_compression_level = 0 8 | default_compression_threshold = '128KB' 9 | default_log_long_running_threshold = '0ns' 10 | default_report_metrics = false 11 | default_send_retry_times = 1 12 | default_timeout = '1s' 13 | enable_rdma_control = false 14 | force_use_tcp = false 15 | 16 | [client.io_worker] 17 | num_event_loop = 1 18 | rdma_connect_timeout = '5s' 19 | read_write_rdma_in_event_thread = false 20 | read_write_tcp_in_event_thread = false 21 | tcp_connect_timeout = '1s' 22 | wait_to_retry_send = '100ms' 23 | 24 | [client.io_worker.connect_concurrency_limiter] 25 | max_concurrency = 4 26 | 27 | [client.io_worker.ibsocket] 28 | buf_ack_batch = 8 29 | buf_signal_batch = 8 30 | buf_size = 16384 31 | drain_timeout = '5s' 32 | drop_connections = 0 33 | event_ack_batch = 128 34 | max_rd_atomic = 16 35 | max_rdma_wr = 128 36 | max_rdma_wr_per_post = 32 37 | max_sge = 1 38 | min_rnr_timer = 1 39 | record_bytes_per_peer = false 40 | record_latency_per_peer = false 41 | retry_cnt = 7 42 | rnr_retry = 0 43 | send_buf_cnt = 32 44 | sl = 0 45 | start_psn = 0 46 | timeout = 14 47 | 48 | [client.io_worker.transport_pool] 49 | max_connections = 1 50 | 51 | [client.processor] 52 | enable_coroutines_pool = true 53 | max_coroutines_num = 256 54 | max_processing_requests_num = 4096 55 | response_compression_level = 1 56 | response_compression_threshold = '128KB' 57 | 58 | [client.rdma_control] 59 | max_concurrent_transmission = 64 60 | 61 | [client.thread_pool] 62 | bg_thread_pool_stratetry = 'SHARED_QUEUE' 63 | collect_stats = false 64 | enable_work_stealing = false 65 | io_thread_pool_stratetry = 'SHARED_QUEUE' 66 | num_bg_threads = 2 67 | num_connect_threads = 2 68 | num_io_threads = 2 69 | num_proc_threads = 2 70 | proc_thread_pool_stratetry = 'SHARED_QUEUE' 71 | 72 | [ib_devices] 73 | allow_no_usable_devices = false 74 | allow_unknown_zone = true 75 | default_network_zone = 'UNKNOWN' 76 | default_pkey_index = 0 77 | default_roce_pkey_index = 0 78 | default_traffic_class = 0 79 | device_filter = [] 80 | fork_safe = true 81 | prefer_ibdevice = true 82 | skip_inactive_ports = true 83 | skip_unusable_device = true 84 | subnets = [] 85 | 86 | [mgmtd_client] 87 | accept_incomplete_routing_info_during_mgmtd_bootstrapping = true 88 | auto_extend_client_session_interval = '10s' 89 | auto_heartbeat_interval = '10s' 90 | auto_refresh_interval = '10s' 91 | enable_auto_extend_client_session = true 92 | enable_auto_heartbeat = false 93 | enable_auto_refresh = true 94 | mgmtd_server_addresses = {{ .MgmtdServerAddresses }} 95 | work_queue_size = 100 96 | -------------------------------------------------------------------------------- /pkg/artifact/tasks.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package artifact 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/open3fs/m3fs/pkg/config" 21 | "github.com/open3fs/m3fs/pkg/errors" 22 | "github.com/open3fs/m3fs/pkg/log" 23 | "github.com/open3fs/m3fs/pkg/task" 24 | ) 25 | 26 | // ExportArtifactTask is a task for exporting a 3fs artifact. 27 | type ExportArtifactTask struct { 28 | task.BaseTask 29 | 30 | localSteps []task.LocalStep 31 | } 32 | 33 | // Init initializes the task. 34 | func (t *ExportArtifactTask) Init(r *task.Runtime, logger log.Interface) { 35 | t.BaseTask.SetName("ExportArtifactTask") 36 | t.BaseTask.Init(r, logger) 37 | t.localSteps = []task.LocalStep{ 38 | new(prepareTmpDirStep), 39 | new(downloadImagesStep), 40 | new(tarFilesStep), 41 | } 42 | } 43 | 44 | // Run runs task steps 45 | func (t *ExportArtifactTask) Run(ctx context.Context) error { 46 | for _, step := range t.localSteps { 47 | step.Init(t.Runtime, log.Logger.Subscribe(log.FieldKeyNode, "")) 48 | if err := step.Execute(ctx); err != nil { 49 | return errors.Trace(err) 50 | } 51 | } 52 | return nil 53 | } 54 | 55 | // ImportArtifactTask is a task for importing the 3fs artifact. 56 | type ImportArtifactTask struct { 57 | task.BaseTask 58 | } 59 | 60 | // Init initializes the task. 61 | func (t *ImportArtifactTask) Init(r *task.Runtime, logger log.Interface) { 62 | t.BaseTask.SetName("ImportArtifactTask") 63 | t.BaseTask.Init(r, logger) 64 | t.SetSteps([]task.StepConfig{ 65 | { 66 | Nodes: []config.Node{r.Cfg.Nodes[0]}, 67 | NewStep: func() task.Step { return new(sha256sumArtifactStep) }, 68 | }, 69 | { 70 | Nodes: r.Cfg.Nodes, 71 | Parallel: true, 72 | NewStep: func() task.Step { return new(distributeArtifactStep) }, 73 | }, 74 | { 75 | Nodes: r.Cfg.Nodes, 76 | Parallel: true, 77 | NewStep: func() task.Step { return new(importArtifactStep) }, 78 | }, 79 | { 80 | Nodes: r.Cfg.Nodes, 81 | Parallel: true, 82 | NewStep: func() task.Step { return new(removeArtifactStep) }, 83 | }, 84 | }) 85 | } 86 | -------------------------------------------------------------------------------- /pkg/clickhouse/steps_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package clickhouse 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | "testing" 21 | 22 | "github.com/stretchr/testify/mock" 23 | "github.com/stretchr/testify/suite" 24 | 25 | "github.com/open3fs/m3fs/pkg/common" 26 | "github.com/open3fs/m3fs/pkg/config" 27 | "github.com/open3fs/m3fs/pkg/external" 28 | "github.com/open3fs/m3fs/pkg/task" 29 | ttask "github.com/open3fs/m3fs/tests/task" 30 | ) 31 | 32 | var suiteRun = suite.Run 33 | 34 | func TestGenClickhouseConfigStep(t *testing.T) { 35 | suiteRun(t, &genClickhouseConfigStepSuite{}) 36 | } 37 | 38 | type genClickhouseConfigStepSuite struct { 39 | ttask.StepSuite 40 | 41 | step *genClickhouseConfigStep 42 | } 43 | 44 | func (s *genClickhouseConfigStepSuite) SetupTest() { 45 | s.StepSuite.SetupTest() 46 | 47 | s.step = &genClickhouseConfigStep{} 48 | s.SetupRuntime() 49 | s.step.Init(s.Runtime, s.MockEm, config.Node{}, s.Logger) 50 | } 51 | 52 | func (s *genClickhouseConfigStepSuite) Test() { 53 | s.MockLocalFS.On("MkdirTemp", os.TempDir(), "3fs-clickhouse"). 54 | Return("/tmp/3fs-clickhouse.xxx", nil) 55 | s.MockLocalFS.On("WriteFile", "/tmp/3fs-clickhouse.xxx/config.xml", 56 | mock.AnythingOfType("[]uint8"), os.FileMode(0644)).Return(nil) 57 | s.MockLocalFS.On("WriteFile", "/tmp/3fs-clickhouse.xxx/3fs-monitor.sql", 58 | mock.AnythingOfType("[]uint8"), os.FileMode(0644)).Return(nil) 59 | 60 | s.NoError(s.step.Execute(s.Ctx())) 61 | 62 | tmpDirValue, ok := s.Runtime.Load(task.RuntimeClickhouseTmpDirKey) 63 | s.True(ok) 64 | tmpDir := tmpDirValue.(string) 65 | s.Equal("/tmp/3fs-clickhouse.xxx", tmpDir) 66 | } 67 | 68 | func TestStartContainerStep(t *testing.T) { 69 | suiteRun(t, &startContainerStepSuite{}) 70 | } 71 | 72 | type startContainerStepSuite struct { 73 | ttask.StepSuite 74 | 75 | step *startContainerStep 76 | } 77 | 78 | func (s *startContainerStepSuite) SetupTest() { 79 | s.StepSuite.SetupTest() 80 | 81 | s.step = &startContainerStep{} 82 | s.step.Init(s.Runtime, s.MockEm, config.Node{}, s.Logger) 83 | s.Runtime.Store(task.RuntimeClickhouseTmpDirKey, "/tmp/3f-clickhouse.xxx") 84 | } 85 | 86 | func (s *startContainerStepSuite) TestStartContainerStep() { 87 | dataDir := "/root/3fs/clickhouse/data" 88 | logDir := "/root/3fs/clickhouse/log" 89 | configDir := "/root/3fs/clickhouse/config.d" 90 | sqlDir := "/root/3fs/clickhouse/sql" 91 | s.MockFS.On("MkdirAll", dataDir).Return(nil) 92 | s.MockFS.On("MkdirAll", logDir).Return(nil) 93 | s.MockFS.On("MkdirAll", configDir).Return(nil) 94 | s.MockFS.On("MkdirAll", sqlDir).Return(nil) 95 | s.MockRunner.On("Scp", "/tmp/3f-clickhouse.xxx/config.xml", 96 | "/root/3fs/clickhouse/config.d/config.xml").Return(nil) 97 | s.MockRunner.On("Scp", "/tmp/3f-clickhouse.xxx/3fs-monitor.sql", 98 | "/root/3fs/clickhouse/sql/3fs-monitor.sql").Return(nil) 99 | img, err := s.Runtime.Cfg.Images.GetImage(config.ImageNameClickhouse) 100 | s.NoError(err) 101 | s.MockDocker.On("Run", &external.RunArgs{ 102 | Image: img, 103 | Name: common.Pointer("3fs-clickhouse"), 104 | HostNetwork: true, 105 | Detach: common.Pointer(true), 106 | Envs: map[string]string{ 107 | "CLICKHOUSE_USER": "default", 108 | "CLICKHOUSE_PASSWORD": "password", 109 | }, 110 | Volumes: []*external.VolumeArgs{ 111 | { 112 | Source: dataDir, 113 | Target: "/var/lib/clickhouse", 114 | }, 115 | { 116 | Source: logDir, 117 | Target: "/var/log/clickhouse-server", 118 | }, 119 | { 120 | Source: configDir, 121 | Target: "/etc/clickhouse-server/config.d", 122 | }, 123 | { 124 | Source: sqlDir, 125 | Target: "/tmp/sql", 126 | }, 127 | }, 128 | }).Return("", nil) 129 | 130 | s.NotNil(s.step) 131 | s.NoError(s.step.Execute(s.Ctx())) 132 | 133 | s.MockRunner.AssertExpectations(s.T()) 134 | s.MockFS.AssertExpectations(s.T()) 135 | s.MockDocker.AssertExpectations(s.T()) 136 | } 137 | 138 | func TestInitClusterStepSuite(t *testing.T) { 139 | suiteRun(t, &initClusterStepSuite{}) 140 | } 141 | 142 | type initClusterStepSuite struct { 143 | ttask.StepSuite 144 | 145 | step *initClusterStep 146 | } 147 | 148 | func (s *initClusterStepSuite) SetupTest() { 149 | s.StepSuite.SetupTest() 150 | 151 | s.step = &initClusterStep{} 152 | s.step.Init(s.Runtime, s.MockEm, config.Node{}, s.Logger) 153 | } 154 | 155 | func (s *initClusterStepSuite) TestInit() { 156 | s.MockDocker.On("Exec", s.Runtime.Services.Clickhouse.ContainerName, 157 | "bash", []string{ 158 | "-c", 159 | fmt.Sprintf(`"clickhouse-client --port %d -n < /tmp/sql/3fs-monitor.sql"`, 160 | s.Runtime.Services.Clickhouse.TCPPort), 161 | }). 162 | Return("", nil) 163 | 164 | s.NoError(s.step.Execute(s.Ctx())) 165 | 166 | s.MockDocker.AssertExpectations(s.T()) 167 | } 168 | 169 | func TestRmContainerStep(t *testing.T) { 170 | suiteRun(t, &rmContainerStepSuite{}) 171 | } 172 | 173 | type rmContainerStepSuite struct { 174 | ttask.StepSuite 175 | 176 | step *rmContainerStep 177 | dataDir string 178 | logDir string 179 | configDir string 180 | sqlDir string 181 | } 182 | 183 | func (s *rmContainerStepSuite) SetupTest() { 184 | s.StepSuite.SetupTest() 185 | 186 | s.step = &rmContainerStep{} 187 | s.SetupRuntime() 188 | s.step.Init(s.Runtime, s.MockEm, config.Node{}, s.Logger) 189 | s.dataDir = "/root/3fs/clickhouse/data" 190 | s.logDir = "/root/3fs/clickhouse/log" 191 | s.configDir = "/root/3fs/clickhouse/config.d" 192 | s.sqlDir = "/root/3fs/clickhouse/sql" 193 | } 194 | 195 | func (s *rmContainerStepSuite) TestRmContainerStep() { 196 | s.MockDocker.On("Rm", s.Cfg.Services.Clickhouse.ContainerName, true).Return("", nil) 197 | s.MockRunner.On("Exec", "rm", []string{"-rf", s.dataDir}).Return("", nil) 198 | s.MockRunner.On("Exec", "rm", []string{"-rf", s.logDir}).Return("", nil) 199 | s.MockRunner.On("Exec", "rm", []string{"-rf", s.configDir}).Return("", nil) 200 | s.MockRunner.On("Exec", "rm", []string{"-rf", s.sqlDir}).Return("", nil) 201 | 202 | s.NoError(s.step.Execute(s.Ctx())) 203 | 204 | s.MockRunner.AssertExpectations(s.T()) 205 | s.MockDocker.AssertExpectations(s.T()) 206 | } 207 | -------------------------------------------------------------------------------- /pkg/clickhouse/tasks.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package clickhouse 16 | 17 | import ( 18 | "github.com/open3fs/m3fs/pkg/config" 19 | "github.com/open3fs/m3fs/pkg/log" 20 | "github.com/open3fs/m3fs/pkg/task" 21 | "github.com/open3fs/m3fs/pkg/task/steps" 22 | ) 23 | 24 | // CreateClickhouseClusterTask is a task for creating a new clickhouse cluster. 25 | type CreateClickhouseClusterTask struct { 26 | task.BaseTask 27 | } 28 | 29 | // Init initializes the task. 30 | func (t *CreateClickhouseClusterTask) Init(r *task.Runtime, logger log.Interface) { 31 | t.BaseTask.SetName("CreateClickhouseClusterTask") 32 | t.BaseTask.Init(r, logger) 33 | nodes := make([]config.Node, len(r.Cfg.Services.Clickhouse.Nodes)) 34 | for i, node := range r.Cfg.Services.Clickhouse.Nodes { 35 | nodes[i] = r.Nodes[node] 36 | } 37 | t.SetSteps([]task.StepConfig{ 38 | { 39 | Nodes: []config.Node{nodes[0]}, 40 | NewStep: func() task.Step { return new(genClickhouseConfigStep) }, 41 | }, 42 | { 43 | Nodes: []config.Node{nodes[0]}, 44 | NewStep: func() task.Step { return new(startContainerStep) }, 45 | }, 46 | { 47 | Nodes: []config.Node{nodes[0]}, 48 | NewStep: func() task.Step { return new(initClusterStep) }, 49 | }, 50 | { 51 | Nodes: []config.Node{nodes[0]}, 52 | NewStep: steps.NewCleanupLocalStepFunc(task.RuntimeClickhouseTmpDirKey), 53 | }, 54 | }) 55 | } 56 | 57 | // DeleteClickhouseClusterTask is a task for deleting a clickhouse cluster. 58 | type DeleteClickhouseClusterTask struct { 59 | task.BaseTask 60 | } 61 | 62 | // Init initializes the task. 63 | func (t *DeleteClickhouseClusterTask) Init(r *task.Runtime, logger log.Interface) { 64 | t.BaseTask.SetName("DeleteClickhouseClusterTask") 65 | t.BaseTask.Init(r, logger) 66 | nodes := make([]config.Node, len(r.Cfg.Services.Clickhouse.Nodes)) 67 | for i, node := range r.Cfg.Services.Clickhouse.Nodes { 68 | nodes[i] = r.Nodes[node] 69 | } 70 | t.SetSteps([]task.StepConfig{ 71 | { 72 | Nodes: nodes, 73 | NewStep: func() task.Step { return new(rmContainerStep) }, 74 | }, 75 | }) 76 | } 77 | -------------------------------------------------------------------------------- /pkg/clickhouse/templates/config.tmpl: -------------------------------------------------------------------------------- 1 | 2 | :: 3 | 0.0.0.0 4 | 1 5 | {{ .TCPPort }} 6 | 7 | -------------------------------------------------------------------------------- /pkg/clickhouse/templates/sql.tmpl: -------------------------------------------------------------------------------- 1 | CREATE DATABASE IF NOT EXISTS {{ .Db }}; 2 | 3 | CREATE TABLE IF NOT EXISTS {{ .Db }}.counters ( 4 | `TIMESTAMP` DateTime CODEC(DoubleDelta), 5 | `metricName` LowCardinality(String) CODEC(ZSTD(1)), 6 | `host` LowCardinality(String) CODEC(ZSTD(1)), 7 | `tag` LowCardinality(String) CODEC(ZSTD(1)), 8 | `val` Int64 CODEC(ZSTD(1)), 9 | `mount_name` LowCardinality(String) CODEC(ZSTD(1)), 10 | `instance` String CODEC(ZSTD(1)), 11 | `io` LowCardinality(String) CODEC(ZSTD(1)), 12 | `uid` LowCardinality(String) CODEC(ZSTD(1)), 13 | `pod` String CODEC(ZSTD(1)), 14 | `thread` LowCardinality(String) CODEC(ZSTD(1)), 15 | `statusCode` LowCardinality(String) CODEC(ZSTD(1)) 16 | ) 17 | ENGINE = MergeTree 18 | PRIMARY KEY (metricName, host, pod, instance, TIMESTAMP) 19 | PARTITION BY toDate(TIMESTAMP) 20 | ORDER BY (metricName, host, pod, instance, TIMESTAMP) 21 | TTL TIMESTAMP + toIntervalMonth(1) 22 | SETTINGS index_granularity = 8192; 23 | 24 | CREATE TABLE IF NOT EXISTS {{ .Db }}.distributions ( 25 | `TIMESTAMP` DateTime CODEC(DoubleDelta), 26 | `metricName` LowCardinality(String) CODEC(ZSTD(1)), 27 | `host` LowCardinality(String) CODEC(ZSTD(1)), 28 | `tag` LowCardinality(String) CODEC(ZSTD(1)), 29 | `count` Float64 CODEC(ZSTD(1)), 30 | `mean` Float64 CODEC(ZSTD(1)), 31 | `min` Float64 CODEC(ZSTD(1)), 32 | `max` Float64 CODEC(ZSTD(1)), 33 | `p50` Float64 CODEC(ZSTD(1)), 34 | `p90` Float64 CODEC(ZSTD(1)), 35 | `p95` Float64 CODEC(ZSTD(1)), 36 | `p99` Float64 CODEC(ZSTD(1)), 37 | `mount_name` LowCardinality(String) CODEC(ZSTD(1)), 38 | `instance` String CODEC(ZSTD(1)), 39 | `io` LowCardinality(String) CODEC(ZSTD(1)), 40 | `uid` LowCardinality(String) CODEC(ZSTD(1)), 41 | `method` LowCardinality(String) CODEC(ZSTD(1)), 42 | `pod` String CODEC(ZSTD(1)), 43 | `thread` LowCardinality(String) CODEC(ZSTD(1)), 44 | `statusCode` LowCardinality(String) CODEC(ZSTD(1)) 45 | ) 46 | ENGINE = MergeTree 47 | PRIMARY KEY (metricName, host, pod, instance, TIMESTAMP) 48 | PARTITION BY toDate(TIMESTAMP) 49 | ORDER BY (metricName, host, pod, instance, TIMESTAMP) 50 | TTL TIMESTAMP + toIntervalMonth(1) 51 | SETTINGS index_granularity = 8192; 52 | -------------------------------------------------------------------------------- /pkg/common/copy.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import ( 18 | "fmt" 19 | "reflect" 20 | ) 21 | 22 | // CopyFields copies fields from src to dest. 23 | // The dest and src can be different types but the field with same name 24 | // must has the same type. 25 | func CopyFields(dest, src any, fields ...string) error { 26 | if len(fields) == 0 { 27 | // Enforce the caller to specify fields to be copied. 28 | return fmt.Errorf("no field to be copied") 29 | } 30 | 31 | destValue := reflect.ValueOf(dest) 32 | if destValue.Kind() != reflect.Ptr { 33 | return fmt.Errorf("dest must be a pointer") 34 | } 35 | destValue = destValue.Elem() 36 | if destValue.Kind() != reflect.Struct { 37 | return fmt.Errorf("dest must be a pointer to a struct") 38 | } 39 | 40 | srcValue := reflect.ValueOf(src) 41 | if srcValue.Kind() != reflect.Ptr { 42 | return fmt.Errorf("src must be a pointer") 43 | } 44 | srcValue = srcValue.Elem() 45 | if srcValue.Kind() != reflect.Struct { 46 | return fmt.Errorf("src must be a pointer to a struct") 47 | } 48 | 49 | for _, name := range fields { 50 | srcFieldValue := srcValue.FieldByName(name) 51 | if !srcFieldValue.IsValid() { 52 | return fmt.Errorf("field %s not found in src", name) 53 | } 54 | destFieldValue := destValue.FieldByName(name) 55 | if !destFieldValue.IsValid() { 56 | return fmt.Errorf("field %s not found in dest", name) 57 | } 58 | if destFieldValue.Type().Name() != srcFieldValue.Type().Name() { 59 | return fmt.Errorf("type mismatched of field %s", name) 60 | } 61 | destFieldValue.Set(srcFieldValue) 62 | } 63 | 64 | return nil 65 | } 66 | 67 | // UpdateStructByMap set map's values to struct's fields, 68 | // it will ignore fields that don't exist in the struct. 69 | // return: 70 | // updateFields: the updated fields to struct, the slice element format is snake 71 | func UpdateStructByMap(target any, source map[string]any) ([]string, error) { 72 | if len(source) == 0 { 73 | return nil, nil 74 | } 75 | 76 | if reflect.TypeOf(target).Kind() != reflect.Ptr { 77 | return nil, fmt.Errorf("target must be a pointer") 78 | } 79 | 80 | targetValue := reflect.ValueOf(target).Elem() 81 | if targetValue.Kind() != reflect.Struct { 82 | return nil, fmt.Errorf("target must be a pointer to a struct") 83 | } 84 | 85 | var updateFields []string 86 | for key, value := range source { 87 | fieldValue := targetValue.FieldByName(key) 88 | 89 | if fieldValue.IsValid() && fieldValue.CanSet() { 90 | if fieldValue.Type() == reflect.TypeOf(value) { 91 | fieldValue.Set(reflect.ValueOf(value)) 92 | updateFields = append(updateFields, CamelToSnake(key)) 93 | } else { 94 | return nil, fmt.Errorf("type mismatch for field '%s'", key) 95 | } 96 | } 97 | } 98 | 99 | return updateFields, nil 100 | } 101 | -------------------------------------------------------------------------------- /pkg/common/pprint.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import ( 18 | "github.com/davecgh/go-spew/spew" 19 | ) 20 | 21 | var ( 22 | spewConfig = &spew.ConfigState{ 23 | Indent: "\t", 24 | MaxDepth: 3, 25 | DisableMethods: true, 26 | } 27 | ) 28 | 29 | // PrettyDump prints Golang objects in a beautiful way. 30 | func PrettyDump(a ...any) { 31 | spewConfig.Dump(a...) 32 | } 33 | 34 | // PrettySdump prints Golang objects in a beautiful way to string. 35 | func PrettySdump(a ...any) string { 36 | return spewConfig.Sdump(a...) 37 | } 38 | -------------------------------------------------------------------------------- /pkg/common/string.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import ( 18 | "math/rand" 19 | "time" 20 | ) 21 | 22 | var ( 23 | chars = [...]rune{ 24 | 'a', 'b', 'c', 'd', 'e', 'f', 'g', 25 | 'h', 'i', 'j', 'k', 'l', 'm', 'n', 26 | 'o', 'p', 'q', 'r', 's', 't', 27 | 'u', 'v', 'w', 'x', 'y', 'z', 28 | '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 29 | } 30 | ) 31 | 32 | // RandomString return random string with specified length 33 | func RandomString(strlen int, withoutNumber ...bool) string { 34 | rand := rand.New(rand.NewSource(time.Now().UnixNano())) 35 | result := make([]rune, strlen) 36 | var max int 37 | if len(withoutNumber) > 0 && withoutNumber[0] { 38 | max = len(chars) - 10 39 | } else { 40 | max = len(chars) 41 | } 42 | for i := 0; i < strlen; i++ { 43 | result[i] = chars[rand.Intn(max)] 44 | } 45 | return string(result) 46 | } 47 | -------------------------------------------------------------------------------- /pkg/common/string_camelcase.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // The support for split camelcase word to a list of words implemented by 16 | // github.com/fatih/camelcase. 17 | // Original file: https://github.com/fatih/camelcase/blob/master/camelcase.go 18 | // Give credit where credit is due. 19 | 20 | package common 21 | 22 | import ( 23 | "strings" 24 | "unicode" 25 | "unicode/utf8" 26 | 27 | "golang.org/x/text/cases" 28 | "golang.org/x/text/language" 29 | ) 30 | 31 | // Split splits the camelcase word and returns a list of words. It also 32 | // supports digits. Both lower camel case and upper camel case are supported. 33 | // For more info please check: http://en.wikipedia.org/wiki/CamelCase 34 | // 35 | // Examples 36 | // 37 | // "" => [""] 38 | // "lowercase" => ["lowercase"] 39 | // "Class" => ["Class"] 40 | // "MyClass" => ["My", "Class"] 41 | // "MyC" => ["My", "C"] 42 | // "HTML" => ["HTML"] 43 | // "PDFLoader" => ["PDF", "Loader"] 44 | // "AString" => ["A", "String"] 45 | // "SimpleXMLParser" => ["Simple", "XML", "Parser"] 46 | // "vimRPCPlugin" => ["vim", "RPC", "Plugin"] 47 | // "GL11Version" => ["GL", "11", "Version"] 48 | // "99Bottles" => ["99", "Bottles"] 49 | // "May5" => ["May", "5"] 50 | // "BFG9000" => ["BFG", "9000"] 51 | // "BöseÜberraschung" => ["Böse", "Überraschung"] 52 | // "Two spaces" => ["Two", " ", "spaces"] 53 | // "BadUTF8\xe2\xe2\xa1" => ["BadUTF8\xe2\xe2\xa1"] 54 | // 55 | // Splitting rules 56 | // 57 | // 1. If string is not valid UTF-8, return it without splitting as 58 | // single item array. 59 | // 2. Assign all unicode characters into one of 4 sets: lower case 60 | // letters, upper case letters, numbers, and all other characters. 61 | // 3. Iterate through characters of string, introducing splits 62 | // between adjacent characters that belong to different sets. 63 | // 4. Iterate through array of split strings, and if a given string 64 | // is upper case: 65 | // if subsequent string is lower case: 66 | // move last character of upper case string to beginning of 67 | // lower case string 68 | func Split(src string) (entries []string) { 69 | // don't split invalid utf8 70 | if !utf8.ValidString(src) { 71 | return []string{src} 72 | } 73 | entries = []string{} 74 | var runes [][]rune 75 | lastClass := 0 76 | class := 0 77 | // split into fields based on class of unicode character 78 | for _, r := range src { 79 | switch true { 80 | case unicode.IsLower(r): 81 | class = 1 82 | case unicode.IsUpper(r): 83 | class = 2 84 | case unicode.IsDigit(r): 85 | class = 3 86 | default: 87 | class = 4 88 | } 89 | if class == lastClass { 90 | runes[len(runes)-1] = append(runes[len(runes)-1], r) 91 | } else { 92 | runes = append(runes, []rune{r}) 93 | } 94 | lastClass = class 95 | } 96 | // handle upper case -> lower case sequences, e.g. 97 | // "PDFL", "oader" -> "PDF", "Loader" 98 | for i := 0; i < len(runes)-1; i++ { 99 | if unicode.IsUpper(runes[i][0]) && unicode.IsLower(runes[i+1][0]) { 100 | runes[i+1] = append([]rune{runes[i][len(runes[i])-1]}, runes[i+1]...) 101 | runes[i] = runes[i][:len(runes[i])-1] 102 | } 103 | } 104 | // construct []string from results 105 | for _, s := range runes { 106 | if len(s) > 0 { 107 | entries = append(entries, string(s)) 108 | } 109 | } 110 | return entries 111 | } 112 | 113 | func camelToConcatenatedStr(camel string, sep rune) string { 114 | runes := []rune(camel) 115 | length := len(runes) 116 | 117 | out := make([]rune, 0, length) 118 | for i := 0; i < length; i++ { 119 | if i > 0 && unicode.IsUpper(runes[i]) && 120 | ((i+1 < length && unicode.IsLower(runes[i+1])) || unicode.IsLower(runes[i-1])) { 121 | out = append(out, sep) 122 | } 123 | out = append(out, unicode.ToLower(runes[i])) 124 | } 125 | 126 | return string(out) 127 | } 128 | 129 | // CamelToKebab cast camel case to kebab case 130 | func CamelToKebab(camel string) string { 131 | return camelToConcatenatedStr(camel, '-') 132 | } 133 | 134 | // CamelToSnake cast camel case to snake case 135 | func CamelToSnake(camel string) string { 136 | return camelToConcatenatedStr(camel, '_') 137 | } 138 | 139 | // SnakeToCamel cast snake case to camel case 140 | func SnakeToCamel(snake string) string { 141 | caser := cases.Title(language.English) 142 | return strings.Join( 143 | strings.Split( 144 | caser.String( 145 | strings.Join( 146 | strings.Split(snake, "_"), 147 | " "), 148 | ), " ", 149 | ), 150 | "") 151 | } 152 | 153 | // FormattedCamel cast to formatted cammel. 154 | // For example: VirtualMachineID to VirtualMachineId 155 | func FormattedCamel(str string) string { 156 | return SnakeToCamel(CamelToSnake(str)) 157 | } 158 | 159 | // CamelListToSnakeList cast each string in camel list to snake case. 160 | func CamelListToSnakeList(camelList []string) []string { 161 | snakeList := make([]string, len(camelList)) 162 | for i := range camelList { 163 | snakeList[i] = CamelToSnake(camelList[i]) 164 | } 165 | return snakeList 166 | } 167 | 168 | // SnakeListToCamelList cast each string in snake list to camel case. 169 | func SnakeListToCamelList(snakeList []string) []string { 170 | camelList := make([]string, len(snakeList)) 171 | for i := range snakeList { 172 | camelList[i] = SnakeToCamel(snakeList[i]) 173 | } 174 | return camelList 175 | } 176 | -------------------------------------------------------------------------------- /pkg/common/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | // Pointer returns pointer of val 18 | func Pointer[T any](val T) *T { 19 | return &val 20 | } 21 | -------------------------------------------------------------------------------- /pkg/common/version.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | // Version info of this binary, set by -ldflags 18 | var ( 19 | Version string 20 | GitSha string 21 | BuildTime string 22 | ) 23 | -------------------------------------------------------------------------------- /pkg/common/worker.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import ( 18 | "context" 19 | "sync" 20 | ) 21 | 22 | // WorkerPool is a pool of workers that process items concurrently. 23 | type WorkerPool[T any] struct { 24 | lock sync.Mutex 25 | wg sync.WaitGroup 26 | start bool 27 | ch chan T 28 | size int 29 | errors []error 30 | procFunc func(context.Context, T) error 31 | cancel func() 32 | } 33 | 34 | // Add adds an item to the pool. 35 | func (wp *WorkerPool[T]) Add(item T) { 36 | wp.ch <- item 37 | } 38 | 39 | // Errors returns the errors encountered during processing. 40 | func (wp *WorkerPool[T]) Errors() []error { 41 | wp.lock.Lock() 42 | defer wp.lock.Unlock() 43 | return wp.errors 44 | } 45 | 46 | // Start starts the worker pool. 47 | func (wp *WorkerPool[T]) Start(ctx context.Context) { 48 | wp.lock.Lock() 49 | defer wp.lock.Unlock() 50 | if wp.start { 51 | return 52 | } 53 | wp.start = true 54 | 55 | var extCtx context.Context 56 | extCtx, wp.cancel = context.WithCancel(context.Background()) 57 | for i := 0; i < wp.size; i++ { 58 | wp.wg.Add(1) 59 | go func() { 60 | defer wp.wg.Done() 61 | for { 62 | select { 63 | case <-extCtx.Done(): 64 | return 65 | case item := <-wp.ch: 66 | err := wp.procFunc(ctx, item) 67 | if err != nil { 68 | wp.lock.Lock() 69 | wp.errors = append(wp.errors, err) 70 | wp.lock.Unlock() 71 | } 72 | } 73 | } 74 | }() 75 | } 76 | } 77 | 78 | // Join waits for all workers to finish. 79 | func (wp *WorkerPool[T]) Join() { 80 | wp.lock.Lock() 81 | if wp.cancel != nil { 82 | wp.cancel() 83 | } 84 | wp.lock.Unlock() 85 | wp.wg.Wait() 86 | wp.start = false 87 | } 88 | 89 | // NewWorkerPool creates a new worker pool. 90 | func NewWorkerPool[T any](procFunc func(context.Context, T) error, size int) *WorkerPool[T] { 91 | return &WorkerPool[T]{ 92 | ch: make(chan T), 93 | procFunc: procFunc, 94 | size: size, 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /pkg/config/image.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package config 16 | 17 | import ( 18 | "fmt" 19 | "net/url" 20 | 21 | "github.com/open3fs/m3fs/pkg/errors" 22 | ) 23 | 24 | // defines image names 25 | const ( 26 | ImageNameFdb = "foundationdb" 27 | ImageNameClickhouse = "clickhouse" 28 | ImageName3FS = "3fs" 29 | ImageNameGrafana = "grafana" 30 | ) 31 | 32 | // Image is component container image config 33 | type Image struct { 34 | Repo string 35 | Tag string 36 | } 37 | 38 | // Images contains all component container image configs 39 | type Images struct { 40 | Registry string `yaml:"registry"` 41 | FFFS Image `yaml:"3fs"` // 3fs cannot used as struct filed name, so we use fffs instead 42 | Clickhouse Image `yaml:"clickhouse"` 43 | Grafana Image `yaml:"grafana"` 44 | Fdb Image `yaml:"fdb"` 45 | } 46 | 47 | func (i *Images) getImage(imgName string) (Image, error) { 48 | switch imgName { 49 | case ImageNameFdb: 50 | return i.Fdb, nil 51 | case ImageName3FS: 52 | return i.FFFS, nil 53 | case ImageNameClickhouse: 54 | return i.Clickhouse, nil 55 | case ImageNameGrafana: 56 | return i.Grafana, nil 57 | default: 58 | return Image{}, errors.Errorf("invalid image name %s", imgName) 59 | } 60 | } 61 | 62 | // GetImage get image path of target component 63 | func (i *Images) GetImage(imgName string) (string, error) { 64 | imagePath, err := i.GetImageWithoutRegistry(imgName) 65 | if err != nil { 66 | return "", errors.Trace(err) 67 | } 68 | if i.Registry != "" { 69 | var err error 70 | imagePath, err = url.JoinPath(i.Registry, imagePath) 71 | if err != nil { 72 | return "", errors.Annotatef(err, "get image path of %s", imgName) 73 | } 74 | } 75 | 76 | return imagePath, nil 77 | } 78 | 79 | // GetImageWithoutRegistry get image path without registry 80 | func (i *Images) GetImageWithoutRegistry(imgName string) (string, error) { 81 | img, err := i.getImage(imgName) 82 | if err != nil { 83 | return "", errors.Trace(err) 84 | } 85 | return fmt.Sprintf("%s:%s", img.Repo, img.Tag), nil 86 | } 87 | 88 | // GetImageFileName gets image file name 89 | func (i Images) GetImageFileName(imgName string) (string, error) { 90 | img, err := i.getImage(imgName) 91 | if err != nil { 92 | return "", errors.Trace(err) 93 | } 94 | return fmt.Sprintf("%s_%s_amd64.docker", imgName, img.Tag), nil 95 | } 96 | -------------------------------------------------------------------------------- /pkg/config/image_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package config 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/stretchr/testify/suite" 21 | 22 | "github.com/open3fs/m3fs/tests/base" 23 | ) 24 | 25 | func TestImageSuite(t *testing.T) { 26 | suite.Run(t, new(imageSuite)) 27 | } 28 | 29 | type imageSuite struct { 30 | base.Suite 31 | } 32 | 33 | func (s *imageSuite) TestGetImage() { 34 | cfg := NewConfigWithDefaults() 35 | cfg.Images.FFFS.Tag = "1.1.1" 36 | 37 | img, err := cfg.Images.GetImage(ImageName3FS) 38 | s.NoError(err) 39 | s.Equal("open3fs/3fs:1.1.1", img) 40 | } 41 | 42 | func (s *imageSuite) TestGetImageWithRegistry() { 43 | cfg := NewConfigWithDefaults() 44 | cfg.Images.Registry = "hub.docker.com" 45 | cfg.Images.FFFS.Tag = "1.1.1" 46 | 47 | img, err := cfg.Images.GetImage(ImageName3FS) 48 | s.NoError(err) 49 | s.Equal("hub.docker.com/open3fs/3fs:1.1.1", img) 50 | } 51 | -------------------------------------------------------------------------------- /pkg/config/types.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package config 16 | 17 | // ServiceType defines the type of service in the m3fs cluster 18 | type ServiceType string 19 | 20 | // Service type constants 21 | const ( 22 | ServiceMgmtd ServiceType = "mgmtd" 23 | ServiceMonitor ServiceType = "monitor" 24 | ServiceStorage ServiceType = "storage" 25 | ServiceFdb ServiceType = "fdb" 26 | ServiceClickhouse ServiceType = "clickhouse" 27 | ServiceMeta ServiceType = "meta" 28 | ServiceClient ServiceType = "client" 29 | ) 30 | 31 | // AllServiceTypes is a list of all service types 32 | var AllServiceTypes = []ServiceType{ 33 | ServiceStorage, 34 | ServiceFdb, 35 | ServiceMeta, 36 | ServiceMgmtd, 37 | ServiceMonitor, 38 | ServiceClickhouse, 39 | ServiceClient, 40 | } 41 | 42 | // ServiceDisplayNames is a map of service type to display name 43 | var ServiceDisplayNames = map[ServiceType]string{ 44 | ServiceStorage: "storage", 45 | ServiceFdb: "foundationdb", 46 | ServiceMeta: "meta", 47 | ServiceMgmtd: "mgmtd", 48 | ServiceMonitor: "monitor", 49 | ServiceClickhouse: "clickhouse", 50 | ServiceClient: "client", 51 | } 52 | -------------------------------------------------------------------------------- /pkg/errors/base_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package errors 16 | 17 | import ( 18 | "os" 19 | "testing" 20 | 21 | "github.com/stretchr/testify/require" 22 | "github.com/stretchr/testify/suite" 23 | ) 24 | 25 | type Suite struct { 26 | suite.Suite 27 | } 28 | 29 | // R returns a require context. 30 | func (s *Suite) R() *require.Assertions { 31 | return s.Require() 32 | } 33 | 34 | func TestMain(m *testing.M) { 35 | os.Exit(m.Run()) 36 | } 37 | -------------------------------------------------------------------------------- /pkg/errors/error_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package errors 16 | 17 | import ( 18 | "fmt" 19 | "strings" 20 | "testing" 21 | 22 | "github.com/stretchr/testify/suite" 23 | ) 24 | 25 | const ( 26 | thisFile = "pkg/errors/error_test.go" 27 | ) 28 | 29 | func TestBaseErrorSuite(t *testing.T) { 30 | suite.Run(t, new(baseErrorSuite)) 31 | } 32 | 33 | type baseErrorSuite struct { 34 | Suite 35 | } 36 | 37 | func (s *baseErrorSuite) TestBasicError() { 38 | r := s.R() 39 | 40 | err := New("basic") 41 | r.Equal("basic", err.Error()) 42 | r.Nil(err.(Underlying).Underlie()) 43 | r.Nil(err.(Underlying).Underlie()) 44 | r.Equal(fmt.Sprintf("%s:%d:(*baseErrorSuite).TestBasicError: basic", thisFile, err.(*Err).frame.line), 45 | fmt.Sprintf("%+v", err)) 46 | 47 | err = Errorf("basic %d", 10) 48 | r.Equal("basic 10", err.Error()) 49 | r.Nil(err.(Underlying).Underlie()) 50 | r.Equal(fmt.Sprintf("%s:%d:(*baseErrorSuite).TestBasicError: basic 10", thisFile, err.(*Err).frame.line), 51 | fmt.Sprintf("%+v", err)) 52 | } 53 | 54 | func (s *baseErrorSuite) TestAnnotateNormalError() { 55 | r := s.R() 56 | 57 | err1 := fmt.Errorf("err1") 58 | err2 := Trace(err1) 59 | err3 := Annotate(err2, "err3") 60 | 61 | // Cause 62 | r.Equal(err1, Cause(err1)) 63 | r.Equal(err1, Cause(err2)) 64 | r.Equal(err1, Cause(err3)) 65 | } 66 | 67 | //nolint:dupl 68 | func (s *baseErrorSuite) TestStackTrace() { 69 | r := s.R() 70 | 71 | err1 := New("err1") 72 | err2 := Trace(err1) 73 | err3 := Annotate(err2, "err3") 74 | err4 := Annotatef(err3, "err%d", 4) 75 | 76 | // Cause 77 | r.Equal(err1, Cause(err1)) 78 | r.Equal(err1, Cause(err2)) 79 | r.Equal(err1, Cause(err3)) 80 | r.Equal(err1, Cause(err4)) 81 | 82 | // StackTrace 83 | err1Line := err1.(*Err).frame.line 84 | r.NotEmpty(err1Line) 85 | err2Line := err1Line + 1 86 | err3Line := err1Line + 2 87 | err4Line := err1Line + 3 88 | result := StackTrace(err4) 89 | lines := strings.Split(result, "\n") 90 | r.Len(lines, 4) 91 | r.Equal(fmt.Sprintf("%s:%d:(*baseErrorSuite).TestStackTrace: err1", thisFile, err1Line), lines[0]) 92 | r.Equal(fmt.Sprintf("%s:%d:(*baseErrorSuite).TestStackTrace: ", thisFile, err2Line), lines[1]) 93 | r.Equal(fmt.Sprintf("%s:%d:(*baseErrorSuite).TestStackTrace: err3", thisFile, err3Line), lines[2]) 94 | r.Equal(fmt.Sprintf("%s:%d:(*baseErrorSuite).TestStackTrace: err4", thisFile, err4Line), lines[3]) 95 | } 96 | 97 | func (s *baseErrorSuite) TestStackTraceWithNormalError() { 98 | r := s.R() 99 | 100 | err1 := fmt.Errorf("err1") 101 | err2 := Trace(err1) 102 | err3 := Annotate(err2, "err3") 103 | err4 := Annotatef(err3, "err%d", 4) 104 | 105 | // Cause 106 | r.Equal(err1, Cause(err1)) 107 | r.Equal(err1, Cause(err2)) 108 | r.Equal(err1, Cause(err3)) 109 | r.Equal(err1, Cause(err4)) 110 | 111 | // StackTrace 112 | err2Line := err2.(*Err).frame.line 113 | r.NotEmpty(err2Line) 114 | err3Line := err2Line + 1 115 | err4Line := err2Line + 2 116 | result := StackTrace(err4) 117 | lines := strings.Split(result, "\n") 118 | r.Len(lines, 4) 119 | r.Equal("err1", lines[0]) 120 | r.Equal(fmt.Sprintf("%s:%d:(*baseErrorSuite).TestStackTraceWithNormalError: ", thisFile, err2Line), lines[1]) 121 | r.Equal(fmt.Sprintf("%s:%d:(*baseErrorSuite).TestStackTraceWithNormalError: err3", thisFile, err3Line), lines[2]) 122 | r.Equal( 123 | fmt.Sprintf("%s:%d:(*baseErrorSuite).TestStackTraceWithNormalError: err4", thisFile, err4Line), 124 | lines[3]) 125 | } 126 | 127 | // TestError error 128 | type TestError struct { 129 | *Err 130 | Field string 131 | } 132 | 133 | // NewTestError error 134 | func NewTestError(message string) error { 135 | err := &TestError{} 136 | err.Field = "field" 137 | err.Err = rawNew("[TestError] " + message + " " + err.Field) 138 | err.Caller(1) 139 | return err 140 | } 141 | 142 | func TestCustomErrorSuite(t *testing.T) { 143 | suite.Run(t, new(customErrorSuite)) 144 | } 145 | 146 | type customErrorSuite struct { 147 | Suite 148 | } 149 | 150 | func (s *customErrorSuite) Test() { 151 | r := s.R() 152 | 153 | err := NewTestError("test message") 154 | r.Equal("[TestError] test message field", err.Error()) 155 | r.Nil(err.(Underlying).Underlie()) 156 | } 157 | 158 | //nolint:dupl 159 | func (s *customErrorSuite) TestStackTrace() { 160 | r := s.R() 161 | 162 | err1 := NewTestError("test message") 163 | err2 := Trace(err1) 164 | err3 := Annotate(err2, "err3") 165 | err4 := Annotatef(err3, "err%d", 4) 166 | 167 | // Cause 168 | r.Equal(err1, Cause(err1)) 169 | r.Equal(err1, Cause(err2)) 170 | r.Equal(err1, Cause(err3)) 171 | r.Equal(err1, Cause(err4)) 172 | 173 | // StackTrace 174 | errLine := err1.(*TestError).frame.line 175 | r.NotEmpty(errLine) 176 | err2Line := errLine + 1 177 | err3Line := errLine + 2 178 | err4Line := errLine + 3 179 | result := StackTrace(err4) 180 | lines := strings.Split(result, "\n") 181 | r.Len(lines, 4) 182 | r.Equal(fmt.Sprintf("%s:%d:(*customErrorSuite).TestStackTrace: [TestError] test message field", 183 | thisFile, errLine), lines[0]) 184 | r.Equal(fmt.Sprintf("%s:%d:(*customErrorSuite).TestStackTrace: ", thisFile, err2Line), lines[1]) 185 | r.Equal(fmt.Sprintf("%s:%d:(*customErrorSuite).TestStackTrace: err3", thisFile, err3Line), lines[2]) 186 | r.Equal(fmt.Sprintf("%s:%d:(*customErrorSuite).TestStackTrace: err4", thisFile, err4Line), lines[3]) 187 | } 188 | -------------------------------------------------------------------------------- /pkg/errors/path.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package errors 16 | 17 | import ( 18 | "runtime" 19 | "strings" 20 | ) 21 | 22 | var ( 23 | goPath string 24 | goPathLen int 25 | ) 26 | 27 | func init() { 28 | _, file, _, ok := runtime.Caller(0) 29 | if !ok { 30 | return 31 | } 32 | size := len(file) 33 | suffixLen := len("pkg/errors/path.go") // also trim project directory here 34 | goPath = file[:size-suffixLen] 35 | goPathLen = len(goPath) 36 | } 37 | 38 | func trimGOPATH(path string) string { 39 | if strings.HasPrefix(path, goPath) { 40 | return path[goPathLen:] 41 | } 42 | return path 43 | } 44 | -------------------------------------------------------------------------------- /pkg/external/base_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package external_test 16 | 17 | import ( 18 | "context" 19 | "os" 20 | "testing" 21 | 22 | "github.com/sirupsen/logrus" 23 | "github.com/stretchr/testify/require" 24 | "github.com/stretchr/testify/suite" 25 | 26 | "github.com/open3fs/m3fs/pkg/external" 27 | "github.com/open3fs/m3fs/pkg/log" 28 | ) 29 | 30 | var suiteRun = suite.Run 31 | 32 | type Suite struct { 33 | suite.Suite 34 | 35 | r *MockedRunner 36 | em *external.Manager 37 | } 38 | 39 | func (s *Suite) SetupSuite() { 40 | s.T().Parallel() 41 | } 42 | 43 | func (s *Suite) SetupTest() { 44 | s.r = NewMockedRunner(s.T()) 45 | log.InitLogger(logrus.DebugLevel) 46 | s.em = external.NewManager(s.r, log.Logger) 47 | } 48 | 49 | // Ctx returns a context used in test. 50 | func (s *Suite) Ctx() context.Context { 51 | return context.TODO() 52 | } 53 | 54 | // R returns a require context. 55 | func (s *Suite) R() *require.Assertions { 56 | return s.Require() 57 | } 58 | 59 | func TestMain(m *testing.M) { 60 | os.Exit(m.Run()) 61 | } 62 | -------------------------------------------------------------------------------- /pkg/external/command.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package external 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "strings" 21 | ) 22 | 23 | // Command define command 24 | type Command struct { 25 | runner RunnerInterface 26 | 27 | cmdName string 28 | args []string 29 | } 30 | 31 | // Command gets the command 32 | func (cmd *Command) Command() string { 33 | if len(cmd.args) > 0 { 34 | return cmd.cmdName + " " + strings.Join(cmd.args, " ") 35 | } 36 | return cmd.cmdName 37 | } 38 | 39 | // AppendArgs append new args to current args 40 | func (cmd *Command) AppendArgs(args ...any) { 41 | for _, arg := range args { 42 | cmd.args = append(cmd.args, fmt.Sprintf("%v", arg)) 43 | } 44 | } 45 | 46 | // Exec execute the command 47 | func (cmd *Command) Exec(ctx context.Context) (out string, err error) { 48 | if cmd.cmdName == "" { 49 | return "", fmt.Errorf("No command") 50 | } 51 | return cmd.runner.NonSudoExec(ctx, cmd.cmdName, cmd.args...) 52 | } 53 | 54 | // SudoExec execute the command 55 | func (cmd *Command) SudoExec(ctx context.Context) (out string, err error) { 56 | if cmd.cmdName == "" { 57 | return "", fmt.Errorf("No command") 58 | } 59 | return cmd.runner.Exec(ctx, cmd.cmdName, cmd.args...) 60 | } 61 | 62 | func (cmd *Command) String() string { 63 | return fmt.Sprintf("cmd: %s", cmd.Command()) 64 | } 65 | 66 | // NewCommand inits a new command 67 | func NewCommand(cmdName string, args ...any) *Command { 68 | cmd := &Command{ 69 | cmdName: cmdName, 70 | } 71 | cmd.AppendArgs(args...) 72 | 73 | return cmd 74 | } 75 | -------------------------------------------------------------------------------- /pkg/external/disk.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package external 16 | 17 | import "github.com/open3fs/m3fs/pkg/log" 18 | 19 | // DiskInterface provides interface about disk. 20 | type DiskInterface interface { 21 | GetNvmeDisks() ([]string, error) 22 | } 23 | 24 | type diskExternal struct { 25 | externalBase 26 | } 27 | 28 | func (de *diskExternal) init(em *Manager, logger log.Interface) { 29 | de.externalBase.init(em, logger) 30 | em.Disk = de 31 | } 32 | 33 | func (de *diskExternal) GetNvmeDisks() ([]string, error) { 34 | // TODO: implement GetNvmeDisks 35 | return nil, nil 36 | } 37 | 38 | func init() { 39 | registerNewExternalFunc(func() externalInterface { 40 | return new(diskExternal) 41 | }) 42 | } 43 | -------------------------------------------------------------------------------- /pkg/external/docker.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package external 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | 21 | "github.com/open3fs/m3fs/pkg/errors" 22 | "github.com/open3fs/m3fs/pkg/log" 23 | ) 24 | 25 | // DockerInterface provides interface about docker. 26 | type DockerInterface interface { 27 | GetContainer(string) string 28 | Run(ctx context.Context, args *RunArgs) (out string, err error) 29 | Rm(ctx context.Context, name string, force bool) (out string, err error) 30 | Exec(context.Context, string, string, ...string) (out string, err error) 31 | Load(ctx context.Context, path string) (out string, err error) 32 | Tag(ctx context.Context, src, dst string) error 33 | } 34 | 35 | type dockerExternal struct { 36 | externalBase 37 | } 38 | 39 | func (de *dockerExternal) init(em *Manager, logger log.Interface) { 40 | de.externalBase.init(em, logger) 41 | em.Docker = de 42 | } 43 | 44 | func (de *dockerExternal) GetContainer(name string) string { 45 | // TODO: implement docker.GetContainer 46 | return "" 47 | } 48 | 49 | // RunArgs defines args for docker run command. 50 | type RunArgs struct { 51 | Image string 52 | HostNetwork bool 53 | Entrypoint *string 54 | Rm *bool 55 | Command []string 56 | Privileged *bool 57 | Ulimits map[string]string 58 | Name *string 59 | Detach *bool 60 | Publish []*PublishArgs 61 | Volumes []*VolumeArgs 62 | Envs map[string]string 63 | } 64 | 65 | // PublishArgs defines args for publishing a container port. 66 | type PublishArgs struct { 67 | HostAddress *string 68 | HostPort int 69 | ContainerPort int 70 | Protocol *string 71 | } 72 | 73 | // VolumeArgs defines args for binding a volume. 74 | type VolumeArgs struct { 75 | Source string 76 | Target string 77 | Rshare *bool 78 | } 79 | 80 | func (de *dockerExternal) Run(ctx context.Context, args *RunArgs) (out string, err error) { 81 | params := []string{"run"} 82 | if args.Name != nil { 83 | params = append(params, "--name", *args.Name) 84 | } 85 | if args.Detach != nil && *args.Detach { 86 | params = append(params, "--detach") 87 | } 88 | if args.HostNetwork { 89 | params = append(params, "--network", "host") 90 | } 91 | for key, val := range args.Envs { 92 | params = append(params, "-e", fmt.Sprintf("%s=%s", key, val)) 93 | } 94 | if args.Entrypoint != nil { 95 | params = append(params, "--entrypoint", *args.Entrypoint) 96 | } 97 | if args.Rm != nil && *args.Rm { 98 | params = append(params, "--rm") 99 | } 100 | if args.Privileged != nil && *args.Privileged { 101 | params = append(params, "--privileged") 102 | } 103 | for key, val := range args.Ulimits { 104 | params = append(params, "--ulimit", fmt.Sprintf("%s=%s", key, val)) 105 | } 106 | for _, publishArg := range args.Publish { 107 | publishInfo := fmt.Sprintf("%d:%d", publishArg.HostPort, publishArg.ContainerPort) 108 | if publishArg.HostAddress != nil { 109 | publishInfo = *publishArg.HostAddress + ":" + publishInfo 110 | } 111 | if publishArg.Protocol != nil { 112 | publishInfo = publishInfo + "/" + *publishArg.Protocol 113 | } 114 | params = append(params, "-p", publishInfo) 115 | } 116 | for _, volumeArg := range args.Volumes { 117 | volBind := fmt.Sprintf("%s:%s", volumeArg.Source, volumeArg.Target) 118 | if volumeArg.Rshare != nil && *volumeArg.Rshare { 119 | volBind += ":rshared" 120 | } 121 | params = append(params, "--volume", volBind) 122 | } 123 | params = append(params, args.Image) 124 | if len(args.Command) > 0 { 125 | params = append(params, args.Command...) 126 | } 127 | out, err = de.run(ctx, "docker", params...) 128 | return out, errors.Trace(err) 129 | } 130 | 131 | func (de *dockerExternal) Rm(ctx context.Context, name string, force bool) (out string, err error) { 132 | args := []string{"rm"} 133 | if force { 134 | args = append(args, "--force") 135 | } 136 | args = append(args, name) 137 | out, err = de.run(ctx, "docker", args...) 138 | return out, errors.Trace(err) 139 | } 140 | 141 | func (de *dockerExternal) Exec( 142 | ctx context.Context, container, cmd string, args ...string) (out string, err error) { 143 | 144 | params := []string{"exec", container, cmd} 145 | params = append(params, args...) 146 | out, err = de.run(ctx, "docker", params...) 147 | return out, errors.Trace(err) 148 | } 149 | 150 | func (de *dockerExternal) Load(ctx context.Context, path string) (out string, err error) { 151 | out, err = de.run(ctx, "docker", "load", "-i", path) 152 | return out, errors.Trace(err) 153 | } 154 | 155 | func (de *dockerExternal) Tag(ctx context.Context, src, dst string) error { 156 | _, err := de.run(ctx, "docker", "tag", src, dst) 157 | return errors.Trace(err) 158 | } 159 | 160 | func init() { 161 | registerNewExternalFunc(func() externalInterface { 162 | return new(dockerExternal) 163 | }) 164 | } 165 | -------------------------------------------------------------------------------- /pkg/external/docker_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package external_test 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/open3fs/m3fs/pkg/common" 21 | "github.com/open3fs/m3fs/pkg/external" 22 | ) 23 | 24 | func TestDockerRunSuite(t *testing.T) { 25 | suiteRun(t, new(dockerRunSuite)) 26 | } 27 | 28 | type dockerRunSuite struct { 29 | Suite 30 | } 31 | 32 | func (s *dockerRunSuite) Test() { 33 | containerName := "3fs-clickhouse" 34 | detach := true 35 | hostAddress := "127.0.0.1" 36 | protocol := "tcp" 37 | args := &external.RunArgs{ 38 | Image: "clickhouse/clickhouse-server:latest", 39 | Name: &containerName, 40 | Detach: &detach, 41 | Entrypoint: common.Pointer("''"), 42 | Rm: common.Pointer(true), 43 | Command: []string{"ls"}, 44 | Privileged: common.Pointer(true), 45 | Ulimits: map[string]string{ 46 | "nproc": "65535:65535", 47 | }, 48 | Envs: map[string]string{ 49 | "A": "B", 50 | }, 51 | HostNetwork: true, 52 | Publish: []*external.PublishArgs{ 53 | { 54 | HostAddress: &hostAddress, 55 | HostPort: 9000, 56 | ContainerPort: 9000, 57 | Protocol: &protocol, 58 | }, 59 | }, 60 | Volumes: []*external.VolumeArgs{ 61 | { 62 | Source: "/path/to/data", 63 | Target: "/clickhouse/data", 64 | Rshare: common.Pointer(true), 65 | }, 66 | }, 67 | } 68 | mockCmd := "docker run --name 3fs-clickhouse --detach --network host -e A=B --entrypoint '' --rm " + 69 | "--privileged --ulimit nproc=65535:65535 -p 127.0.0.1:9000:9000/tcp " + 70 | "--volume /path/to/data:/clickhouse/data:rshared clickhouse/clickhouse-server:latest ls" 71 | s.r.MockExec(mockCmd, "", nil) 72 | _, err := s.em.Docker.Run(s.Ctx(), args) 73 | s.NoError(err) 74 | } 75 | 76 | func TestDockerRmSuite(t *testing.T) { 77 | suiteRun(t, new(dockerRmSuite)) 78 | } 79 | 80 | type dockerRmSuite struct { 81 | Suite 82 | } 83 | 84 | func (s *dockerRmSuite) Test() { 85 | mockCmd := "docker rm --force test" 86 | s.r.MockExec(mockCmd, "", nil) 87 | _, err := s.em.Docker.Rm(s.Ctx(), "test", true) 88 | s.NoError(err) 89 | } 90 | 91 | func TestDockerExecSuite(t *testing.T) { 92 | suiteRun(t, new(dockerExecSuite)) 93 | } 94 | 95 | type dockerExecSuite struct { 96 | Suite 97 | } 98 | 99 | func (s *dockerExecSuite) Test() { 100 | mockCmd := "docker exec fdb fdbcli --exec status" 101 | s.r.MockExec(mockCmd, "", nil) 102 | _, err := s.em.Docker.Exec(s.Ctx(), "fdb", "fdbcli", "--exec", "status") 103 | s.NoError(err) 104 | } 105 | -------------------------------------------------------------------------------- /pkg/external/manager.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package external 16 | 17 | import ( 18 | "context" 19 | "sync" 20 | 21 | "github.com/open3fs/m3fs/pkg/config" 22 | "github.com/open3fs/m3fs/pkg/errors" 23 | "github.com/open3fs/m3fs/pkg/log" 24 | ) 25 | 26 | type externalInterface interface { 27 | init(em *Manager, logger log.Interface) 28 | } 29 | 30 | type externalBase struct { 31 | em *Manager 32 | logger log.Interface 33 | } 34 | 35 | func (eb *externalBase) init(em *Manager, logger log.Interface) { 36 | eb.em = em 37 | eb.logger = logger 38 | } 39 | 40 | func (eb *externalBase) run(ctx context.Context, cmdName string, args ...string) (string, error) { 41 | anyArgs := []any{} 42 | for _, arg := range args { 43 | anyArgs = append(anyArgs, arg) 44 | } 45 | 46 | cmd := NewCommand(cmdName, anyArgs...) 47 | cmd.runner = eb.em.Runner 48 | out, err := cmd.SudoExec(ctx) 49 | if err != nil { 50 | return out, errors.Annotatef(err, "sudo run cmd [%s]", cmd.String()) 51 | } 52 | return out, nil 53 | 54 | } 55 | 56 | // create a new external 57 | type newExternalFunc func() externalInterface 58 | 59 | var ( 60 | newExternals []newExternalFunc 61 | lock sync.Mutex 62 | ) 63 | 64 | func registerNewExternalFunc(f newExternalFunc) { 65 | lock.Lock() 66 | defer lock.Unlock() 67 | newExternals = append(newExternals, f) 68 | } 69 | 70 | // Manager provides a way to use all external interfaces 71 | type Manager struct { 72 | Runner RunnerInterface 73 | 74 | Net NetInterface 75 | Docker DockerInterface 76 | Disk DiskInterface 77 | FS FSInterface 78 | } 79 | 80 | // NewManagerFunc type of new manager func. 81 | type NewManagerFunc func() *Manager 82 | 83 | // NewManager create a new external manager 84 | func NewManager(runner RunnerInterface, logger log.Interface) (em *Manager) { 85 | em = &Manager{ 86 | Runner: runner, 87 | } 88 | for _, newExternal := range newExternals { 89 | newExternal().init(em, logger) 90 | } 91 | return em 92 | } 93 | 94 | var remoteManagerCache sync.Map 95 | 96 | // NewRemoteRunnerManager create a new remote runner manager 97 | func NewRemoteRunnerManager(node *config.Node, logger log.Interface) (*Manager, error) { 98 | mgr, ok := remoteManagerCache.Load(node) 99 | if ok { 100 | return mgr.(*Manager), nil 101 | } 102 | runner, err := NewRemoteRunner(&RemoteRunnerCfg{ 103 | Username: node.Username, 104 | Password: node.Password, 105 | TargetHost: node.Host, 106 | TargetPort: node.Port, 107 | Logger: logger, 108 | // TODO: add timeout config 109 | }) 110 | if err != nil { 111 | return nil, errors.Annotatef(err, "create remote runner for node [%s]", node.Name) 112 | } 113 | 114 | return NewManager(runner, logger), nil 115 | } 116 | -------------------------------------------------------------------------------- /pkg/external/network.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package external 16 | 17 | import "github.com/open3fs/m3fs/pkg/log" 18 | 19 | // NetInterface provides interface about network. 20 | type NetInterface interface { 21 | GetRdmaLinks() ([]string, error) 22 | } 23 | 24 | type netExternal struct { 25 | externalBase 26 | } 27 | 28 | func (ne *netExternal) init(em *Manager, logger log.Interface) { 29 | ne.externalBase.init(em, logger) 30 | em.Net = ne 31 | } 32 | 33 | func (ne *netExternal) GetRdmaLinks() ([]string, error) { 34 | // TODO: add get list of RDMA links logic 35 | return nil, nil 36 | } 37 | 38 | func init() { 39 | registerNewExternalFunc(func() externalInterface { 40 | return new(netExternal) 41 | }) 42 | } 43 | -------------------------------------------------------------------------------- /pkg/fdb/steps.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package fdb 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "net" 21 | "path" 22 | "strconv" 23 | "strings" 24 | "time" 25 | 26 | "github.com/open3fs/m3fs/pkg/common" 27 | "github.com/open3fs/m3fs/pkg/config" 28 | "github.com/open3fs/m3fs/pkg/errors" 29 | "github.com/open3fs/m3fs/pkg/external" 30 | "github.com/open3fs/m3fs/pkg/task" 31 | ) 32 | 33 | type genClusterFileContentStep struct { 34 | task.BaseStep 35 | } 36 | 37 | func (s *genClusterFileContentStep) Execute(context.Context) error { 38 | nodes := make([]string, len(s.Runtime.Services.Fdb.Nodes)) 39 | fdb := s.Runtime.Services.Fdb 40 | for i, fdbNode := range fdb.Nodes { 41 | for _, node := range s.Runtime.Nodes { 42 | if node.Name == fdbNode { 43 | nodes[i] = net.JoinHostPort(node.Host, strconv.Itoa(fdb.Port)) 44 | } 45 | } 46 | } 47 | 48 | clusterFileContent := fmt.Sprintf("%s:%s@%s", 49 | common.RandomString(10), common.RandomString(10), strings.Join(nodes, ",")) 50 | s.Logger.Debugf("fdb cluster file content: %s", clusterFileContent) 51 | s.Runtime.Store(task.RuntimeFdbClusterFileContentKey, clusterFileContent) 52 | return nil 53 | } 54 | 55 | func getServiceWorkDir(workDir string) string { 56 | return path.Join(workDir, "fdb") 57 | } 58 | 59 | type runContainerStep struct { 60 | task.BaseStep 61 | } 62 | 63 | func (s *runContainerStep) Execute(ctx context.Context) error { 64 | workDir := getServiceWorkDir(s.Runtime.WorkDir) 65 | dataDir := path.Join(workDir, "data") 66 | err := s.Em.FS.MkdirAll(ctx, dataDir) 67 | if err != nil { 68 | return errors.Annotatef(err, "mkdir %s", dataDir) 69 | } 70 | logDir := path.Join(workDir, "logs") 71 | err = s.Em.FS.MkdirAll(ctx, logDir) 72 | if err != nil { 73 | return errors.Annotatef(err, "mkdir %s", logDir) 74 | } 75 | img, err := s.Runtime.Cfg.Images.GetImage(config.ImageNameFdb) 76 | if err != nil { 77 | return errors.Trace(err) 78 | } 79 | clusterContentI, _ := s.Runtime.Load(task.RuntimeFdbClusterFileContentKey) 80 | clusterContent := clusterContentI.(string) 81 | args := &external.RunArgs{ 82 | Image: img, 83 | Name: &s.Runtime.Services.Fdb.ContainerName, 84 | HostNetwork: true, 85 | Detach: common.Pointer(true), 86 | Envs: map[string]string{ 87 | "FDB_CLUSTER_FILE_CONTENTS": clusterContent, 88 | }, 89 | Volumes: []*external.VolumeArgs{ 90 | { 91 | Source: dataDir, 92 | Target: "/var/fdb/data", 93 | }, 94 | { 95 | Source: logDir, 96 | Target: "/var/fdb/logs", 97 | }, 98 | }, 99 | } 100 | _, err = s.Em.Docker.Run(ctx, args) 101 | if err != nil { 102 | return errors.Trace(err) 103 | } 104 | 105 | s.Logger.Infof("Started fdb container %s successfully", s.Runtime.Services.Fdb.ContainerName) 106 | return nil 107 | } 108 | 109 | type initClusterStep struct { 110 | task.BaseStep 111 | } 112 | 113 | func (s *initClusterStep) Execute(ctx context.Context) error { 114 | err := s.initCluster(ctx) 115 | if err != nil { 116 | return errors.Trace(err) 117 | } 118 | 119 | return s.waitClusterInitialized(ctx) 120 | } 121 | 122 | func (s *initClusterStep) initCluster(ctx context.Context) error { 123 | s.Logger.Infof("Initializing fdb cluster") 124 | // TODO: initialize fdb cluster with replication and coordinator setting 125 | _, err := s.Em.Docker.Exec(ctx, s.Runtime.Services.Fdb.ContainerName, 126 | "fdbcli", "--exec", "'configure new single ssd'") 127 | if err != nil { 128 | return errors.Annotate(err, "initialize fdb cluster") 129 | } 130 | 131 | return nil 132 | } 133 | 134 | func (s *initClusterStep) waitClusterInitialized(ctx context.Context) error { 135 | s.Logger.Infof("Waiting for fdb cluster initialized") 136 | tctx, cancel := context.WithTimeout(ctx, s.Runtime.Services.Fdb.WaitClusterTimeout) 137 | defer cancel() 138 | 139 | for { 140 | out, err := s.Em.Docker.Exec(tctx, s.Runtime.Services.Fdb.ContainerName, 141 | "fdbcli", "--exec", "'status minimal'") 142 | if err != nil { 143 | return errors.Annotate(err, "wait fdb cluster initialized") 144 | } 145 | if strings.Contains(out, "The database is available.") { 146 | break 147 | } 148 | time.Sleep(time.Second) 149 | } 150 | 151 | s.Logger.Infof("Initialized fdb cluster") 152 | return nil 153 | } 154 | 155 | type rmContainerStep struct { 156 | task.BaseStep 157 | } 158 | 159 | func (s *rmContainerStep) Execute(ctx context.Context) error { 160 | containerName := s.Runtime.Services.Fdb.ContainerName 161 | s.Logger.Infof("Removing fdb container %s", containerName) 162 | _, err := s.Em.Docker.Rm(ctx, containerName, true) 163 | if err != nil { 164 | return errors.Trace(err) 165 | } 166 | 167 | workDir := getServiceWorkDir(s.Runtime.WorkDir) 168 | dataDir := path.Join(workDir, "data") 169 | _, err = s.Em.Runner.Exec(ctx, "rm", "-rf", dataDir) 170 | if err != nil { 171 | return errors.Annotatef(err, "rm %s", dataDir) 172 | } 173 | s.Logger.Infof("Removed fdb container data dir %s", dataDir) 174 | 175 | logDir := path.Join(workDir, "logs") 176 | _, err = s.Em.Runner.Exec(ctx, "rm", "-rf", logDir) 177 | if err != nil { 178 | return errors.Annotatef(err, "rm %s", logDir) 179 | } 180 | s.Logger.Infof("Removed fdb container log dir %s", logDir) 181 | 182 | s.Logger.Infof("Removed fdb container %s successfully", containerName) 183 | return nil 184 | } 185 | -------------------------------------------------------------------------------- /pkg/fdb/tasks.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package fdb 16 | 17 | import ( 18 | "github.com/open3fs/m3fs/pkg/config" 19 | "github.com/open3fs/m3fs/pkg/log" 20 | "github.com/open3fs/m3fs/pkg/task" 21 | ) 22 | 23 | // CreateFdbClusterTask is a task for creating a new FoundationDB cluster. 24 | type CreateFdbClusterTask struct { 25 | task.BaseTask 26 | } 27 | 28 | // Init initializes the task. 29 | func (t *CreateFdbClusterTask) Init(r *task.Runtime, logger log.Interface) { 30 | t.BaseTask.SetName("CreateFdbClusterTask") 31 | t.BaseTask.Init(r, logger) 32 | nodes := make([]config.Node, len(r.Cfg.Services.Fdb.Nodes)) 33 | for i, node := range r.Cfg.Services.Fdb.Nodes { 34 | nodes[i] = r.Nodes[node] 35 | } 36 | t.SetSteps([]task.StepConfig{ 37 | { 38 | Nodes: []config.Node{nodes[0]}, 39 | NewStep: func() task.Step { return new(genClusterFileContentStep) }, 40 | }, 41 | { 42 | Nodes: nodes, 43 | Parallel: true, 44 | NewStep: func() task.Step { return new(runContainerStep) }, 45 | }, 46 | { 47 | Nodes: []config.Node{nodes[0]}, 48 | RetryTime: 10, 49 | NewStep: func() task.Step { return new(initClusterStep) }, 50 | }, 51 | }) 52 | } 53 | 54 | // DeleteFdbClusterTask is a task for deleting a FoundationDB cluster. 55 | type DeleteFdbClusterTask struct { 56 | task.BaseTask 57 | } 58 | 59 | // Init initializes the task. 60 | func (t *DeleteFdbClusterTask) Init(r *task.Runtime, logger log.Interface) { 61 | t.BaseTask.SetName("DeleteFdbClusterTask") 62 | t.BaseTask.Init(r, logger) 63 | nodes := make([]config.Node, len(r.Cfg.Services.Fdb.Nodes)) 64 | for i, node := range r.Cfg.Services.Fdb.Nodes { 65 | nodes[i] = r.Nodes[node] 66 | } 67 | t.SetSteps([]task.StepConfig{ 68 | { 69 | Nodes: nodes, 70 | NewStep: func() task.Step { return new(rmContainerStep) }, 71 | }, 72 | }) 73 | } 74 | -------------------------------------------------------------------------------- /pkg/grafana/steps.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package grafana 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "embed" 21 | "fmt" 22 | "math/rand" 23 | "net" 24 | "path" 25 | "text/template" 26 | 27 | "github.com/open3fs/m3fs/pkg/common" 28 | "github.com/open3fs/m3fs/pkg/config" 29 | "github.com/open3fs/m3fs/pkg/errors" 30 | "github.com/open3fs/m3fs/pkg/external" 31 | "github.com/open3fs/m3fs/pkg/task" 32 | ) 33 | 34 | var ( 35 | //go:embed templates/* 36 | templatesFs embed.FS 37 | 38 | // DashboardTmpl is the template content of 3fs grafana dashboard 39 | DashboardTmpl []byte 40 | // DashboardProvisionTmpl is the template content of 3fs dashboard provision file 41 | DashboardProvisionTmpl []byte 42 | // DatasourceProvisionTmpl is the template content of 3fs datasource provision file 43 | DatasourceProvisionTmpl []byte 44 | ) 45 | 46 | func init() { 47 | var err error 48 | DashboardTmpl, err = templatesFs.ReadFile("templates/dashboard.json.tmpl") 49 | if err != nil { 50 | panic(err) 51 | } 52 | 53 | DashboardProvisionTmpl, err = templatesFs.ReadFile("templates/dashboard_provision.yaml.tmpl") 54 | if err != nil { 55 | panic(err) 56 | } 57 | 58 | DatasourceProvisionTmpl, err = templatesFs.ReadFile("templates/datasource_provision.yaml.tmpl") 59 | if err != nil { 60 | panic(err) 61 | } 62 | } 63 | 64 | func getServiceWorkDir(workDir string) string { 65 | return path.Join(workDir, "grafana") 66 | } 67 | 68 | type genGrafanaYamlStep struct { 69 | task.BaseStep 70 | } 71 | 72 | func (s *genGrafanaYamlStep) genYaml(ctx context.Context, dir, filename string, tmpl []byte, 73 | tmplData map[string]any) error { 74 | 75 | filepath := path.Join(dir, filename) 76 | s.Logger.Infof("Generating %s", filename) 77 | 78 | if err := s.Em.FS.MkdirAll(ctx, dir); err != nil { 79 | return errors.Annotatef(err, "mkdir %s", dir) 80 | } 81 | 82 | t, err := template.New(filepath).Parse(string(tmpl)) 83 | if err != nil { 84 | return errors.Annotatef(err, "parse %s template", filename) 85 | } 86 | data := new(bytes.Buffer) 87 | if err := t.Execute(data, tmplData); err != nil { 88 | return errors.Annotatef(err, "execute %s template", filename) 89 | } 90 | if err = s.Em.FS.WriteFile(filepath, data.Bytes(), 0644); err != nil { 91 | return errors.Trace(err) 92 | } 93 | 94 | return nil 95 | } 96 | 97 | func (s *genGrafanaYamlStep) Execute(ctx context.Context) error { 98 | workdir := getServiceWorkDir(s.Runtime.WorkDir) 99 | 100 | chNodeName := s.Runtime.Services.Clickhouse.Nodes[rand.Intn(len(s.Runtime.Services.Clickhouse.Nodes))] 101 | chNode := s.Runtime.Nodes[chNodeName] 102 | err := s.genYaml(ctx, path.Join(workdir, "datasources"), "datasource.yaml", 103 | DatasourceProvisionTmpl, map[string]any{ 104 | "CH_Database": "3fs", 105 | "CH_Host": chNode.Host, 106 | "CH_Port": s.Runtime.Services.Clickhouse.TCPPort, 107 | "CH_Username": s.Runtime.Services.Clickhouse.User, 108 | "CH_Password": s.Runtime.Services.Clickhouse.Password, 109 | }) 110 | if err != nil { 111 | return errors.Annotate(err, "generate datasource provisioning file") 112 | } 113 | 114 | err = s.genYaml(ctx, path.Join(workdir, "dashboards"), "dashboard.yaml", 115 | DashboardProvisionTmpl, nil) 116 | if err != nil { 117 | return errors.Annotate(err, "generate dashboard provisioning file") 118 | } 119 | 120 | err = s.genYaml(ctx, path.Join(workdir, "dashboards"), "3fs.json", 121 | DashboardTmpl, nil) 122 | if err != nil { 123 | return errors.Annotate(err, "generate 3fs dashboard json file") 124 | } 125 | 126 | return nil 127 | } 128 | 129 | type startContainerStep struct { 130 | task.BaseStep 131 | } 132 | 133 | func (s *startContainerStep) Execute(ctx context.Context) error { 134 | img, err := s.Runtime.Cfg.Images.GetImage(config.ImageNameGrafana) 135 | if err != nil { 136 | return errors.Trace(err) 137 | } 138 | workdir := getServiceWorkDir(s.Runtime.WorkDir) 139 | datasourceDir := path.Join(workdir, "datasources") 140 | dashboardDir := path.Join(workdir, "dashboards") 141 | args := &external.RunArgs{ 142 | Image: img, 143 | Name: &s.Runtime.Services.Grafana.ContainerName, 144 | HostNetwork: true, 145 | Detach: common.Pointer(true), 146 | Volumes: []*external.VolumeArgs{ 147 | { 148 | Source: datasourceDir, 149 | Target: "/etc/grafana/provisioning/datasources", 150 | }, 151 | { 152 | Source: dashboardDir, 153 | Target: "/etc/grafana/provisioning/dashboards", 154 | }, 155 | }, 156 | } 157 | _, err = s.Em.Docker.Run(ctx, args) 158 | if err != nil { 159 | return errors.Trace(err) 160 | } 161 | 162 | endpoint := net.JoinHostPort(s.Node.Host, fmt.Sprintf("%d", s.Runtime.Services.Grafana.Port)) 163 | s.Logger.Infof("Started grafana container %s successfully, service endpoint is http://%s,"+ 164 | " login with username \"admin\" and password \"admin\"", 165 | s.Runtime.Services.Grafana.ContainerName, endpoint) 166 | return nil 167 | } 168 | 169 | type rmContainerStep struct { 170 | task.BaseStep 171 | } 172 | 173 | func (s *rmContainerStep) Execute(ctx context.Context) error { 174 | containerName := s.Runtime.Services.Grafana.ContainerName 175 | s.Logger.Infof("Removing grafana container %s", containerName) 176 | _, err := s.Em.Docker.Rm(ctx, containerName, true) 177 | if err != nil { 178 | return errors.Trace(err) 179 | } 180 | 181 | s.Logger.Infof("Removed grafana container %s successfully", containerName) 182 | return nil 183 | } 184 | -------------------------------------------------------------------------------- /pkg/grafana/steps_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package grafana 16 | 17 | import ( 18 | "os" 19 | "path" 20 | "testing" 21 | 22 | "github.com/stretchr/testify/mock" 23 | "github.com/stretchr/testify/suite" 24 | 25 | "github.com/open3fs/m3fs/pkg/common" 26 | "github.com/open3fs/m3fs/pkg/config" 27 | "github.com/open3fs/m3fs/pkg/external" 28 | "github.com/open3fs/m3fs/pkg/task" 29 | ttask "github.com/open3fs/m3fs/tests/task" 30 | ) 31 | 32 | var suiteRun = suite.Run 33 | 34 | func TestGenGrafanaConfigStepSuite(t *testing.T) { 35 | suiteRun(t, &genGrafanaConfigStepSuite{}) 36 | } 37 | 38 | type genGrafanaConfigStepSuite struct { 39 | ttask.StepSuite 40 | 41 | step *genGrafanaYamlStep 42 | } 43 | 44 | func (s *genGrafanaConfigStepSuite) SetupTest() { 45 | s.StepSuite.SetupTest() 46 | 47 | s.step = &genGrafanaYamlStep{} 48 | s.Cfg.Nodes = []config.Node{{Name: "name", Host: "test"}} 49 | s.Cfg.Services.Clickhouse.Nodes = []string{"name"} 50 | s.SetupRuntime() 51 | s.step.Init(s.Runtime, s.MockEm, config.Node{}, s.Logger) 52 | } 53 | 54 | func (s *genGrafanaConfigStepSuite) Test() { 55 | workDir := getServiceWorkDir(s.Runtime.WorkDir) 56 | s.MockFS.On("MkdirAll", path.Join(workDir, "datasources")).Return(nil) 57 | s.MockFS.On("WriteFile", path.Join(workDir, "datasources", "datasource.yaml"), 58 | mock.AnythingOfType("[]uint8"), os.FileMode(0644)).Return(nil) 59 | s.MockFS.On("MkdirAll", path.Join(workDir, "dashboards")).Return(nil) 60 | s.MockFS.On("WriteFile", path.Join(workDir, "dashboards", "dashboard.yaml"), 61 | mock.AnythingOfType("[]uint8"), os.FileMode(0644)).Return(nil) 62 | s.MockFS.On("WriteFile", path.Join(workDir, "dashboards", "3fs.json"), 63 | mock.AnythingOfType("[]uint8"), os.FileMode(0644)).Return(nil) 64 | 65 | s.NoError(s.step.Execute(s.Ctx())) 66 | 67 | s.MockFS.AssertExpectations(s.T()) 68 | } 69 | 70 | func TestStartContainerStepSuite(t *testing.T) { 71 | suiteRun(t, &startContainerStepSuite{}) 72 | } 73 | 74 | type startContainerStepSuite struct { 75 | ttask.StepSuite 76 | 77 | step *startContainerStep 78 | } 79 | 80 | func (s *startContainerStepSuite) SetupTest() { 81 | s.StepSuite.SetupTest() 82 | 83 | s.step = &startContainerStep{} 84 | s.step.Init(s.Runtime, s.MockEm, config.Node{}, s.Logger) 85 | s.Runtime.Store(task.RuntimeGrafanaTmpDirKey, "/tmp/3f-clickhouse.xxx") 86 | } 87 | 88 | func (s *startContainerStepSuite) TestStartContainerStep() { 89 | workDir := getServiceWorkDir(s.Runtime.WorkDir) 90 | img, err := s.Runtime.Cfg.Images.GetImage(config.ImageNameGrafana) 91 | s.NoError(err) 92 | s.MockDocker.On("Run", &external.RunArgs{ 93 | Image: img, 94 | Name: common.Pointer("3fs-grafana"), 95 | HostNetwork: true, 96 | Detach: common.Pointer(true), 97 | Volumes: []*external.VolumeArgs{ 98 | { 99 | Source: path.Join(workDir, "datasources"), 100 | Target: "/etc/grafana/provisioning/datasources", 101 | }, 102 | { 103 | Source: path.Join(workDir, "dashboards"), 104 | Target: "/etc/grafana/provisioning/dashboards", 105 | }, 106 | }, 107 | }).Return("", nil) 108 | 109 | s.NotNil(s.step) 110 | s.NoError(s.step.Execute(s.Ctx())) 111 | 112 | s.MockDocker.AssertExpectations(s.T()) 113 | } 114 | func TestRmContainerStepSuite(t *testing.T) { 115 | suiteRun(t, &rmContainerStepSuite{}) 116 | } 117 | 118 | type rmContainerStepSuite struct { 119 | ttask.StepSuite 120 | 121 | step *rmContainerStep 122 | } 123 | 124 | func (s *rmContainerStepSuite) SetupTest() { 125 | s.StepSuite.SetupTest() 126 | 127 | s.step = &rmContainerStep{} 128 | s.SetupRuntime() 129 | s.step.Init(s.Runtime, s.MockEm, config.Node{}, s.Logger) 130 | } 131 | 132 | func (s *rmContainerStepSuite) TestRmContainerStep() { 133 | s.MockDocker.On("Rm", s.Cfg.Services.Grafana.ContainerName, true).Return("", nil) 134 | 135 | s.NoError(s.step.Execute(s.Ctx())) 136 | 137 | s.MockRunner.AssertExpectations(s.T()) 138 | } 139 | -------------------------------------------------------------------------------- /pkg/grafana/tasks.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package grafana 16 | 17 | import ( 18 | "github.com/open3fs/m3fs/pkg/config" 19 | "github.com/open3fs/m3fs/pkg/log" 20 | "github.com/open3fs/m3fs/pkg/task" 21 | ) 22 | 23 | // CreateGrafanaServiceTask is a task for creating a new grafana service. 24 | type CreateGrafanaServiceTask struct { 25 | task.BaseTask 26 | } 27 | 28 | // Init initializes the task. 29 | func (t *CreateGrafanaServiceTask) Init(r *task.Runtime, logger log.Interface) { 30 | t.BaseTask.SetName("CreateGrafanaServiceTask") 31 | t.BaseTask.Init(r, logger) 32 | nodes := make([]config.Node, len(r.Cfg.Services.Grafana.Nodes)) 33 | for i, node := range r.Cfg.Services.Grafana.Nodes { 34 | nodes[i] = r.Nodes[node] 35 | } 36 | t.SetSteps([]task.StepConfig{ 37 | { 38 | Nodes: nodes, 39 | NewStep: func() task.Step { return new(genGrafanaYamlStep) }, 40 | }, 41 | { 42 | Nodes: nodes, 43 | NewStep: func() task.Step { return new(startContainerStep) }, 44 | }, 45 | }) 46 | } 47 | 48 | // DeleteGrafanaServiceTask is a task for deleting a grafana service. 49 | type DeleteGrafanaServiceTask struct { 50 | task.BaseTask 51 | } 52 | 53 | // Init initializes the task. 54 | func (t *DeleteGrafanaServiceTask) Init(r *task.Runtime, logger log.Interface) { 55 | t.BaseTask.SetName("DeleteGrafanaServiceTask") 56 | t.BaseTask.Init(r, logger) 57 | nodes := make([]config.Node, len(r.Cfg.Services.Grafana.Nodes)) 58 | for i, node := range r.Cfg.Services.Grafana.Nodes { 59 | nodes[i] = r.Nodes[node] 60 | } 61 | t.SetSteps([]task.StepConfig{ 62 | { 63 | Nodes: nodes, 64 | NewStep: func() task.Step { return new(rmContainerStep) }, 65 | }, 66 | }) 67 | } 68 | -------------------------------------------------------------------------------- /pkg/grafana/templates/dashboard_provision.yaml.tmpl: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: '3fs' 5 | type: file 6 | disableDeletion: false 7 | updateIntervalSeconds: 60 8 | options: 9 | path: /etc/grafana/provisioning/dashboards/3fs.json 10 | foldersFromFilesStructure: true -------------------------------------------------------------------------------- /pkg/grafana/templates/datasource_provision.yaml.tmpl: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | datasources: 4 | - name: Clickhouse 5 | type: grafana-clickhouse-datasource 6 | uid: 3fs_clickhouse_uid 7 | jsonData: 8 | defaultDatabase: {{.CH_Database}} 9 | port: {{.CH_Port}} 10 | host: "{{.CH_Host}}" 11 | username: "{{.CH_Username}}" 12 | tlsSkipVerify: false 13 | allowUiUpdates: true 14 | secureJsonData: 15 | password: "{{.CH_Password}}" -------------------------------------------------------------------------------- /pkg/log/logger.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package log 16 | 17 | import ( 18 | "os" 19 | 20 | "github.com/sirupsen/logrus" 21 | ) 22 | 23 | // defines logger field keys. 24 | const ( 25 | FieldKeyNode = "NODE" 26 | FieldKeyTask = "TASK" 27 | FieldKeyStep = "STEP" 28 | ) 29 | 30 | // Interface is the interface of logger. 31 | type Interface interface { 32 | Subscribe(key, val string) Interface 33 | 34 | Debugf(format string, args ...any) 35 | Infof(format string, args ...any) 36 | Warnf(format string, args ...any) 37 | Warningf(format string, args ...any) 38 | Errorf(format string, args ...any) 39 | 40 | Debug(args ...any) 41 | Info(args ...any) 42 | Warn(args ...any) 43 | Warning(args ...any) 44 | Error(args ...any) 45 | 46 | Debugln(args ...any) 47 | Infoln(args ...any) 48 | Warnln(args ...any) 49 | Warningln(args ...any) 50 | Errorln(args ...any) 51 | } 52 | 53 | var _ Interface = new(logger) 54 | 55 | // Logger is the global logger. 56 | var Logger Interface 57 | 58 | // Logger is the management unit of logging functions. 59 | type logger struct { 60 | *logrus.Logger 61 | fields map[string]any 62 | } 63 | 64 | // Debugf logs a message at level Debug on the standard logger. 65 | func (l *logger) Debugf(format string, args ...any) { 66 | l.WithFields(l.fields).Debugf(format, args...) 67 | } 68 | 69 | // Infof logs a message at level Info on the standard logger. 70 | func (l *logger) Infof(format string, args ...any) { 71 | l.WithFields(l.fields).Infof(format, args...) 72 | } 73 | 74 | // Printf logs a message at level Info on the standard logger. 75 | func (l *logger) Printf(format string, args ...any) { 76 | l.WithFields(l.fields).Printf(format, args...) 77 | } 78 | 79 | // Warnf logs a message at level Warn on the standard logger. 80 | func (l *logger) Warnf(format string, args ...any) { 81 | l.WithFields(l.fields).Warnf(format, args...) 82 | } 83 | 84 | // Warningf logs a message at level Warn on the standard logger. 85 | func (l *logger) Warningf(format string, args ...any) { 86 | l.WithFields(l.fields).Warningf(format, args...) 87 | } 88 | 89 | // Errorf logs a message at level Error on the standard logger. 90 | func (l *logger) Errorf(format string, args ...any) { 91 | l.WithFields(l.fields).Errorf(format, args...) 92 | } 93 | 94 | // Debug logs a message at level Debug on the standard logger. 95 | func (l *logger) Debug(args ...any) { 96 | l.WithFields(l.fields).Debug(args...) 97 | } 98 | 99 | // Info logs a message at level Info on the standard logger. 100 | func (l *logger) Info(args ...any) { 101 | l.WithFields(l.fields).Info(args...) 102 | } 103 | 104 | // Warn logs a message at level Warn on the standard logger. 105 | func (l *logger) Warn(args ...any) { 106 | l.WithFields(l.fields).Warn(args...) 107 | } 108 | 109 | // Warning logs a message at level Warn on the standard logger. 110 | func (l *logger) Warning(args ...any) { 111 | l.WithFields(l.fields).Warning(args...) 112 | } 113 | 114 | // Error logs a message at level Error on the standard logger. 115 | func (l *logger) Error(args ...any) { 116 | l.WithFields(l.fields).Error(args...) 117 | } 118 | 119 | // Debugln logs a message at level Debug on the standard logger. 120 | func (l *logger) Debugln(args ...any) { 121 | l.WithFields(l.fields).Debugln(args...) 122 | } 123 | 124 | // Infoln logs a message at level Info on the standard logger. 125 | func (l *logger) Infoln(args ...any) { 126 | l.WithFields(l.fields).Infoln(args...) 127 | } 128 | 129 | // Warnln logs a message at level Warn on the standard logger. 130 | func (l *logger) Warnln(args ...any) { 131 | l.WithFields(l.fields).Warnln(args...) 132 | } 133 | 134 | // Warningln logs a message at level Warn on the standard logger. 135 | func (l *logger) Warningln(args ...any) { 136 | l.WithFields(l.fields).Warningln(args...) 137 | } 138 | 139 | // Errorln logs a message at level Error on the standard logger. 140 | func (l *logger) Errorln(args ...any) { 141 | l.WithFields(l.fields).Errorln(args...) 142 | } 143 | 144 | // Subscribe adds a field base on current logger and returns a new logger. 145 | func (l *logger) Subscribe(key, val string) Interface { 146 | fields := make(map[string]any, len(l.fields)+1) 147 | for k, v := range l.fields { 148 | fields[k] = v 149 | } 150 | fields[key] = val 151 | return &logger{ 152 | Logger: l.Logger, 153 | fields: fields, 154 | } 155 | } 156 | 157 | // InitLogger initializes the global logger. 158 | func InitLogger(level logrus.Level) { 159 | l := &logrus.Logger{ 160 | Out: os.Stderr, 161 | Formatter: new(logrus.TextFormatter), 162 | Hooks: make(logrus.LevelHooks), 163 | Level: logrus.InfoLevel, 164 | ExitFunc: os.Exit, 165 | ReportCaller: false, 166 | } 167 | l.SetLevel(level) 168 | Logger = &logger{ 169 | Logger: l, 170 | fields: map[string]any{}, 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /pkg/meta/steps.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package meta 16 | 17 | import ( 18 | "embed" 19 | ) 20 | 21 | var ( 22 | //go:embed templates/*.tmpl 23 | templatesFs embed.FS 24 | 25 | // MetaMainAppTomlTmpl is the template content of meta_main_app.toml 26 | MetaMainAppTomlTmpl []byte 27 | // MetaMainLauncherTomlTmpl is the template content of meta_main_launcher.toml 28 | MetaMainLauncherTomlTmpl []byte 29 | // MetaMainTomlTmpl is the template content of meta_main.toml 30 | MetaMainTomlTmpl []byte 31 | ) 32 | 33 | func init() { 34 | var err error 35 | MetaMainAppTomlTmpl, err = templatesFs.ReadFile("templates/meta_main_app.toml.tmpl") 36 | if err != nil { 37 | panic(err) 38 | } 39 | 40 | MetaMainLauncherTomlTmpl, err = templatesFs.ReadFile("templates/meta_main_launcher.toml.tmpl") 41 | if err != nil { 42 | panic(err) 43 | } 44 | 45 | MetaMainTomlTmpl, err = templatesFs.ReadFile("templates/meta_main.toml.tmpl") 46 | if err != nil { 47 | panic(err) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /pkg/meta/tasks.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package meta 16 | 17 | import ( 18 | "path" 19 | 20 | "github.com/open3fs/m3fs/pkg/config" 21 | "github.com/open3fs/m3fs/pkg/log" 22 | "github.com/open3fs/m3fs/pkg/task" 23 | "github.com/open3fs/m3fs/pkg/task/steps" 24 | ) 25 | 26 | const ( 27 | // ServiceName is the name of the meta service. 28 | ServiceName = "meta_main" 29 | serviceType = "META" 30 | ) 31 | 32 | func getServiceWorkDir(workDir string) string { 33 | return path.Join(workDir, "meta") 34 | } 35 | 36 | // CreateMetaServiceTask is a task for creating 3fs meta services. 37 | type CreateMetaServiceTask struct { 38 | task.BaseTask 39 | } 40 | 41 | // Init initializes the task. 42 | func (t *CreateMetaServiceTask) Init(r *task.Runtime, logger log.Interface) { 43 | t.BaseTask.SetName("CreateMetaServiceTask") 44 | t.BaseTask.Init(r, logger) 45 | 46 | workDir := getServiceWorkDir(r.WorkDir) 47 | nodes := make([]config.Node, len(r.Cfg.Services.Meta.Nodes)) 48 | for i, node := range r.Cfg.Services.Meta.Nodes { 49 | nodes[i] = r.Nodes[node] 50 | } 51 | t.SetSteps([]task.StepConfig{ 52 | { 53 | Nodes: []config.Node{nodes[0]}, 54 | NewStep: steps.NewGen3FSNodeIDStepFunc(ServiceName, 100, r.Cfg.Services.Meta.Nodes), 55 | }, 56 | { 57 | Nodes: nodes, 58 | Parallel: true, 59 | NewStep: steps.NewPrepare3FSConfigStepFunc(&steps.Prepare3FSConfigStepSetup{ 60 | Service: ServiceName, 61 | ServiceWorkDir: workDir, 62 | MainAppTomlTmpl: MetaMainAppTomlTmpl, 63 | MainLauncherTomlTmpl: MetaMainLauncherTomlTmpl, 64 | MainTomlTmpl: MetaMainTomlTmpl, 65 | RDMAListenPort: r.Services.Meta.RDMAListenPort, 66 | TCPListenPort: r.Services.Meta.TCPListenPort, 67 | }), 68 | }, 69 | { 70 | Nodes: []config.Node{nodes[0]}, 71 | NewStep: steps.NewUpload3FSMainConfigStepFunc( 72 | config.ImageName3FS, 73 | r.Services.Meta.ContainerName, 74 | ServiceName, 75 | workDir, 76 | serviceType, 77 | ), 78 | }, 79 | { 80 | Nodes: nodes, 81 | Parallel: true, 82 | NewStep: steps.NewRun3FSContainerStepFunc( 83 | &steps.Run3FSContainerStepSetup{ 84 | ImgName: config.ImageName3FS, 85 | ContainerName: r.Services.Meta.ContainerName, 86 | Service: ServiceName, 87 | WorkDir: workDir, 88 | UseRdmaNetwork: true, 89 | }, 90 | ), 91 | }, 92 | }) 93 | } 94 | 95 | // DeleteMetaServiceTask is a task for deleting a meta services. 96 | type DeleteMetaServiceTask struct { 97 | task.BaseTask 98 | } 99 | 100 | // Init initializes the task. 101 | func (t *DeleteMetaServiceTask) Init(r *task.Runtime, logger log.Interface) { 102 | t.BaseTask.SetName("DeleteMetaServiceTask") 103 | t.BaseTask.Init(r, logger) 104 | nodes := make([]config.Node, len(r.Cfg.Services.Meta.Nodes)) 105 | for i, node := range r.Cfg.Services.Meta.Nodes { 106 | nodes[i] = r.Nodes[node] 107 | } 108 | t.SetSteps([]task.StepConfig{ 109 | { 110 | Nodes: nodes, 111 | Parallel: true, 112 | NewStep: steps.NewRm3FSContainerStepFunc( 113 | r.Services.Meta.ContainerName, 114 | ServiceName, 115 | getServiceWorkDir(r.WorkDir)), 116 | }, 117 | }) 118 | } 119 | -------------------------------------------------------------------------------- /pkg/meta/templates/meta_main_app.toml.tmpl: -------------------------------------------------------------------------------- 1 | allow_empty_node_id = true 2 | node_id = {{ .NodeID }} -------------------------------------------------------------------------------- /pkg/meta/templates/meta_main_launcher.toml.tmpl: -------------------------------------------------------------------------------- 1 | allow_dev_version = true 2 | cluster_id = '{{ .ClusterID }}' 3 | 4 | [client] 5 | default_compression_level = 0 6 | default_compression_threshold = '128KB' 7 | default_log_long_running_threshold = '0ns' 8 | default_report_metrics = false 9 | default_send_retry_times = 1 10 | default_timeout = '1s' 11 | enable_rdma_control = false 12 | force_use_tcp = false 13 | 14 | [client.io_worker] 15 | num_event_loop = 1 16 | rdma_connect_timeout = '5s' 17 | read_write_rdma_in_event_thread = false 18 | read_write_tcp_in_event_thread = false 19 | tcp_connect_timeout = '1s' 20 | wait_to_retry_send = '100ms' 21 | 22 | [client.io_worker.connect_concurrency_limiter] 23 | max_concurrency = 4 24 | 25 | [client.io_worker.ibsocket] 26 | buf_ack_batch = 8 27 | buf_signal_batch = 8 28 | buf_size = 16384 29 | drain_timeout = '5s' 30 | drop_connections = 0 31 | event_ack_batch = 128 32 | max_rd_atomic = 16 33 | max_rdma_wr = 128 34 | max_rdma_wr_per_post = 32 35 | max_sge = 1 36 | min_rnr_timer = 1 37 | record_bytes_per_peer = false 38 | record_latency_per_peer = false 39 | retry_cnt = 7 40 | rnr_retry = 0 41 | send_buf_cnt = 32 42 | sl = 0 43 | start_psn = 0 44 | timeout = 14 45 | 46 | [client.io_worker.transport_pool] 47 | max_connections = 1 48 | 49 | [client.processor] 50 | enable_coroutines_pool = true 51 | max_coroutines_num = 256 52 | max_processing_requests_num = 4096 53 | response_compression_level = 1 54 | response_compression_threshold = '128KB' 55 | 56 | [client.rdma_control] 57 | max_concurrent_transmission = 64 58 | 59 | [client.thread_pool] 60 | bg_thread_pool_stratetry = 'SHARED_QUEUE' 61 | collect_stats = false 62 | enable_work_stealing = false 63 | io_thread_pool_stratetry = 'SHARED_QUEUE' 64 | num_bg_threads = 2 65 | num_connect_threads = 2 66 | num_io_threads = 2 67 | num_proc_threads = 2 68 | proc_thread_pool_stratetry = 'SHARED_QUEUE' 69 | 70 | [ib_devices] 71 | allow_no_usable_devices = false 72 | allow_unknown_zone = true 73 | default_network_zone = 'UNKNOWN' 74 | default_pkey_index = 0 75 | default_roce_pkey_index = 0 76 | default_traffic_class = 0 77 | device_filter = [] 78 | fork_safe = true 79 | prefer_ibdevice = true 80 | skip_inactive_ports = true 81 | skip_unusable_device = true 82 | subnets = [] 83 | 84 | [mgmtd_client] 85 | accept_incomplete_routing_info_during_mgmtd_bootstrapping = true 86 | auto_extend_client_session_interval = '10s' 87 | auto_heartbeat_interval = '10s' 88 | auto_refresh_interval = '10s' 89 | enable_auto_extend_client_session = false 90 | enable_auto_heartbeat = true 91 | enable_auto_refresh = true 92 | mgmtd_server_addresses = {{ .MgmtdServerAddresses }} 93 | work_queue_size = 100 -------------------------------------------------------------------------------- /pkg/mgmtd/tasks.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package mgmtd 16 | 17 | import ( 18 | "path" 19 | 20 | "github.com/open3fs/m3fs/pkg/config" 21 | "github.com/open3fs/m3fs/pkg/log" 22 | "github.com/open3fs/m3fs/pkg/task" 23 | "github.com/open3fs/m3fs/pkg/task/steps" 24 | ) 25 | 26 | // ServiceName is the name of the mgmtd service. 27 | const ServiceName = "mgmtd_main" 28 | 29 | func getServiceWorkDir(workDir string) string { 30 | return path.Join(workDir, "mgmtd") 31 | } 32 | 33 | // CreateMgmtdServiceTask is a task for creating 3fs mgmtd services. 34 | type CreateMgmtdServiceTask struct { 35 | task.BaseTask 36 | } 37 | 38 | // Init initializes the task. 39 | func (t *CreateMgmtdServiceTask) Init(r *task.Runtime, logger log.Interface) { 40 | t.BaseTask.SetName("CreateMgmtdServiceTask") 41 | t.BaseTask.Init(r, logger) 42 | nodes := make([]config.Node, len(r.Cfg.Services.Mgmtd.Nodes)) 43 | for i, node := range r.Cfg.Services.Mgmtd.Nodes { 44 | nodes[i] = r.Nodes[node] 45 | } 46 | t.SetSteps([]task.StepConfig{ 47 | { 48 | Nodes: []config.Node{nodes[0]}, 49 | NewStep: steps.NewGen3FSNodeIDStepFunc(ServiceName, 1, r.Cfg.Services.Mgmtd.Nodes), 50 | }, 51 | { 52 | Nodes: []config.Node{nodes[0]}, 53 | NewStep: func() task.Step { return new(genAdminCliConfigStep) }, 54 | }, 55 | { 56 | Nodes: nodes, 57 | Parallel: true, 58 | NewStep: steps.NewPrepare3FSConfigStepFunc(&steps.Prepare3FSConfigStepSetup{ 59 | Service: ServiceName, 60 | ServiceWorkDir: getServiceWorkDir(r.WorkDir), 61 | MainAppTomlTmpl: MgmtdMainAppTomlTmpl, 62 | MainLauncherTomlTmpl: MgmtdMainLauncherTomlTmpl, 63 | MainTomlTmpl: MgmtdMainTomlTmpl, 64 | RDMAListenPort: r.Services.Mgmtd.RDMAListenPort, 65 | TCPListenPort: r.Services.Mgmtd.TCPListenPort, 66 | }), 67 | }, 68 | { 69 | Nodes: []config.Node{nodes[0]}, 70 | NewStep: func() task.Step { return new(initClusterStep) }, 71 | }, 72 | { 73 | Nodes: nodes, 74 | Parallel: true, 75 | NewStep: steps.NewRun3FSContainerStepFunc( 76 | &steps.Run3FSContainerStepSetup{ 77 | ImgName: config.ImageName3FS, 78 | ContainerName: r.Services.Mgmtd.ContainerName, 79 | Service: ServiceName, 80 | WorkDir: getServiceWorkDir(r.WorkDir), 81 | UseRdmaNetwork: true, 82 | }), 83 | }, 84 | { 85 | Nodes: nodes, 86 | Parallel: true, 87 | NewStep: func() task.Step { return new(genAdminCliShellStep) }, 88 | }, 89 | }) 90 | } 91 | 92 | // DeleteMgmtdServiceTask is a task for deleting a mgmtd services. 93 | type DeleteMgmtdServiceTask struct { 94 | task.BaseTask 95 | } 96 | 97 | // Init initializes the task. 98 | func (t *DeleteMgmtdServiceTask) Init(r *task.Runtime, logger log.Interface) { 99 | t.BaseTask.SetName("DeleteMgmtdServiceTask") 100 | t.BaseTask.Init(r, logger) 101 | nodes := make([]config.Node, len(r.Cfg.Services.Mgmtd.Nodes)) 102 | for i, node := range r.Cfg.Services.Mgmtd.Nodes { 103 | nodes[i] = r.Nodes[node] 104 | } 105 | t.SetSteps([]task.StepConfig{ 106 | { 107 | Nodes: nodes, 108 | Parallel: true, 109 | NewStep: steps.NewRm3FSContainerStepFunc( 110 | r.Services.Mgmtd.ContainerName, 111 | ServiceName, 112 | getServiceWorkDir(r.WorkDir)), 113 | }, 114 | }) 115 | } 116 | 117 | // InitUserAndChainTask is a task for initializing user and chain. 118 | type InitUserAndChainTask struct { 119 | task.BaseTask 120 | } 121 | 122 | // Init initializes the task. 123 | func (t *InitUserAndChainTask) Init(r *task.Runtime, logger log.Interface) { 124 | t.BaseTask.SetName("InitUserAndChainTask") 125 | t.BaseTask.Init(r, logger) 126 | nodes := make([]config.Node, len(r.Cfg.Services.Mgmtd.Nodes)) 127 | for i, node := range r.Cfg.Services.Mgmtd.Nodes { 128 | nodes[i] = r.Nodes[node] 129 | } 130 | t.SetSteps([]task.StepConfig{ 131 | { 132 | Nodes: []config.Node{nodes[0]}, 133 | NewStep: func() task.Step { return new(initUserAndChainStep) }, 134 | }, 135 | }) 136 | } 137 | -------------------------------------------------------------------------------- /pkg/mgmtd/templates/admin_cli.sh.tmpl: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker exec -it 3fs-mgmtd /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml --config.mgmtd_client.mgmtd_server_addresses '{{ .MgmtdServerAddresses }}' \'$@\' 4 | -------------------------------------------------------------------------------- /pkg/mgmtd/templates/mgmtd_main.toml.tmpl: -------------------------------------------------------------------------------- 1 | [[common.log.categories]] 2 | categories = [ '.' ] 3 | handlers = [ 'normal', 'err', 'fatal' ] 4 | inherit = true 5 | level = '{{ .LogLevel }}' 6 | propagate = 'NONE' 7 | 8 | [[common.log.handlers]] 9 | async = true 10 | file_path = '/var/log/3fs/mgmtd_main.log' 11 | max_file_size = '100MB' 12 | max_files = 10 13 | name = 'normal' 14 | rotate = true 15 | rotate_on_open = false 16 | start_level = 'NONE' 17 | stream_type = 'STDERR' 18 | writer_type = 'FILE' 19 | 20 | [[common.log.handlers]] 21 | async = false 22 | file_path = '/var/log/3fs/mgmtd_main-err.log' 23 | max_file_size = '100MB' 24 | max_files = 10 25 | name = 'err' 26 | rotate = true 27 | rotate_on_open = false 28 | start_level = 'ERR' 29 | stream_type = 'STDERR' 30 | writer_type = 'FILE' 31 | 32 | [[common.log.handlers]] 33 | async = false 34 | file_path = '/var/log/3fs/mgmtd_main-fatal.log' 35 | max_file_size = '100MB' 36 | max_files = 10 37 | name = 'fatal' 38 | rotate = true 39 | rotate_on_open = false 40 | start_level = 'FATAL' 41 | stream_type = 'STDERR' 42 | writer_type = 'STREAM' 43 | 44 | [common.memory] 45 | prof_active = false 46 | prof_prefix = '' 47 | 48 | [common.monitor] 49 | collect_period = '1s' 50 | num_collectors = 1 51 | 52 | [[common.monitor.reporters]] 53 | type = 'monitor_collector' 54 | 55 | [common.monitor.reporters.monitor_collector] 56 | remote_ip = "{{ .MonitorRemoteIP }}" 57 | 58 | [server.base.independent_thread_pool] 59 | bg_thread_pool_stratetry = 'SHARED_QUEUE' 60 | collect_stats = false 61 | enable_work_stealing = false 62 | io_thread_pool_stratetry = 'SHARED_QUEUE' 63 | num_bg_threads = 2 64 | num_connect_threads = 2 65 | num_io_threads = 2 66 | num_proc_threads = 2 67 | proc_thread_pool_stratetry = 'SHARED_QUEUE' 68 | 69 | [server.base.thread_pool] 70 | bg_thread_pool_stratetry = 'SHARED_QUEUE' 71 | collect_stats = false 72 | enable_work_stealing = false 73 | io_thread_pool_stratetry = 'SHARED_QUEUE' 74 | num_bg_threads = 2 75 | num_connect_threads = 2 76 | num_io_threads = 2 77 | num_proc_threads = 2 78 | proc_thread_pool_stratetry = 'SHARED_QUEUE' 79 | 80 | [[server.base.groups]] 81 | check_connections_interval = '1min' 82 | connection_expiration_time = '1day' 83 | network_type = '{{ .MgmtdProtocol }}' 84 | services = [ 'Mgmtd' ] 85 | use_independent_thread_pool = false 86 | 87 | [server.base.groups.io_worker] 88 | num_event_loop = 1 89 | rdma_connect_timeout = '5s' 90 | read_write_rdma_in_event_thread = false 91 | read_write_tcp_in_event_thread = false 92 | tcp_connect_timeout = '1s' 93 | wait_to_retry_send = '100ms' 94 | 95 | [server.base.groups.io_worker.connect_concurrency_limiter] 96 | max_concurrency = 4 97 | 98 | [server.base.groups.io_worker.ibsocket] 99 | buf_ack_batch = 8 100 | buf_signal_batch = 8 101 | buf_size = 16384 102 | drain_timeout = '5s' 103 | drop_connections = 0 104 | event_ack_batch = 128 105 | max_rd_atomic = 16 106 | max_rdma_wr = 128 107 | max_rdma_wr_per_post = 32 108 | max_sge = 1 109 | min_rnr_timer = 1 110 | record_bytes_per_peer = false 111 | record_latency_per_peer = false 112 | retry_cnt = 7 113 | rnr_retry = 0 114 | send_buf_cnt = 32 115 | sl = 0 116 | start_psn = 0 117 | timeout = 14 118 | 119 | [server.base.groups.io_worker.transport_pool] 120 | max_connections = 1 121 | 122 | [server.base.groups.listener] 123 | domain_socket_index = 1 124 | filter_list = [] 125 | listen_port = {{ .RDMAListenPort }} 126 | listen_queue_depth = 4096 127 | rdma_accept_timeout = '15s' 128 | rdma_listen_ethernet = true 129 | reuse_port = false 130 | 131 | [server.base.groups.processor] 132 | enable_coroutines_pool = true 133 | max_coroutines_num = 256 134 | max_processing_requests_num = 4096 135 | response_compression_level = 1 136 | response_compression_threshold = '128KB' 137 | 138 | [[server.base.groups]] 139 | check_connections_interval = '1min' 140 | connection_expiration_time = '1day' 141 | network_type = 'TCP' 142 | services = [ 'Core' ] 143 | use_independent_thread_pool = true 144 | 145 | [server.base.groups.io_worker] 146 | num_event_loop = 1 147 | rdma_connect_timeout = '5s' 148 | read_write_rdma_in_event_thread = false 149 | read_write_tcp_in_event_thread = false 150 | tcp_connect_timeout = '1s' 151 | wait_to_retry_send = '100ms' 152 | 153 | [server.base.groups.io_worker.connect_concurrency_limiter] 154 | max_concurrency = 4 155 | 156 | [server.base.groups.io_worker.ibsocket] 157 | buf_ack_batch = 8 158 | buf_signal_batch = 8 159 | buf_size = 16384 160 | drain_timeout = '5s' 161 | drop_connections = 0 162 | event_ack_batch = 128 163 | max_rd_atomic = 16 164 | max_rdma_wr = 128 165 | max_rdma_wr_per_post = 32 166 | max_sge = 1 167 | min_rnr_timer = 1 168 | record_bytes_per_peer = false 169 | record_latency_per_peer = false 170 | retry_cnt = 7 171 | rnr_retry = 0 172 | send_buf_cnt = 32 173 | sl = 0 174 | start_psn = 0 175 | timeout = 14 176 | 177 | [server.base.groups.io_worker.transport_pool] 178 | max_connections = 1 179 | 180 | [server.base.groups.listener] 181 | domain_socket_index = 1 182 | filter_list = [] 183 | listen_port = {{ .TCPListenPort }} 184 | listen_queue_depth = 4096 185 | rdma_accept_timeout = '15s' 186 | rdma_listen_ethernet = true 187 | reuse_port = false 188 | 189 | [server.base.groups.processor] 190 | enable_coroutines_pool = true 191 | max_coroutines_num = 256 192 | max_processing_requests_num = 4096 193 | response_compression_level = 1 194 | response_compression_threshold = '128KB' 195 | 196 | [server.service] 197 | allow_heartbeat_from_unregistered = true 198 | authenticate = false 199 | bootstrapping_length = '2min' 200 | bump_routing_info_version_interval = '5s' 201 | check_status_interval = '10s' 202 | client_session_timeout = '20min' 203 | enable_routinginfo_cache = true 204 | extend_lease_check_release_version = true 205 | extend_lease_interval = '10s' 206 | heartbeat_fail_interval = '1min' 207 | heartbeat_ignore_stale_targets = true 208 | heartbeat_ignore_unknown_targets = false 209 | heartbeat_timestamp_valid_window = '30s' 210 | lease_length = '1min' 211 | new_chain_bootstrap_interval = '2min' 212 | only_accept_client_uuid = false 213 | retry_times_on_txn_errors = -1 214 | send_heartbeat = true 215 | send_heartbeat_interval = '10s' 216 | suspicious_lease_interval = '20s' 217 | target_info_load_interval = '1s' 218 | target_info_persist_batch = 1000 219 | target_info_persist_interval = '1s' 220 | try_adjust_target_order_as_preferred = false 221 | update_chains_interval = '1s' 222 | update_metrics_interval = '1s' 223 | validate_lease_on_write = true 224 | 225 | [server.service.retry_transaction] 226 | max_backoff = '1s' 227 | max_retry_count = 10 228 | 229 | [server.service.user_cache] 230 | buckets = 127 231 | exist_ttl = '5min' 232 | inexist_ttl = '10s' -------------------------------------------------------------------------------- /pkg/mgmtd/templates/mgmtd_main_app.toml.tmpl: -------------------------------------------------------------------------------- 1 | allow_empty_node_id = true 2 | node_id = {{ .NodeID }} -------------------------------------------------------------------------------- /pkg/mgmtd/templates/mgmtd_main_launcher.toml.tmpl: -------------------------------------------------------------------------------- 1 | allow_dev_version = true 2 | cluster_id = '{{ .ClusterID }}' 3 | use_memkv = false 4 | 5 | [fdb] 6 | casual_read_risky = false 7 | clusterFile = '/opt/3fs/etc/fdb.cluster' 8 | default_backoff = 0 9 | enableMultipleClient = false 10 | externalClientDir = '' 11 | externalClientPath = '' 12 | multipleClientThreadNum = 4 13 | readonly = false 14 | trace_file = '' 15 | trace_format = 'json' 16 | 17 | [ib_devices] 18 | allow_no_usable_devices = false 19 | allow_unknown_zone = true 20 | default_network_zone = 'UNKNOWN' 21 | default_pkey_index = 0 22 | default_roce_pkey_index = 0 23 | default_traffic_class = 0 24 | device_filter = [] 25 | fork_safe = true 26 | prefer_ibdevice = true 27 | skip_inactive_ports = true 28 | skip_unusable_device = true 29 | subnets = [] 30 | 31 | [kv_engine] 32 | use_memkv = false 33 | 34 | [kv_engine.fdb] 35 | casual_read_risky = false 36 | clusterFile = '' 37 | default_backoff = 0 38 | enableMultipleClient = false 39 | externalClientDir = '' 40 | externalClientPath = '' 41 | multipleClientThreadNum = 4 42 | readonly = false 43 | trace_file = '' 44 | trace_format = 'json' -------------------------------------------------------------------------------- /pkg/monitor/steps.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package monitor 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "embed" 21 | "os" 22 | "path" 23 | "path/filepath" 24 | "strconv" 25 | "text/template" 26 | 27 | "github.com/open3fs/m3fs/pkg/common" 28 | "github.com/open3fs/m3fs/pkg/config" 29 | "github.com/open3fs/m3fs/pkg/errors" 30 | "github.com/open3fs/m3fs/pkg/external" 31 | "github.com/open3fs/m3fs/pkg/task" 32 | ) 33 | 34 | var ( 35 | //go:embed templates/* 36 | templatesFs embed.FS 37 | 38 | // MonitorCollectorMainTmpl is the template content of monitor_collector_main.toml 39 | MonitorCollectorMainTmpl []byte 40 | ) 41 | 42 | func init() { 43 | var err error 44 | MonitorCollectorMainTmpl, err = templatesFs.ReadFile("templates/monitor_collector_main.tmpl") 45 | if err != nil { 46 | panic(err) 47 | } 48 | } 49 | 50 | func getServiceWorkDir(workDir string) string { 51 | return path.Join(workDir, "monitor") 52 | } 53 | 54 | type genMonitorConfigStep struct { 55 | task.BaseStep 56 | } 57 | 58 | func (s *genMonitorConfigStep) Execute(ctx context.Context) error { 59 | tempDir, err := s.Runtime.LocalEm.FS.MkdirTemp(ctx, os.TempDir(), "3fs-monitor") 60 | if err != nil { 61 | return errors.Trace(err) 62 | } 63 | s.Runtime.Store(task.RuntimeMonitorTmpDirKey, tempDir) 64 | 65 | fileName := "monitor_collector_main.toml" 66 | tmpl, err := template.New(fileName).Parse(string(MonitorCollectorMainTmpl)) 67 | if err != nil { 68 | return errors.Annotate(err, "parse monitor_collector_main.toml template") 69 | } 70 | var clickhouseHost string 71 | for _, clickhouseNode := range s.Runtime.Services.Clickhouse.Nodes { 72 | for _, node := range s.Runtime.Nodes { 73 | if node.Name == clickhouseNode { 74 | clickhouseHost = node.Host 75 | } 76 | } 77 | } 78 | data := new(bytes.Buffer) 79 | err = tmpl.Execute(data, map[string]string{ 80 | "Port": strconv.Itoa(s.Runtime.Services.Monitor.Port), 81 | "ClickhouseDb": s.Runtime.Services.Clickhouse.Db, 82 | "ClickhouseHost": clickhouseHost, 83 | "ClickhousePassword": s.Runtime.Services.Clickhouse.Password, 84 | "ClickhousePort": strconv.Itoa(s.Runtime.Services.Clickhouse.TCPPort), 85 | "ClickhouseUser": s.Runtime.Services.Clickhouse.User, 86 | }) 87 | if err != nil { 88 | return errors.Annotate(err, "write monitor_collector_main.toml") 89 | } 90 | configPath := filepath.Join(tempDir, fileName) 91 | if err = s.Runtime.LocalEm.FS.WriteFile(configPath, data.Bytes(), 0644); err != nil { 92 | return errors.Trace(err) 93 | } 94 | 95 | return nil 96 | } 97 | 98 | type runContainerStep struct { 99 | task.BaseStep 100 | } 101 | 102 | func (s *runContainerStep) Execute(ctx context.Context) error { 103 | workDir := getServiceWorkDir(s.Runtime.WorkDir) 104 | etcDir := path.Join(workDir, "etc") 105 | err := s.Em.FS.MkdirAll(ctx, etcDir) 106 | if err != nil { 107 | return errors.Annotatef(err, "mkdir %s", etcDir) 108 | } 109 | localConfigDir, _ := s.Runtime.Load(task.RuntimeMonitorTmpDirKey) 110 | localConfigFile := path.Join(localConfigDir.(string), "monitor_collector_main.toml") 111 | remoteConfigFile := path.Join(etcDir, "monitor_collector_main.toml") 112 | if err := s.Em.Runner.Scp(ctx, localConfigFile, remoteConfigFile); err != nil { 113 | return errors.Annotatef(err, "scp monitor_collector_main.toml") 114 | } 115 | logDir := path.Join(workDir, "log") 116 | err = s.Em.FS.MkdirAll(ctx, logDir) 117 | if err != nil { 118 | return errors.Annotatef(err, "mkdir %s", logDir) 119 | } 120 | 121 | img, err := s.Runtime.Cfg.Images.GetImage(config.ImageName3FS) 122 | if err != nil { 123 | return errors.Trace(err) 124 | } 125 | args := &external.RunArgs{ 126 | Image: img, 127 | Name: &s.Runtime.Services.Monitor.ContainerName, 128 | HostNetwork: true, 129 | Privileged: common.Pointer(true), 130 | Detach: common.Pointer(true), 131 | Volumes: []*external.VolumeArgs{ 132 | { 133 | Source: "/dev", 134 | Target: "/dev", 135 | }, 136 | { 137 | Source: etcDir, 138 | Target: "/opt/3fs/etc", 139 | }, 140 | { 141 | Source: logDir, 142 | Target: "/var/log/3fs", 143 | }, 144 | }, 145 | Command: []string{ 146 | "/opt/3fs/bin/monitor_collector_main", 147 | "--cfg", 148 | "/opt/3fs/etc/monitor_collector_main.toml", 149 | }, 150 | } 151 | if err := s.GetErdmaSoPath(ctx); err != nil { 152 | return errors.Trace(err) 153 | } 154 | args.Volumes = append(args.Volumes, s.GetRdmaVolumes()...) 155 | _, err = s.Em.Docker.Run(ctx, args) 156 | if err != nil { 157 | return errors.Trace(err) 158 | } 159 | 160 | s.Logger.Infof("Started monitor container %s successfully", 161 | s.Runtime.Services.Monitor.ContainerName) 162 | return nil 163 | } 164 | 165 | type rmContainerStep struct { 166 | task.BaseStep 167 | } 168 | 169 | func (s *rmContainerStep) Execute(ctx context.Context) error { 170 | containerName := s.Runtime.Services.Monitor.ContainerName 171 | s.Logger.Infof("Removing monitor container %s", containerName) 172 | _, err := s.Em.Docker.Rm(ctx, containerName, true) 173 | if err != nil { 174 | return errors.Trace(err) 175 | } 176 | workDir := getServiceWorkDir(s.Runtime.WorkDir) 177 | etcDir := path.Join(workDir, "etc") 178 | _, err = s.Em.Runner.Exec(ctx, "rm", "-rf", etcDir) 179 | if err != nil { 180 | return errors.Annotatef(err, "rm %s", etcDir) 181 | } 182 | s.Logger.Infof("Removed monitor container etc dir %s", etcDir) 183 | 184 | logDir := path.Join(workDir, "log") 185 | _, err = s.Em.Runner.Exec(ctx, "rm", "-rf", logDir) 186 | if err != nil { 187 | return errors.Annotatef(err, "rm %s", logDir) 188 | } 189 | s.Logger.Infof("Removed monitor container log dir %s", logDir) 190 | 191 | s.Logger.Infof("Removed monitor container %s successfully", containerName) 192 | return nil 193 | } 194 | -------------------------------------------------------------------------------- /pkg/monitor/steps_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package monitor 16 | 17 | import ( 18 | "os" 19 | "testing" 20 | 21 | "github.com/stretchr/testify/mock" 22 | "github.com/stretchr/testify/suite" 23 | 24 | "github.com/open3fs/m3fs/pkg/common" 25 | "github.com/open3fs/m3fs/pkg/config" 26 | "github.com/open3fs/m3fs/pkg/external" 27 | "github.com/open3fs/m3fs/pkg/task" 28 | ttask "github.com/open3fs/m3fs/tests/task" 29 | ) 30 | 31 | var suiteRun = suite.Run 32 | 33 | func TestGenMonitorConfigStep(t *testing.T) { 34 | suiteRun(t, &genMonitorConfigStepSuite{}) 35 | } 36 | 37 | type genMonitorConfigStepSuite struct { 38 | ttask.StepSuite 39 | 40 | step *genMonitorConfigStep 41 | } 42 | 43 | func (s *genMonitorConfigStepSuite) SetupTest() { 44 | s.StepSuite.SetupTest() 45 | 46 | s.step = &genMonitorConfigStep{} 47 | s.SetupRuntime() 48 | s.step.Init(s.Runtime, s.MockEm, config.Node{}, s.Logger) 49 | } 50 | 51 | func (s *genMonitorConfigStepSuite) Test() { 52 | s.MockLocalFS.On("MkdirTemp", os.TempDir(), "3fs-monitor").Return("/tmp/3fs-monitor.xxx", nil) 53 | s.MockLocalFS.On("WriteFile", "/tmp/3fs-monitor.xxx/monitor_collector_main.toml", 54 | mock.AnythingOfType("[]uint8"), os.FileMode(0644)).Return(nil) 55 | 56 | s.NoError(s.step.Execute(s.Ctx())) 57 | 58 | tmpDirValue, ok := s.Runtime.Load(task.RuntimeMonitorTmpDirKey) 59 | s.True(ok) 60 | tmpDir := tmpDirValue.(string) 61 | s.Equal("/tmp/3fs-monitor.xxx", tmpDir) 62 | } 63 | 64 | func TestRunContainerStep(t *testing.T) { 65 | suiteRun(t, &runContainerStepSuite{}) 66 | } 67 | 68 | type runContainerStepSuite struct { 69 | ttask.StepSuite 70 | 71 | step *runContainerStep 72 | } 73 | 74 | func (s *runContainerStepSuite) SetupTest() { 75 | s.StepSuite.SetupTest() 76 | 77 | s.step = &runContainerStep{} 78 | s.SetupRuntime() 79 | s.step.Init(s.Runtime, s.MockEm, config.Node{}, s.Logger) 80 | s.Runtime.Store(task.RuntimeMonitorTmpDirKey, "/tmp/3f-monitor.xxx") 81 | } 82 | 83 | func (s *runContainerStepSuite) Test() { 84 | etcDir := "/root/3fs/monitor/etc" 85 | logDir := "/root/3fs/monitor/log" 86 | s.MockFS.On("MkdirAll", etcDir).Return(nil) 87 | s.MockFS.On("MkdirAll", logDir).Return(nil) 88 | s.MockRunner.On("Scp", "/tmp/3f-monitor.xxx/monitor_collector_main.toml", 89 | "/root/3fs/monitor/etc/monitor_collector_main.toml").Return(nil) 90 | img, err := s.Runtime.Cfg.Images.GetImage(config.ImageName3FS) 91 | s.NoError(err) 92 | args := &external.RunArgs{ 93 | Image: img, 94 | Name: common.Pointer("3fs-monitor"), 95 | HostNetwork: true, 96 | Privileged: common.Pointer(true), 97 | Detach: common.Pointer(true), 98 | Volumes: []*external.VolumeArgs{ 99 | { 100 | Source: "/dev", 101 | Target: "/dev", 102 | }, 103 | { 104 | Source: etcDir, 105 | Target: "/opt/3fs/etc", 106 | }, 107 | { 108 | Source: logDir, 109 | Target: "/var/log/3fs", 110 | }, 111 | }, 112 | Command: []string{ 113 | "/opt/3fs/bin/monitor_collector_main", 114 | "--cfg", 115 | "/opt/3fs/etc/monitor_collector_main.toml", 116 | }, 117 | } 118 | s.Runtime.Store(s.step.GetErdmaSoPathKey(), 119 | "/usr/lib/x86_64-linux-gnu/libibverbs/liberdma-rdmav34.so") 120 | args.Volumes = append(args.Volumes, s.step.GetRdmaVolumes()...) 121 | s.MockDocker.On("Run", args).Return("", nil) 122 | 123 | s.NoError(s.step.Execute(s.Ctx())) 124 | 125 | s.MockRunner.AssertExpectations(s.T()) 126 | s.MockFS.AssertExpectations(s.T()) 127 | s.MockDocker.AssertExpectations(s.T()) 128 | } 129 | 130 | func TestRmContainerStep(t *testing.T) { 131 | suiteRun(t, &rmContainerStepSuite{}) 132 | } 133 | 134 | type rmContainerStepSuite struct { 135 | ttask.StepSuite 136 | 137 | step *rmContainerStep 138 | etcDir string 139 | logDir string 140 | } 141 | 142 | func (s *rmContainerStepSuite) SetupTest() { 143 | s.StepSuite.SetupTest() 144 | 145 | s.step = &rmContainerStep{} 146 | s.SetupRuntime() 147 | s.step.Init(s.Runtime, s.MockEm, config.Node{}, s.Logger) 148 | s.etcDir = "/root/3fs/monitor/etc" 149 | s.logDir = "/root/3fs/monitor/log" 150 | } 151 | 152 | func (s *rmContainerStepSuite) TestRmContainerStep() { 153 | s.MockDocker.On("Rm", s.Cfg.Services.Monitor.ContainerName, true).Return("", nil) 154 | s.MockRunner.On("Exec", "rm", []string{"-rf", s.etcDir}).Return("", nil) 155 | s.MockRunner.On("Exec", "rm", []string{"-rf", s.logDir}).Return("", nil) 156 | 157 | s.NoError(s.step.Execute(s.Ctx())) 158 | 159 | s.MockRunner.AssertExpectations(s.T()) 160 | s.MockDocker.AssertExpectations(s.T()) 161 | } 162 | -------------------------------------------------------------------------------- /pkg/monitor/tasks.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package monitor 16 | 17 | import ( 18 | "github.com/open3fs/m3fs/pkg/config" 19 | "github.com/open3fs/m3fs/pkg/log" 20 | "github.com/open3fs/m3fs/pkg/task" 21 | "github.com/open3fs/m3fs/pkg/task/steps" 22 | ) 23 | 24 | // CreateMonitorTask is a task for creating a 3fs monitor. 25 | type CreateMonitorTask struct { 26 | task.BaseTask 27 | } 28 | 29 | // Init initializes the task. 30 | func (t *CreateMonitorTask) Init(r *task.Runtime, logger log.Interface) { 31 | t.BaseTask.SetName("CreateMonitorTask") 32 | t.BaseTask.Init(r, logger) 33 | nodes := make([]config.Node, len(r.Cfg.Services.Monitor.Nodes)) 34 | for i, node := range r.Cfg.Services.Monitor.Nodes { 35 | nodes[i] = r.Nodes[node] 36 | } 37 | t.SetSteps([]task.StepConfig{ 38 | { 39 | Nodes: []config.Node{nodes[0]}, 40 | NewStep: func() task.Step { return new(genMonitorConfigStep) }, 41 | }, 42 | { 43 | Nodes: []config.Node{nodes[0]}, 44 | NewStep: func() task.Step { return new(runContainerStep) }, 45 | }, 46 | { 47 | Nodes: []config.Node{nodes[0]}, 48 | NewStep: steps.NewCleanupLocalStepFunc(task.RuntimeMonitorTmpDirKey), 49 | }, 50 | }) 51 | } 52 | 53 | // DeleteMonitorTask is a task for deleting a 3fs monitor. 54 | type DeleteMonitorTask struct { 55 | task.BaseTask 56 | } 57 | 58 | // Init initializes the task. 59 | func (t *DeleteMonitorTask) Init(r *task.Runtime, logger log.Interface) { 60 | t.BaseTask.SetName("DeleteMonitorTask") 61 | t.BaseTask.Init(r, logger) 62 | nodes := make([]config.Node, len(r.Cfg.Services.Monitor.Nodes)) 63 | for i, node := range r.Cfg.Services.Monitor.Nodes { 64 | nodes[i] = r.Nodes[node] 65 | } 66 | t.SetSteps([]task.StepConfig{ 67 | { 68 | Nodes: nodes, 69 | NewStep: func() task.Step { return new(rmContainerStep) }, 70 | }, 71 | }) 72 | } 73 | -------------------------------------------------------------------------------- /pkg/monitor/templates/monitor_collector_main.tmpl: -------------------------------------------------------------------------------- 1 | [common] 2 | cluster_id = '' 3 | 4 | [common.ib_devices] 5 | allow_unknown_zone = true 6 | default_network_zone = 'UNKNOWN' 7 | device_filter = [] 8 | subnets = [] 9 | allow_no_usable_devices = false 10 | 11 | [[common.log.categories]] 12 | categories = [ '.' ] 13 | handlers = [ 'normal', 'err', 'fatal' ] 14 | inherit = true 15 | level = 'INFO' 16 | propagate = 'NONE' 17 | 18 | [[common.log.handlers]] 19 | async = true 20 | file_path = '/var/log/3fs/monitor_collector_main.log' 21 | max_file_size = '100MB' 22 | max_files = 10 23 | name = 'normal' 24 | rotate = true 25 | rotate_on_open = false 26 | start_level = 'NONE' 27 | stream_type = 'STDERR' 28 | writer_type = 'FILE' 29 | 30 | [[common.log.handlers]] 31 | async = false 32 | file_path = '/var/log/3fs/monitor_collector_main-err.log' 33 | max_file_size = '100MB' 34 | max_files = 10 35 | name = 'err' 36 | rotate = true 37 | rotate_on_open = false 38 | start_level = 'ERR' 39 | stream_type = 'STDERR' 40 | writer_type = 'FILE' 41 | 42 | [[common.log.handlers]] 43 | async = false 44 | file_path = '/var/log/3fs/monitor_collector_main-fatal.log' 45 | max_file_size = '100MB' 46 | max_files = 10 47 | name = 'fatal' 48 | rotate = true 49 | rotate_on_open = false 50 | start_level = 'FATAL' 51 | stream_type = 'STDERR' 52 | writer_type = 'STREAM' 53 | 54 | [server.base.independent_thread_pool] 55 | bg_thread_pool_stratetry = 'SHARED_QUEUE' 56 | collect_stats = false 57 | enable_work_stealing = false 58 | io_thread_pool_stratetry = 'SHARED_QUEUE' 59 | num_bg_threads = 2 60 | num_connect_threads = 2 61 | num_io_threads = 2 62 | num_proc_threads = 2 63 | proc_thread_pool_stratetry = 'SHARED_QUEUE' 64 | 65 | [server.base.thread_pool] 66 | bg_thread_pool_stratetry = 'SHARED_QUEUE' 67 | collect_stats = false 68 | enable_work_stealing = false 69 | io_thread_pool_stratetry = 'SHARED_QUEUE' 70 | num_bg_threads = 2 71 | num_connect_threads = 2 72 | num_io_threads = 2 73 | num_proc_threads = 2 74 | proc_thread_pool_stratetry = 'SHARED_QUEUE' 75 | 76 | [[server.base.groups]] 77 | #default_timeout = '1s' 78 | #drop_connections_interval = '1h' 79 | network_type = 'TCP' 80 | services = [ 'MonitorCollector' ] 81 | use_independent_thread_pool = false 82 | 83 | [server.base.groups.io_worker] 84 | num_event_loop = 1 85 | rdma_connect_timeout = '5s' 86 | read_write_rdma_in_event_thread = false 87 | read_write_tcp_in_event_thread = false 88 | tcp_connect_timeout = '1s' 89 | wait_to_retry_send = '100ms' 90 | 91 | [server.base.groups.io_worker.ibsocket] 92 | buf_ack_batch = 8 93 | buf_signal_batch = 8 94 | buf_size = 16384 95 | drop_connections = 0 96 | event_ack_batch = 128 97 | #gid_index = 0 98 | max_rd_atomic = 16 99 | max_rdma_wr = 128 100 | max_rdma_wr_per_post = 32 101 | max_sge = 1 102 | min_rnr_timer = 1 103 | pkey_index = 0 104 | record_bytes_per_peer = false 105 | record_latency_per_peer = false 106 | retry_cnt = 7 107 | rnr_retry = 0 108 | send_buf_cnt = 32 109 | sl = 0 110 | start_psn = 0 111 | timeout = 14 112 | traffic_class = 0 113 | 114 | [server.base.groups.io_worker.transport_pool] 115 | max_connections = 1 116 | 117 | [server.base.groups.listener] 118 | filter_list = [] 119 | listen_port = {{ .Port }} 120 | listen_queue_depth = 4096 121 | rdma_listen_ethernet = true 122 | reuse_port = false 123 | 124 | [server.base.groups.processor] 125 | enable_coroutines_pool = true 126 | max_coroutines_num = 256 127 | max_processing_requests_num = 4096 128 | 129 | [server.monitor_collector] 130 | batch_commit_size = 4096 131 | conn_threads = 32 132 | queue_capacity = 204800 133 | 134 | [server.monitor_collector.reporter] 135 | type = 'clickhouse' 136 | 137 | [server.monitor_collector.reporter.clickhouse] 138 | db = '{{ .ClickhouseDb }}' 139 | host = '{{ .ClickhouseHost }}' 140 | passwd = '{{ .ClickhousePassword }}' 141 | port = '{{ .ClickhousePort }}' 142 | user = '{{ .ClickhouseUser }}' 143 | -------------------------------------------------------------------------------- /pkg/network/speed.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package network 16 | 17 | import ( 18 | "context" 19 | "regexp" 20 | "strings" 21 | 22 | "github.com/open3fs/m3fs/pkg/config" 23 | "github.com/open3fs/m3fs/pkg/errors" 24 | "github.com/open3fs/m3fs/pkg/external" 25 | ) 26 | 27 | var ( 28 | ibSpeedPattern = regexp.MustCompile(`rate:\s+(\d+)\s+Gb/sec`) 29 | ethSpeedPattern = regexp.MustCompile(`Speed:\s+(\d+)\s*([GMK]b/?s)`) 30 | ) 31 | 32 | // GetSpeed returns the speed of the network interface 33 | func GetSpeed(ctx context.Context, runner external.RunnerInterface, networkType config.NetworkType) string { 34 | 35 | if networkType == config.NetworkTypeIB { 36 | speed, _ := getIBNetworkSpeed(ctx, runner) 37 | if speed != "" { 38 | return speed 39 | } 40 | } 41 | 42 | speed, _ := getEthernetSpeed(ctx, runner) 43 | if speed != "" { 44 | return speed 45 | } 46 | 47 | if speed == "" || speed == "Unknown!" { 48 | switch networkType { 49 | case config.NetworkTypeIB: 50 | speed = "50 Gbps" 51 | case config.NetworkTypeRDMA: 52 | speed = "100 Gbps" 53 | default: 54 | speed = "10 Gbps" 55 | } 56 | } 57 | 58 | return speed 59 | } 60 | 61 | func getIBNetworkSpeed(ctx context.Context, runner external.RunnerInterface) (string, error) { 62 | output, err := runner.Exec(ctx, "ibstatus") 63 | if err != nil { 64 | return "", errors.Annotate(err, "ibstatus") 65 | } 66 | 67 | matches := ibSpeedPattern.FindStringSubmatch(output) 68 | if len(matches) > 1 { 69 | return matches[1] + " Gb/sec", nil 70 | } 71 | 72 | return "", nil 73 | } 74 | 75 | // getEthernetSpeed returns the Ethernet network speed 76 | func getEthernetSpeed(ctx context.Context, runner external.RunnerInterface) (string, error) { 77 | interfaceName, err := getDefaultInterface(ctx, runner) 78 | if err != nil { 79 | return "", errors.Annotate(err, "get default interface") 80 | } 81 | if interfaceName == "" { 82 | return "", errors.New("no default interface found") 83 | } 84 | 85 | speed, err := getInterfaceSpeed(ctx, runner, interfaceName) 86 | if err != nil { 87 | return "", errors.Annotatef(err, "get interface speed for %s", interfaceName) 88 | } 89 | return speed, nil 90 | } 91 | 92 | // getDefaultInterface returns the default network interface 93 | func getDefaultInterface(ctx context.Context, runner external.RunnerInterface) (string, error) { 94 | output, err := runner.Exec(ctx, "ip route | grep default | awk '{print $5}'") 95 | if err != nil { 96 | return "", errors.Annotate(err, "failed to get default interface") 97 | } 98 | 99 | interfaceName := strings.TrimSpace(output) 100 | if interfaceName == "" { 101 | return "", errors.New("no default interface found") 102 | } 103 | return interfaceName, nil 104 | } 105 | 106 | // getInterfaceSpeed returns the speed of a network interface 107 | func getInterfaceSpeed(ctx context.Context, runner external.RunnerInterface, interfaceName string) (string, error) { 108 | if interfaceName == "" { 109 | return "", errors.New("empty interface name") 110 | } 111 | 112 | output, err := runner.Exec(ctx, "ethtool", interfaceName) 113 | if err != nil { 114 | return "", errors.Annotatef(err, "failed to get interface speed for %s", interfaceName) 115 | } 116 | 117 | matches := ethSpeedPattern.FindStringSubmatch(output) 118 | if len(matches) > 2 { 119 | return strings.TrimSpace(matches[1] + " " + matches[2]), nil 120 | } 121 | 122 | return "", errors.Errorf("no speed found in ethtool output for interface %s", interfaceName) 123 | } 124 | -------------------------------------------------------------------------------- /pkg/network/tasks.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package network 16 | 17 | import ( 18 | "github.com/open3fs/m3fs/pkg/config" 19 | "github.com/open3fs/m3fs/pkg/log" 20 | "github.com/open3fs/m3fs/pkg/task" 21 | ) 22 | 23 | // PrepareNetworkTask is a task for preparing network for a new node. 24 | type PrepareNetworkTask struct { 25 | task.BaseTask 26 | } 27 | 28 | // Init initializes the task. 29 | func (t *PrepareNetworkTask) Init(r *task.Runtime, logger log.Interface) { 30 | t.BaseTask.SetName("PrepareNetworkTask") 31 | t.BaseTask.Init(r, logger) 32 | nodes := r.Cfg.Nodes 33 | 34 | steps := []task.StepConfig{} 35 | switch r.Cfg.NetworkType { 36 | case config.NetworkTypeRXE: 37 | rxeSteps := []task.StepConfig{ 38 | { 39 | Nodes: nodes, 40 | Parallel: true, 41 | NewStep: func() task.Step { return new(installRdmaPackageStep) }, 42 | }, 43 | { 44 | Nodes: nodes, 45 | Parallel: true, 46 | NewStep: func() task.Step { return new(loadRdmaRxeModuleStep) }, 47 | }, 48 | { 49 | Nodes: nodes, 50 | Parallel: true, 51 | NewStep: func() task.Step { return new(createRdmaRxeLinkStep) }, 52 | }, 53 | } 54 | steps = append(steps, rxeSteps...) 55 | case config.NetworkTypeERDMA: 56 | erdmaSteps := []task.StepConfig{ 57 | { 58 | Nodes: nodes, 59 | Parallel: true, 60 | NewStep: func() task.Step { return new(loadErdmaModuleStep) }, 61 | }, 62 | } 63 | steps = append(steps, erdmaSteps...) 64 | } 65 | if r.Cfg.NetworkType != config.NetworkTypeRDMA { 66 | steps = append(steps, task.StepConfig{ 67 | Nodes: nodes, 68 | NewStep: func() task.Step { return new(genIbdev2netdevScriptStep) }, 69 | }) 70 | } 71 | t.SetSteps(steps) 72 | } 73 | 74 | // DeleteNetworkTask is a task for deleting configration added on setup network. 75 | type DeleteNetworkTask struct { 76 | task.BaseTask 77 | } 78 | 79 | // Init initializes the task. 80 | func (t *DeleteNetworkTask) Init(r *task.Runtime, logger log.Interface) { 81 | t.BaseTask.SetName("DeleteNetworkTask") 82 | t.BaseTask.Init(r, logger) 83 | nodes := r.Cfg.Nodes 84 | 85 | steps := []task.StepConfig{ 86 | { 87 | Nodes: nodes, 88 | Parallel: true, 89 | NewStep: func() task.Step { return new(deleteIbdev2netdevScriptStep) }, 90 | }, 91 | } 92 | switch r.Cfg.NetworkType { 93 | case config.NetworkTypeRXE: 94 | rxeSteps := []task.StepConfig{ 95 | { 96 | Nodes: nodes, 97 | Parallel: true, 98 | NewStep: func() task.Step { return new(deleteRdmaRxeLinkScriptStep) }, 99 | }, 100 | } 101 | steps = append(steps, rxeSteps...) 102 | case config.NetworkTypeERDMA: 103 | // TODO 104 | } 105 | t.SetSteps(steps) 106 | } 107 | -------------------------------------------------------------------------------- /pkg/storage/tasks.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package storage 16 | 17 | import ( 18 | "embed" 19 | "fmt" 20 | "path" 21 | "strconv" 22 | "strings" 23 | 24 | "github.com/open3fs/m3fs/pkg/config" 25 | "github.com/open3fs/m3fs/pkg/external" 26 | "github.com/open3fs/m3fs/pkg/log" 27 | "github.com/open3fs/m3fs/pkg/task" 28 | "github.com/open3fs/m3fs/pkg/task/steps" 29 | ) 30 | 31 | var ( 32 | //go:embed templates/*.tmpl 33 | templatesFs embed.FS 34 | 35 | // StorageMainAppTomlTmpl is the template content of storage_main_app.toml 36 | StorageMainAppTomlTmpl []byte 37 | // StorageMainLauncherTomlTmpl is the template content of storage_main_launcher.toml 38 | StorageMainLauncherTomlTmpl []byte 39 | // StorageMainTomlTmpl is the template content of storage_main.toml 40 | StorageMainTomlTmpl []byte 41 | // DiskToolScriptTmpl is the template content of disk_tool.sh 42 | DiskToolScriptTmpl []byte 43 | ) 44 | 45 | func init() { 46 | var err error 47 | StorageMainAppTomlTmpl, err = templatesFs.ReadFile("templates/storage_main_app.toml.tmpl") 48 | if err != nil { 49 | panic(err) 50 | } 51 | 52 | StorageMainLauncherTomlTmpl, err = templatesFs.ReadFile("templates/storage_main_launcher.toml.tmpl") 53 | if err != nil { 54 | panic(err) 55 | } 56 | 57 | StorageMainTomlTmpl, err = templatesFs.ReadFile("templates/storage_main.toml.tmpl") 58 | if err != nil { 59 | panic(err) 60 | } 61 | 62 | DiskToolScriptTmpl, err = templatesFs.ReadFile("templates/disk_tool.sh.tmpl") 63 | if err != nil { 64 | panic(err) 65 | } 66 | } 67 | 68 | func makeTargetPaths(diskNum int) string { 69 | targets := make([]string, diskNum) 70 | for i := 0; i < diskNum; i++ { 71 | targets[i] = fmt.Sprintf(`"%s"`, 72 | path.Join("/mnt", "3fsdata", "data"+strconv.Itoa(i), "3fs")) 73 | } 74 | 75 | return fmt.Sprintf("[%s]", strings.Join(targets, ",")) 76 | } 77 | 78 | const ( 79 | // ServiceName is the name of the storage service. 80 | ServiceName = "storage_main" 81 | serviceType = "STORAGE" 82 | ) 83 | 84 | func getServiceWorkDir(workDir string) string { 85 | return path.Join(workDir, "storage") 86 | } 87 | 88 | // CreateStorageServiceTask is a task for creating 3fs storage services. 89 | type CreateStorageServiceTask struct { 90 | task.BaseTask 91 | } 92 | 93 | // Init initializes the task. 94 | func (t *CreateStorageServiceTask) Init(r *task.Runtime, logger log.Interface) { 95 | t.BaseTask.SetName("CreateStorageServiceTask") 96 | t.BaseTask.Init(r, logger) 97 | 98 | storage := r.Cfg.Services.Storage 99 | workDir := getServiceWorkDir(r.WorkDir) 100 | nodes := make([]config.Node, len(storage.Nodes)) 101 | for i, node := range storage.Nodes { 102 | nodes[i] = r.Nodes[node] 103 | } 104 | t.SetSteps([]task.StepConfig{ 105 | { 106 | Nodes: []config.Node{nodes[0]}, 107 | NewStep: steps.NewGen3FSNodeIDStepFunc(ServiceName, 10001, storage.Nodes), 108 | }, 109 | { 110 | Nodes: nodes, 111 | Parallel: true, 112 | NewStep: steps.NewRemoteRunScriptStepFunc( 113 | workDir, 114 | "disk_tool.sh", 115 | DiskToolScriptTmpl, 116 | map[string]any{ 117 | "SectorSize": t.Runtime.Cfg.Services.Storage.SectorSize, 118 | }, 119 | []string{ 120 | workDir, 121 | strconv.Itoa(storage.DiskNumPerNode), 122 | string(storage.DiskType), 123 | "prepare", 124 | }), 125 | }, 126 | { 127 | Nodes: nodes, 128 | Parallel: true, 129 | NewStep: steps.NewPrepare3FSConfigStepFunc(&steps.Prepare3FSConfigStepSetup{ 130 | Service: ServiceName, 131 | ServiceWorkDir: workDir, 132 | MainAppTomlTmpl: StorageMainAppTomlTmpl, 133 | MainLauncherTomlTmpl: StorageMainLauncherTomlTmpl, 134 | MainTomlTmpl: StorageMainTomlTmpl, 135 | RDMAListenPort: storage.RDMAListenPort, 136 | TCPListenPort: storage.TCPListenPort, 137 | ExtraMainTomlData: map[string]any{ 138 | "TargetPaths": makeTargetPaths(storage.DiskNumPerNode), 139 | }, 140 | }), 141 | }, 142 | { 143 | Nodes: []config.Node{nodes[0]}, 144 | NewStep: steps.NewUpload3FSMainConfigStepFunc( 145 | config.ImageName3FS, 146 | storage.ContainerName, 147 | ServiceName, 148 | workDir, 149 | serviceType, 150 | ), 151 | }, 152 | { 153 | Nodes: nodes, 154 | Parallel: true, 155 | NewStep: steps.NewRun3FSContainerStepFunc( 156 | &steps.Run3FSContainerStepSetup{ 157 | ImgName: config.ImageName3FS, 158 | ContainerName: storage.ContainerName, 159 | Service: ServiceName, 160 | WorkDir: workDir, 161 | UseRdmaNetwork: true, 162 | ExtraVolumes: []*external.VolumeArgs{ 163 | { 164 | Source: path.Join(workDir, "3fsdata"), 165 | Target: "/mnt/3fsdata", 166 | }, 167 | }, 168 | }), 169 | }, 170 | }) 171 | } 172 | 173 | // DeleteStorageServiceTask is a task for deleting a storage services. 174 | type DeleteStorageServiceTask struct { 175 | task.BaseTask 176 | } 177 | 178 | // Init initializes the task. 179 | func (t *DeleteStorageServiceTask) Init(r *task.Runtime, logger log.Interface) { 180 | t.BaseTask.SetName("DeleteStorageServiceTask") 181 | t.BaseTask.Init(r, logger) 182 | nodes := make([]config.Node, len(r.Cfg.Services.Storage.Nodes)) 183 | for i, node := range r.Cfg.Services.Storage.Nodes { 184 | nodes[i] = r.Nodes[node] 185 | } 186 | storage := r.Services.Storage 187 | workDir := getServiceWorkDir(r.WorkDir) 188 | t.SetSteps([]task.StepConfig{ 189 | { 190 | Nodes: nodes, 191 | Parallel: true, 192 | NewStep: steps.NewRm3FSContainerStepFunc( 193 | r.Services.Storage.ContainerName, 194 | ServiceName, 195 | workDir), 196 | }, 197 | { 198 | Nodes: nodes, 199 | Parallel: true, 200 | NewStep: steps.NewRemoteRunScriptStepFunc( 201 | workDir, 202 | "disk_tool.sh", 203 | DiskToolScriptTmpl, 204 | map[string]any{ 205 | "SectorSize": t.Runtime.Cfg.Services.Storage.SectorSize, 206 | }, 207 | []string{ 208 | workDir, 209 | strconv.Itoa(storage.DiskNumPerNode), 210 | string(storage.DiskType), 211 | "clear", 212 | }), 213 | }, 214 | }) 215 | } 216 | -------------------------------------------------------------------------------- /pkg/storage/templates/disk_tool.sh.tmpl: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | if [ "$#" -lt 4 ]; then 5 | echo "Usage: $0 " 6 | exit 1 7 | fi 8 | 9 | MOUNT_DIR="$1/3fsdata" 10 | DISK_NUM=$2 11 | DISK_TYPE=$3 12 | ACTION=$4 13 | mkdir -p $MOUNT_DIR 14 | 15 | function prepare_dir_disk() { 16 | for i in $(seq 0 $((DISK_NUM-1))); do 17 | MOUNT_POINT="${MOUNT_DIR}/data${i}" 18 | mkdir -p ${MOUNT_POINT} 19 | mkdir -p ${MOUNT_POINT}/3fs 20 | done 21 | } 22 | 23 | function clear_dir_disk() { 24 | for i in $(seq 0 $((DISK_NUM-1))); do 25 | MOUNT_POINT="${MOUNT_DIR}/data${i}" 26 | rm -rf ${MOUNT_POINT} 27 | done 28 | rm -r ${MOUNT_DIR} 29 | } 30 | 31 | function prepare_nvme_disk() { 32 | # Find NVMe disks 33 | NVME_DISKS=() 34 | for disk in $(lsblk -d -n -o NAME,TYPE,SIZE | grep -i "nvme" | grep -i "disk" | awk '{print $1}'|sort -V);do 35 | # Check if disk is already mounted 36 | if grep -q "/dev/${disk}" /proc/mounts; then 37 | echo "Disk ${disk} is already mounted. Skipping..." 38 | continue 39 | fi 40 | NVME_DISKS+=($disk) 41 | done 42 | 43 | if [[ -z "${NVME_DISKS}" ]]; then 44 | echo "No NVMe disks found!" 45 | exit 1 46 | fi 47 | 48 | if [[ ${DISK_NUM} -gt ${#NVME_DISKS[@]} ]]; then 49 | echo "Number of disks(${DISK_NUM}) requested is greater than available NVMe disks(${#NVME_DISKS[@]})!" 50 | exit 1 51 | fi 52 | NVME_DISKS=("${NVME_DISKS[@]:0:${DISK_NUM}}") 53 | 54 | # Format and mount each NVMe disk 55 | ID=0 56 | for DISK in ${NVME_DISKS[@]}; do 57 | DISK_PATH="/dev/${DISK}" 58 | MOUNT_POINT="${MOUNT_DIR}/data${ID}" 59 | 60 | echo "Processing disk: ${DISK_PATH}" 61 | 62 | # Check if disk is already formatted with XFS 63 | if blkid -s TYPE -o value ${DISK_PATH} | grep -q "xfs"; then 64 | echo "Disk ${DISK_PATH} is already formatted with XFS." 65 | else 66 | echo "Formatting ${DISK_PATH} with XFS..." 67 | mkfs.xfs -f -L "3fs-data-${ID}" -s size={{ .SectorSize }} ${DISK_PATH} 68 | fi 69 | 70 | # Create mount point 71 | mkdir -p ${MOUNT_POINT} 72 | 73 | # Mount disk 74 | echo "Mounting ${DISK_PATH} to ${MOUNT_POINT}..." 75 | mount -t xfs ${DISK_PATH} ${MOUNT_POINT} 76 | mkdir -p ${MOUNT_POINT}/3fs 77 | 78 | echo "Disk ${DISK_PATH} successfully mounted at ${MOUNT_POINT}" 79 | ID=$((ID + 1)) 80 | done 81 | 82 | echo "All NVMe disks have been processed." 83 | echo "Total disks mounted: ${ID}" 84 | 85 | # List all mounted disks for verification 86 | echo "Mounted disks:" 87 | df -h | grep "${MOUNT_DIR}" 88 | } 89 | 90 | function clear_nvme_disk() { 91 | # Unmount and clear each NVMe disk 92 | for i in $(seq 0 $((DISK_NUM-1))); do 93 | MOUNT_POINT="${MOUNT_DIR}/data${i}" 94 | 95 | rm -rf ${MOUNT_POINT}/3fs 96 | # Check if disk is already mounted 97 | if grep -q "${MOUNT_POINT}" /proc/mounts; then 98 | echo "Unmounting ${MOUNT_POINT}..." 99 | umount ${MOUNT_POINT} 100 | else 101 | echo "${MOUNT_POINT} is not mounted. Skipping..." 102 | fi 103 | 104 | # Clear mount point 105 | rm -rf ${MOUNT_POINT} 106 | 107 | echo "${MOUNT_DIR} successfully unmounted and cleared." 108 | done 109 | rm -r ${MOUNT_DIR} 110 | 111 | echo "All NVMe disks have been processed." 112 | } 113 | 114 | if [[ "${DISK_TYPE}" == "nvme" ]]; then 115 | if [[ "${ACTION}" == "prepare" ]]; then 116 | prepare_nvme_disk 117 | elif [[ "${ACTION}" == "clear" ]]; then 118 | clear_nvme_disk 119 | else 120 | echo "Invalid action: ${ACTION}" 121 | exit 1 122 | fi 123 | else 124 | if [[ "${ACTION}" == "prepare" ]]; then 125 | prepare_dir_disk 126 | elif [[ "${ACTION}" == "clear" ]]; then 127 | clear_dir_disk 128 | else 129 | echo "Invalid action: ${ACTION}" 130 | exit 1 131 | fi 132 | fi 133 | -------------------------------------------------------------------------------- /pkg/storage/templates/storage_main_app.toml.tmpl: -------------------------------------------------------------------------------- 1 | allow_empty_node_id = true 2 | node_id = {{ .NodeID }} 3 | -------------------------------------------------------------------------------- /pkg/storage/templates/storage_main_launcher.toml.tmpl: -------------------------------------------------------------------------------- 1 | allow_dev_version = true 2 | cluster_id = '{{ .ClusterID }}' 3 | 4 | [client] 5 | default_compression_level = 0 6 | default_compression_threshold = '128KB' 7 | default_log_long_running_threshold = '0ns' 8 | default_report_metrics = false 9 | default_send_retry_times = 1 10 | default_timeout = '1s' 11 | enable_rdma_control = false 12 | force_use_tcp = false 13 | 14 | [client.io_worker] 15 | num_event_loop = 1 16 | rdma_connect_timeout = '5s' 17 | read_write_rdma_in_event_thread = false 18 | read_write_tcp_in_event_thread = false 19 | tcp_connect_timeout = '1s' 20 | wait_to_retry_send = '100ms' 21 | 22 | [client.io_worker.connect_concurrency_limiter] 23 | max_concurrency = 4 24 | 25 | [client.io_worker.ibsocket] 26 | buf_ack_batch = 8 27 | buf_signal_batch = 8 28 | buf_size = 16384 29 | drain_timeout = '5s' 30 | drop_connections = 0 31 | event_ack_batch = 128 32 | max_rd_atomic = 16 33 | max_rdma_wr = 128 34 | max_rdma_wr_per_post = 32 35 | max_sge = 1 36 | min_rnr_timer = 1 37 | record_bytes_per_peer = false 38 | record_latency_per_peer = false 39 | retry_cnt = 7 40 | rnr_retry = 0 41 | send_buf_cnt = 32 42 | sl = 0 43 | start_psn = 0 44 | timeout = 14 45 | 46 | [client.io_worker.transport_pool] 47 | max_connections = 1 48 | 49 | [client.processor] 50 | enable_coroutines_pool = true 51 | max_coroutines_num = 256 52 | max_processing_requests_num = 4096 53 | response_compression_level = 1 54 | response_compression_threshold = '128KB' 55 | 56 | [client.rdma_control] 57 | max_concurrent_transmission = 64 58 | 59 | [client.thread_pool] 60 | bg_thread_pool_stratetry = 'SHARED_QUEUE' 61 | collect_stats = false 62 | enable_work_stealing = false 63 | io_thread_pool_stratetry = 'SHARED_QUEUE' 64 | num_bg_threads = 2 65 | num_connect_threads = 2 66 | num_io_threads = 2 67 | num_proc_threads = 2 68 | proc_thread_pool_stratetry = 'SHARED_QUEUE' 69 | 70 | [ib_devices] 71 | allow_no_usable_devices = false 72 | allow_unknown_zone = true 73 | default_network_zone = 'UNKNOWN' 74 | default_pkey_index = 0 75 | default_roce_pkey_index = 0 76 | default_traffic_class = 0 77 | device_filter = [] 78 | fork_safe = true 79 | prefer_ibdevice = true 80 | skip_inactive_ports = true 81 | skip_unusable_device = true 82 | subnets = [] 83 | 84 | [mgmtd_client] 85 | accept_incomplete_routing_info_during_mgmtd_bootstrapping = true 86 | auto_extend_client_session_interval = '10s' 87 | auto_heartbeat_interval = '10s' 88 | auto_refresh_interval = '10s' 89 | enable_auto_extend_client_session = false 90 | enable_auto_heartbeat = true 91 | enable_auto_refresh = true 92 | mgmtd_server_addresses = {{ .MgmtdServerAddresses }} 93 | work_queue_size = 100 94 | -------------------------------------------------------------------------------- /pkg/task/base_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package task 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/stretchr/testify/mock" 21 | "github.com/stretchr/testify/suite" 22 | 23 | "github.com/open3fs/m3fs/pkg/log" 24 | "github.com/open3fs/m3fs/tests/base" 25 | ) 26 | 27 | type baseSuite struct { 28 | base.Suite 29 | } 30 | 31 | var suiteRun = suite.Run 32 | 33 | type mockTask struct { 34 | mock.Mock 35 | Interface 36 | } 37 | 38 | func (m *mockTask) Run(context.Context) error { 39 | args := m.Called() 40 | return args.Error(0) 41 | } 42 | 43 | func (m *mockTask) Name() string { 44 | args := m.Called() 45 | return args.String(0) 46 | } 47 | 48 | func (m *mockTask) Init(r *Runtime, logger log.Interface) { 49 | m.Called(r) 50 | } 51 | -------------------------------------------------------------------------------- /pkg/task/runner.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package task 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "strings" 21 | "sync" 22 | 23 | "github.com/fatih/color" 24 | "github.com/sirupsen/logrus" 25 | 26 | "github.com/open3fs/m3fs/pkg/config" 27 | "github.com/open3fs/m3fs/pkg/errors" 28 | "github.com/open3fs/m3fs/pkg/external" 29 | "github.com/open3fs/m3fs/pkg/log" 30 | "github.com/open3fs/m3fs/pkg/utils" 31 | ) 32 | 33 | // defines keys of runtime cache. 34 | const ( 35 | RuntimeArtifactTmpDirKey = "artifact/tmp_dir" 36 | RuntimeArtifactPathKey = "artifact/path" 37 | RuntimeArtifactGzipKey = "artifact/gzip" 38 | RuntimeArtifactSha256sumKey = "artifact/sha256sum" 39 | RuntimeArtifactFilePathsKey = "artifact/file_paths" 40 | 41 | RuntimeClickhouseTmpDirKey = "clickhouse/tmp_dir" 42 | RuntimeGrafanaTmpDirKey = "grafana/tmp_dir" 43 | RuntimeMonitorTmpDirKey = "monitor/tmp_dir" 44 | RuntimeFdbClusterFileContentKey = "fdb/cluster_file_content" 45 | RuntimeMgmtdServerAddressesKey = "mgmtd/server_addresses" 46 | RuntimeUserTokenKey = "user_token" 47 | RuntimeAdminCliTomlKey = "admin_cli_toml" 48 | ) 49 | 50 | // Runtime contains task run info 51 | type Runtime struct { 52 | sync.Map 53 | Cfg *config.Config 54 | Nodes map[string]config.Node 55 | Services *config.Services 56 | WorkDir string 57 | LocalEm *external.Manager 58 | LocalNode *config.Node 59 | 60 | // MgmtdProtocol is used to set the protocol of mgmtd address. 61 | // It maps RDMA types to RDMA:// 62 | // It maps IB types to IPoIB:// 63 | // Currently, only mgmtd address uses IPoIB protocol, all other services still use RDMA protocol. 64 | // TODO: Find the reason from 3FS code base. 65 | MgmtdProtocol string 66 | } 67 | 68 | // LoadString load string value form sync map 69 | func (r *Runtime) LoadString(key any) (string, bool) { 70 | valI, ok := r.Load(key) 71 | if !ok { 72 | return "", false 73 | } 74 | 75 | return valI.(string), true 76 | } 77 | 78 | // LoadBool load bool value form sync map 79 | func (r *Runtime) LoadBool(key any) (bool, bool) { 80 | valI, ok := r.Load(key) 81 | if !ok { 82 | return false, false 83 | } 84 | 85 | return valI.(bool), true 86 | } 87 | 88 | // LoadInt load int value form sync map 89 | func (r *Runtime) LoadInt(key any) (int, bool) { 90 | valI, ok := r.Load(key) 91 | if !ok { 92 | return 0, false 93 | } 94 | 95 | return valI.(int), true 96 | } 97 | 98 | // Runner is a task runner. 99 | type Runner struct { 100 | Runtime *Runtime 101 | tasks []Interface 102 | cfg *config.Config 103 | localNode *config.Node 104 | init bool 105 | } 106 | 107 | // Init initializes all tasks. 108 | func (r *Runner) Init() { 109 | r.Runtime = &Runtime{Cfg: r.cfg, WorkDir: r.cfg.WorkDir, LocalNode: r.localNode} 110 | r.Runtime.MgmtdProtocol = "RDMA" 111 | if r.cfg.NetworkType == config.NetworkTypeIB { 112 | r.Runtime.MgmtdProtocol = "IPoIB" 113 | } 114 | r.Runtime.Nodes = make(map[string]config.Node, len(r.cfg.Nodes)) 115 | for _, node := range r.cfg.Nodes { 116 | r.Runtime.Nodes[node.Name] = node 117 | } 118 | r.Runtime.Services = &r.cfg.Services 119 | logger := log.Logger.Subscribe(log.FieldKeyNode, "") 120 | runnerCfg := &external.LocalRunnerCfg{ 121 | Logger: logger, 122 | MaxExitTimeout: r.cfg.CmdMaxExitTimeout, 123 | } 124 | if r.localNode != nil { 125 | runnerCfg.User = r.localNode.Username 126 | if r.localNode.Password != nil { 127 | runnerCfg.Password = *r.localNode.Password 128 | } 129 | } 130 | em := external.NewManager(external.NewLocalRunner(runnerCfg), logger) 131 | r.Runtime.LocalEm = em 132 | 133 | for _, task := range r.tasks { 134 | task.Init(r.Runtime, log.Logger.Subscribe(log.FieldKeyTask, task.Name())) 135 | } 136 | r.init = true 137 | } 138 | 139 | // Store sets the value for a key. 140 | func (r *Runner) Store(key, value any) error { 141 | if r.Runtime == nil { 142 | return errors.Errorf("Runtime hasn't been initialized") 143 | } 144 | r.Runtime.Store(key, value) 145 | return nil 146 | } 147 | 148 | // Register registers tasks. 149 | func (r *Runner) Register(task ...Interface) error { 150 | if r.init { 151 | return errors.New("runner has been initialized") 152 | } 153 | r.tasks = append(r.tasks, task...) 154 | return nil 155 | } 156 | 157 | // getColorAttribute returns the corresponding color.Attribute based on the color name in configuration 158 | // Returns -1 if the color name is "none" or not recognized 159 | func getColorAttribute(colorName string) color.Attribute { 160 | if strings.ToLower(colorName) == "none" { 161 | return color.Attribute(-1) // Special value to indicate no color 162 | } 163 | 164 | colorMap := map[string]color.Attribute{ 165 | "green": color.FgHiGreen, 166 | "cyan": color.FgHiCyan, 167 | "yellow": color.FgHiYellow, 168 | "blue": color.FgHiBlue, 169 | "magenta": color.FgHiMagenta, 170 | "red": color.FgHiRed, 171 | "white": color.FgHiWhite, 172 | } 173 | 174 | if attr, ok := colorMap[strings.ToLower(colorName)]; ok { 175 | return attr 176 | } 177 | 178 | // Return invalid attribute to indicate no color 179 | return color.Attribute(-1) 180 | } 181 | 182 | // Run runs all tasks. 183 | func (r *Runner) Run(ctx context.Context) error { 184 | useColor := false 185 | var highlightColor color.Attribute 186 | if r.cfg != nil && r.cfg.UI.TaskInfoColor != "" { 187 | highlightColor = getColorAttribute(r.cfg.UI.TaskInfoColor) 188 | useColor = int(highlightColor) >= 0 189 | } 190 | for _, task := range r.tasks { 191 | var message string 192 | if useColor { 193 | taskHighlight := color.New(highlightColor, color.Bold).SprintFunc() 194 | message = taskHighlight(fmt.Sprintf("Running task %s", task.Name())) 195 | } else { 196 | message = fmt.Sprintf("Running task %s", task.Name()) 197 | } 198 | logrus.Info(message) 199 | if err := task.Run(ctx); err != nil { 200 | return errors.Annotatef(err, "run task %s", task.Name()) 201 | } 202 | } 203 | return nil 204 | } 205 | 206 | // NewRunner creates a new task runner. 207 | func NewRunner(cfg *config.Config, tasks ...Interface) (*Runner, error) { 208 | localIPs, err := utils.GetLocalIPs() 209 | if err != nil { 210 | return nil, errors.Trace(err) 211 | } 212 | var localNode *config.Node 213 | for i, node := range cfg.Nodes { 214 | if isLocal, err := utils.IsLocalHost(node.Host, localIPs); err != nil { 215 | return nil, errors.Trace(err) 216 | } else if isLocal { 217 | localNode = &cfg.Nodes[i] 218 | break 219 | } 220 | } 221 | return &Runner{ 222 | tasks: tasks, 223 | localNode: localNode, 224 | cfg: cfg, 225 | }, nil 226 | } 227 | -------------------------------------------------------------------------------- /pkg/task/runner_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package task 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/fatih/color" 21 | "github.com/stretchr/testify/mock" 22 | 23 | "github.com/open3fs/m3fs/pkg/config" 24 | ) 25 | 26 | func TestRunnerSuite(t *testing.T) { 27 | suiteRun(t, new(runnerSuite)) 28 | } 29 | 30 | type runnerSuite struct { 31 | baseSuite 32 | runner *Runner 33 | mockTask *mockTask 34 | } 35 | 36 | func (s *runnerSuite) SetupTest() { 37 | s.baseSuite.SetupTest() 38 | s.mockTask = new(mockTask) 39 | s.runner = &Runner{ 40 | tasks: []Interface{s.mockTask}, 41 | cfg: new(config.Config), 42 | } 43 | } 44 | 45 | func (s *runnerSuite) TestInit() { 46 | s.mockTask.On("Init", mock.AnythingOfType("*task.Runtime")) 47 | s.mockTask.On("Name").Return("mockTask") 48 | 49 | s.runner.Init() 50 | 51 | s.mockTask.AssertExpectations(s.T()) 52 | } 53 | 54 | func (s *runnerSuite) TestInitWithIB() { 55 | s.runner.cfg.NetworkType = config.NetworkTypeIB 56 | s.TestInit() 57 | 58 | s.Equal(s.runner.Runtime.MgmtdProtocol, "IPoIB") 59 | } 60 | 61 | func (s *runnerSuite) TestRegisterAfterInit() { 62 | s.TestInit() 63 | s.mockTask.On("Name").Return("mockTask") 64 | 65 | s.Error(s.runner.Register(s.mockTask), "runner has been initialized") 66 | } 67 | 68 | func (s *runnerSuite) TestRegister() { 69 | task2 := new(mockTask) 70 | s.NoError(s.runner.Register(s.mockTask)) 71 | 72 | s.Equal(s.runner.tasks, []Interface{s.mockTask, task2}) 73 | } 74 | 75 | func (s *runnerSuite) TestRun() { 76 | s.mockTask.On("Name").Return("mockTask") 77 | s.mockTask.On("Run").Return(nil) 78 | 79 | s.NoError(s.runner.Run(s.Ctx())) 80 | 81 | s.mockTask.AssertExpectations(s.T()) 82 | } 83 | 84 | func (s *runnerSuite) testTaskInfoHighlighting() { 85 | s.mockTask.On("Name").Return("mockTask") 86 | s.mockTask.On("Run").Return(nil) 87 | 88 | s.NoError(s.runner.Run(s.Ctx())) 89 | 90 | s.mockTask.AssertExpectations(s.T()) 91 | } 92 | 93 | func (s *runnerSuite) TestTaskInfoHighlightingWithValidColor() { 94 | s.runner.cfg = &config.Config{ 95 | UI: config.UIConfig{ 96 | TaskInfoColor: "green", 97 | }, 98 | } 99 | s.testTaskInfoHighlighting() 100 | } 101 | 102 | func (s *runnerSuite) TestTaskInfoHighlightingWithNoneColor() { 103 | s.runner.cfg = &config.Config{ 104 | UI: config.UIConfig{ 105 | TaskInfoColor: "none", 106 | }, 107 | } 108 | s.testTaskInfoHighlighting() 109 | } 110 | 111 | func (s *runnerSuite) TestTaskInfoHighlightingWithInvalidColor() { 112 | s.runner.cfg = &config.Config{ 113 | UI: config.UIConfig{ 114 | TaskInfoColor: "invalid-color", 115 | }, 116 | } 117 | s.testTaskInfoHighlighting() 118 | } 119 | 120 | func (s *runnerSuite) TestTaskInfoHighlightingWithEmptyColor() { 121 | s.runner.cfg = &config.Config{ 122 | UI: config.UIConfig{ 123 | TaskInfoColor: "", 124 | }, 125 | } 126 | s.testTaskInfoHighlighting() 127 | } 128 | 129 | func (s *runnerSuite) TestTaskInfoHighlightingWithNoUIConfig() { 130 | s.runner.cfg = &config.Config{} 131 | s.testTaskInfoHighlighting() 132 | } 133 | 134 | func (s *runnerSuite) TestGetColorAttribute() { 135 | cases := []struct { 136 | expected color.Attribute 137 | colorName string 138 | }{ 139 | {color.FgHiGreen, "green"}, 140 | {color.FgHiCyan, "cyan"}, 141 | {color.FgHiYellow, "yellow"}, 142 | {color.FgHiBlue, "blue"}, 143 | {color.FgHiMagenta, "magenta"}, 144 | {color.FgHiRed, "red"}, 145 | {color.FgHiWhite, "white"}, 146 | {color.FgHiGreen, "GREEN"}, 147 | {color.FgHiCyan, "Cyan"}, 148 | {color.Attribute(-1), "none"}, 149 | {color.Attribute(-1), "NONE"}, 150 | {color.Attribute(-1), "invalid-color"}, 151 | {color.Attribute(-1), ""}, 152 | } 153 | 154 | for _, c := range cases { 155 | s.Equal(c.expected, getColorAttribute(c.colorName)) 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /pkg/task/steps/local_steps.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package steps 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/open3fs/m3fs/pkg/errors" 21 | "github.com/open3fs/m3fs/pkg/task" 22 | ) 23 | 24 | type cleanupLocalStep struct { 25 | task.BaseStep 26 | 27 | tmpDirKey string 28 | } 29 | 30 | func (s *cleanupLocalStep) Execute(ctx context.Context) error { 31 | tmpDir, ok := s.Runtime.LoadString(s.tmpDirKey) 32 | if !ok { 33 | return errors.Errorf("Failed to get value of %s", s.tmpDirKey) 34 | } 35 | if err := s.Runtime.LocalEm.FS.RemoveAll(ctx, tmpDir); err != nil { 36 | return errors.Trace(err) 37 | } 38 | return nil 39 | } 40 | 41 | // NewCleanupLocalStepFunc is the cleanup local step factory func. 42 | func NewCleanupLocalStepFunc(tmpDirKey string) func() task.Step { 43 | return func() task.Step { 44 | return &cleanupLocalStep{ 45 | tmpDirKey: tmpDirKey, 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /pkg/task/steps/local_steps_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package steps 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/open3fs/m3fs/pkg/config" 21 | "github.com/open3fs/m3fs/pkg/task" 22 | ttask "github.com/open3fs/m3fs/tests/task" 23 | ) 24 | 25 | func TestCleanupLocalStepSuite(t *testing.T) { 26 | suiteRun(t, &cleanupLocalStepSuite{}) 27 | } 28 | 29 | type cleanupLocalStepSuite struct { 30 | ttask.StepSuite 31 | 32 | step *cleanupLocalStep 33 | } 34 | 35 | func (s *cleanupLocalStepSuite) SetupTest() { 36 | s.StepSuite.SetupTest() 37 | 38 | s.step = NewCleanupLocalStepFunc(task.RuntimeClickhouseTmpDirKey)().(*cleanupLocalStep) 39 | s.SetupRuntime() 40 | s.step.Init(s.Runtime, s.MockEm, config.Node{}, s.Logger) 41 | s.Runtime.Store(task.RuntimeClickhouseTmpDirKey, "/tmp/3fs/clickhouse-xxx") 42 | } 43 | 44 | func (s *cleanupLocalStepSuite) Test() { 45 | s.MockLocalFS.On("RemoveAll", "/tmp/3fs/clickhouse-xxx").Return(nil) 46 | 47 | s.NoError(s.step.Execute(s.Ctx())) 48 | } 49 | -------------------------------------------------------------------------------- /pkg/utils/net.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utils 16 | 17 | import ( 18 | "net" 19 | 20 | "github.com/open3fs/m3fs/pkg/errors" 21 | ) 22 | 23 | // GetLocalIPs returns all local IP addresses. 24 | func GetLocalIPs() ([]*net.IP, error) { 25 | addrs, err := net.InterfaceAddrs() 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | var ips []*net.IP 31 | for _, addr := range addrs { 32 | if ipNet, ok := addr.(*net.IPNet); ok && !ipNet.IP.IsLoopback() { 33 | ips = append(ips, &ipNet.IP) 34 | } 35 | } 36 | 37 | return ips, nil 38 | } 39 | 40 | // IsLocalHost checks if the given host is a local host. 41 | func IsLocalHost(host string, localIPs []*net.IP) (bool, error) { 42 | hostIPs, err := net.LookupIP(host) 43 | if err != nil { 44 | return false, errors.Annotatef(err, "lookup IP address of %s", host) 45 | } 46 | 47 | for _, ip := range hostIPs { 48 | for _, localIP := range localIPs { 49 | if ip.Equal(*localIP) { 50 | return true, nil 51 | } 52 | } 53 | } 54 | 55 | return false, nil 56 | } 57 | 58 | // GenerateIPRange generates a list of IP addresses in the range of [ipStart, ipEnd]. 59 | func GenerateIPRange(ipStart, ipEnd string) ([]string, error) { 60 | start := net.ParseIP(ipStart) 61 | if start == nil { 62 | return nil, errors.Errorf("invalid start IP: %s", ipStart) 63 | } 64 | end := net.ParseIP(ipEnd) 65 | if end == nil { 66 | return nil, errors.Errorf("invalid end IP: %s", ipEnd) 67 | } 68 | 69 | if (start.To4() == nil) != (end.To4() == nil) { 70 | return nil, errors.Errorf("start IP and end IP are not in the same format") 71 | } 72 | 73 | startInt, endInt := uint64(0), uint64(0) 74 | ipv4 := start.To4() != nil 75 | if ipv4 { 76 | // ipv4 77 | startInt = ipToInt(start) 78 | endInt = ipToInt(end) 79 | } else { 80 | // ipv6 81 | startInt = ipv6ToInt(start) 82 | endInt = ipv6ToInt(end) 83 | } 84 | if startInt > endInt { 85 | return nil, errors.Errorf("start IP %s is greater than end IP %s", ipStart, ipEnd) 86 | } 87 | 88 | ips := make([]string, 0, endInt-startInt+1) 89 | for i := startInt; i <= endInt; i++ { 90 | var ip net.IP 91 | if ipv4 { 92 | ip = intToIP(uint32(i)) 93 | } else { 94 | ip = intToIPv6(i) 95 | } 96 | ips = append(ips, ip.String()) 97 | } 98 | 99 | return ips, nil 100 | } 101 | 102 | func ipToInt(ip net.IP) uint64 { 103 | ip = ip.To4() 104 | return uint64(uint32(ip[0])<<24 | uint32(ip[1])<<16 | uint32(ip[2])<<8 | uint32(ip[3])) 105 | } 106 | 107 | func ipv6ToInt(ip net.IP) uint64 { 108 | ip = ip.To16() 109 | return uint64(ip[0])<<56 | uint64(ip[1])<<48 | uint64(ip[2])<<40 | uint64(ip[3])<<32 | 110 | uint64(ip[4])<<24 | uint64(ip[5])<<16 | uint64(ip[6])<<8 | uint64(ip[7]) 111 | } 112 | 113 | func intToIP(n uint32) net.IP { 114 | ip := make(net.IP, 4) 115 | ip[0] = byte(n >> 24) 116 | ip[1] = byte(n >> 16) 117 | ip[2] = byte(n >> 8) 118 | ip[3] = byte(n) 119 | return ip 120 | } 121 | 122 | func intToIPv6(n uint64) net.IP { 123 | ip := make(net.IP, 16) 124 | ip[0] = byte(n >> 56) 125 | ip[1] = byte(n >> 48) 126 | ip[2] = byte(n >> 40) 127 | ip[3] = byte(n >> 32) 128 | ip[4] = byte(n >> 24) 129 | ip[5] = byte(n >> 16) 130 | ip[6] = byte(n >> 8) 131 | ip[7] = byte(n) 132 | return ip 133 | } 134 | -------------------------------------------------------------------------------- /pkg/utils/set.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utils 16 | 17 | import "github.com/open3fs/m3fs/pkg/common" 18 | 19 | // Set is a set 20 | type Set[T comparable] map[T]struct{} 21 | 22 | // Add adds an item to the set 23 | func (s Set[T]) Add(item T) { 24 | s[item] = struct{}{} 25 | } 26 | 27 | // Remove removes an item from the set 28 | func (s Set[T]) Remove(item T) { 29 | delete(s, item) 30 | } 31 | 32 | // Contains returns true if the set contains the item 33 | func (s Set[T]) Contains(item T) bool { 34 | _, ok := s[item] 35 | return ok 36 | } 37 | 38 | // Len returns the number of items in the set 39 | func (s Set[T]) Len() int { 40 | return len(s) 41 | } 42 | 43 | // AddIfNotExists adds an item to the set if it does not already exist 44 | func (s Set[T]) AddIfNotExists(item T) bool { 45 | add := false 46 | if !s.Contains(item) { 47 | s.Add(item) 48 | add = true 49 | } 50 | 51 | return add 52 | } 53 | 54 | // ToSlice converts the set to a slice 55 | func (s Set[T]) ToSlice() []T { 56 | ret := make([]T, 0, len(s)) 57 | for item := range s { 58 | ret = append(ret, item) 59 | } 60 | return ret 61 | } 62 | 63 | // Equal check two set equal 64 | func (s Set[T]) Equal(other Set[T]) bool { 65 | if len(s) != len(other) { 66 | return false 67 | } 68 | for item := range s { 69 | if _, ok := other[item]; !ok { 70 | return false 71 | } 72 | } 73 | return true 74 | } 75 | 76 | // NewSet creates a new Set 77 | func NewSet[T comparable](elems ...T) *Set[T] { 78 | s := common.Pointer(make(Set[T])) 79 | for _, elem := range elems { 80 | s.Add(elem) 81 | } 82 | return s 83 | } 84 | -------------------------------------------------------------------------------- /tests/external/docker.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package external 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/stretchr/testify/mock" 21 | 22 | "github.com/open3fs/m3fs/pkg/external" 23 | ) 24 | 25 | // MockDocker is an mock type for the DockerInterface 26 | type MockDocker struct { 27 | mock.Mock 28 | external.DockerInterface 29 | } 30 | 31 | // Run mock. 32 | func (m *MockDocker) Run(ctx context.Context, args *external.RunArgs) (string, error) { 33 | arg := m.Called(args) 34 | err1 := arg.Error(1) 35 | if err1 != nil { 36 | return "", err1 37 | } 38 | return arg.String(0), nil 39 | } 40 | 41 | // Rm mock. 42 | func (m *MockDocker) Rm(ctx context.Context, name string, force bool) (string, error) { 43 | arg := m.Called(name, force) 44 | err1 := arg.Error(1) 45 | if err1 != nil { 46 | return "", err1 47 | } 48 | return arg.String(0), nil 49 | } 50 | 51 | // Exec mock. 52 | func (m *MockDocker) Exec(ctx context.Context, container, cmd string, args ...string) ( 53 | string, error) { 54 | 55 | arg := m.Called(container, cmd, args) 56 | err1 := arg.Error(1) 57 | if err1 != nil { 58 | return "", err1 59 | } 60 | return arg.String(0), nil 61 | } 62 | 63 | // Load mock. 64 | func (m *MockDocker) Load(ctx context.Context, path string) (string, error) { 65 | arg := m.Called(path) 66 | return arg.String(0), arg.Error(1) 67 | } 68 | 69 | // Tag mock. 70 | func (m *MockDocker) Tag(ctx context.Context, src, dst string) error { 71 | return m.Called(src, dst).Error(0) 72 | } 73 | -------------------------------------------------------------------------------- /tests/external/fs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package external 16 | 17 | import ( 18 | "context" 19 | "os" 20 | 21 | "github.com/stretchr/testify/mock" 22 | 23 | "github.com/open3fs/m3fs/pkg/external" 24 | ) 25 | 26 | // MockFS is an mock type for the FSInterface 27 | type MockFS struct { 28 | mock.Mock 29 | external.FSInterface 30 | } 31 | 32 | // MkdirTemp mock. 33 | func (m *MockFS) MkdirTemp(ctx context.Context, dir, prefix string) (string, error) { 34 | arg := m.Called(dir, prefix) 35 | return arg.String(0), arg.Error(1) 36 | } 37 | 38 | // MkTempFile mock. 39 | func (m *MockFS) MkTempFile(ctx context.Context, dir string) (string, error) { 40 | arg := m.Called(dir) 41 | return arg.String(0), arg.Error(1) 42 | } 43 | 44 | // MkdirAll mock. 45 | func (m *MockFS) MkdirAll(ctx context.Context, path string) error { 46 | return m.Called(path).Error(0) 47 | } 48 | 49 | // RemoveAll mock. 50 | func (m *MockFS) RemoveAll(ctx context.Context, path string) error { 51 | return m.Called(path).Error(0) 52 | } 53 | 54 | // WriteFile mock. 55 | func (m *MockFS) WriteFile(path string, data []byte, perm os.FileMode) error { 56 | return m.Called(path, data, perm).Error(0) 57 | } 58 | 59 | // DownloadFile mock. 60 | func (m *MockFS) DownloadFile(url, dstPath string) error { 61 | return m.Called(url, dstPath).Error(0) 62 | } 63 | 64 | // ReadRemoteFile mock. 65 | func (m *MockFS) ReadRemoteFile(url string) (string, error) { 66 | arg := m.Called(url) 67 | return arg.String(0), arg.Error(1) 68 | } 69 | 70 | // IsNotExist mock. 71 | func (m *MockFS) IsNotExist(path string) (bool, error) { 72 | arg := m.Called(path) 73 | return arg.Bool(0), arg.Error(1) 74 | } 75 | 76 | // Sha256sum mock. 77 | func (m *MockFS) Sha256sum(ctx context.Context, path string) (string, error) { 78 | arg := m.Called(path) 79 | return arg.String(0), arg.Error(1) 80 | } 81 | 82 | // Tar mock. 83 | func (m *MockFS) Tar(srcPaths []string, basePath, dstPath string, needGzip bool) error { 84 | return m.Called(srcPaths, basePath, dstPath, needGzip).Error(0) 85 | } 86 | 87 | // ExtractTar mock. 88 | func (m *MockFS) ExtractTar(ctx context.Context, srcPath, dstDir string) error { 89 | return m.Called(srcPath, dstDir).Error(0) 90 | } 91 | -------------------------------------------------------------------------------- /tests/external/runner.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package external 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/stretchr/testify/mock" 21 | 22 | "github.com/open3fs/m3fs/pkg/external" 23 | ) 24 | 25 | // MockRunner is an mock type for the RunnerInterface 26 | type MockRunner struct { 27 | mock.Mock 28 | external.RunnerInterface 29 | } 30 | 31 | // NonSudoExec mock. 32 | func (m *MockRunner) NonSudoExec(ctx context.Context, cmd string, args ...string) (string, error) { 33 | arg := m.Called(cmd, args) 34 | err1 := arg.Error(1) 35 | if err1 != nil { 36 | return "", err1 37 | } 38 | return arg.String(0), nil 39 | } 40 | 41 | // Exec mock. 42 | func (m *MockRunner) Exec(ctx context.Context, cmd string, args ...string) (string, error) { 43 | arg := m.Called(cmd, args) 44 | err1 := arg.Error(1) 45 | if err1 != nil { 46 | return "", err1 47 | } 48 | return arg.String(0), nil 49 | } 50 | 51 | // Scp mock. 52 | func (m *MockRunner) Scp(ctx context.Context, local, remote string) error { 53 | arg := m.Called(local, remote) 54 | return arg.Error(0) 55 | } 56 | -------------------------------------------------------------------------------- /tests/task/step.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Open3FS Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package task 16 | 17 | import ( 18 | "github.com/open3fs/m3fs/pkg/config" 19 | "github.com/open3fs/m3fs/pkg/external" 20 | "github.com/open3fs/m3fs/pkg/task" 21 | "github.com/open3fs/m3fs/tests/base" 22 | texternal "github.com/open3fs/m3fs/tests/external" 23 | ) 24 | 25 | // StepSuite is the base Suite for all step suites. 26 | type StepSuite struct { 27 | base.Suite 28 | 29 | Cfg *config.Config 30 | Runtime *task.Runtime 31 | MockEm *external.Manager 32 | MockRunner *texternal.MockRunner 33 | MockDocker *texternal.MockDocker 34 | MockFS *texternal.MockFS 35 | // NOTE: external.FSInterface is not implemented for remote runner. 36 | // MockFS *texternal.MockFS 37 | MockLocalEm *external.Manager 38 | MockLocalRunner *texternal.MockRunner 39 | MockLocalFS *texternal.MockFS 40 | MockLocalDocker *texternal.MockDocker 41 | } 42 | 43 | // SetupTest runs before each test in the step suite. 44 | func (s *StepSuite) SetupTest() { 45 | s.Suite.SetupTest() 46 | 47 | s.Cfg = config.NewConfigWithDefaults() 48 | s.Cfg.Name = "test-cluster" 49 | s.Cfg.WorkDir = "/root/3fs" 50 | 51 | s.MockRunner = new(texternal.MockRunner) 52 | s.MockDocker = new(texternal.MockDocker) 53 | s.MockFS = new(texternal.MockFS) 54 | s.MockEm = &external.Manager{ 55 | Runner: s.MockRunner, 56 | Docker: s.MockDocker, 57 | FS: s.MockFS, 58 | } 59 | 60 | s.MockLocalDocker = new(texternal.MockDocker) 61 | s.MockLocalRunner = new(texternal.MockRunner) 62 | s.MockLocalFS = new(texternal.MockFS) 63 | s.MockLocalEm = &external.Manager{ 64 | Runner: s.MockLocalRunner, 65 | FS: s.MockLocalFS, 66 | Docker: s.MockLocalDocker, 67 | } 68 | 69 | s.SetupRuntime() 70 | } 71 | 72 | // SetupRuntime setup runtime with the test config. 73 | func (s *StepSuite) SetupRuntime() { 74 | s.Runtime = &task.Runtime{ 75 | Cfg: s.Cfg, 76 | WorkDir: s.Cfg.WorkDir, 77 | Services: &s.Cfg.Services, 78 | LocalEm: s.MockLocalEm, 79 | } 80 | s.Runtime.Nodes = make(map[string]config.Node, len(s.Cfg.Nodes)) 81 | for _, node := range s.Cfg.Nodes { 82 | s.Runtime.Nodes[node.Name] = node 83 | } 84 | s.Runtime.Services = &s.Cfg.Services 85 | } 86 | --------------------------------------------------------------------------------