├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── ci.md │ ├── documentation.md │ ├── feature_request.md │ └── question.md ├── PULL_REQUEST_TEMPLATE.md ├── config.yml ├── dependabot.yml ├── label-commenter-config.yml ├── labeler.yml ├── readme │ └── images │ │ ├── community.svg │ │ ├── layer5-community-sign.png │ │ ├── meshery-logo-dark-text-side.svg │ │ ├── meshery-logo-light-text-side.svg │ │ ├── meshsync.svg │ │ ├── slack-128.png │ │ └── slack-dark-128.png ├── release-drafter.yml ├── security-insights.yml ├── stale.yml ├── welcome │ ├── meshery_dark.png │ └── meshery_light.png └── workflows │ ├── build-and-release.yml │ ├── ci.yml │ ├── error-codes-updater.yml │ ├── integration-tests-ci.yml │ ├── label-commenter.yml │ ├── labeler.yml │ ├── multi-platform.yml │ ├── newcomers-alert.yml │ ├── release-drafter.yml │ └── slack.yml ├── .gitignore ├── .golangci.yml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING-gitflow.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── all-certmgs.yaml ├── cert-manager-cainjector-c7d4dbdd9-jlstt.yaml ├── cert-manager-webhook-847d7676c9-89rtq.yaml ├── config.yaml ├── go.mod ├── go.sum ├── helpers └── component_info.json ├── install ├── Makefile.core.mk └── Makefile.show-help.mk ├── integration-tests ├── .gitignore ├── default_cluster_integration_test.go ├── default_cluster_test_cases_test.go ├── infrastructure │ ├── docker-compose.yaml │ ├── meshsync.yaml │ ├── setup.sh │ └── test-deployment.yaml └── unmarshal_object_test.go ├── internal ├── channels │ ├── broker.go │ ├── channel.go │ ├── generic.go │ └── system.go ├── config │ ├── config.go │ ├── config_local.go │ ├── crd_config.go │ ├── crd_config_test.go │ ├── default_config.go │ ├── error.go │ └── types.go ├── file │ ├── file.go │ ├── writer.go │ └── yaml_writer.go ├── output │ ├── composite.go │ ├── file.go │ ├── inmemory_deduplicator.go │ ├── nats.go │ ├── output.go │ └── processor.go └── pipeline │ ├── error.go │ ├── handlers.go │ ├── pipeline.go │ └── step.go ├── main.go ├── meshsync ├── discovery.go ├── error.go ├── exec.go ├── handlers.go ├── handlers_test.go ├── logstream.go └── meshsync.go ├── pkg ├── model │ ├── exec.go │ ├── log.go │ ├── model.go │ ├── model_converter.go │ ├── preprocessor.go │ └── process.go └── utils │ └── utils.go └── scripts ├── go-mod-tidy.sh └── go-test.sh /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug/issue report 3 | about: Report an issue to help improve the project. 4 | title: '' 5 | labels: 'kind/bug' 6 | assignees: '' 7 | --- 8 | **Description** 9 | 10 | 11 | **Expected Behavior** 12 | 13 | 14 | **Screenshots** 15 | 16 | 17 | **Environment:** 18 | - OS: [e.g. Ubuntu] 19 | - Browser: [e.g. Chrome, Safari] 20 | - Version: [e.g. 22] 21 | - Device: [e.g. laptop, iPhone 8] 22 | 23 | --- 24 | [Optional] **To Reproduce** 25 | Steps to reproduce the behavior: 26 | 1. Go to '...' 27 | 2. Click on '....' 28 | 3. Scroll down to '....' 29 | 4. See error 30 | 31 | [Optional] **Additional Context** 32 | 33 | 34 | ### Contributor Resources 35 | 36 | The meshery.io website uses Jekyll and GitHub Pages. Site content is found under the [`master` branch](https://github.com/meshery/meshery.io/tree/master). 37 | - See the [Contributing to Meshery.io Website](https://github.com/mesheryio/meshery.io#contributing-to-the-mesheryio-website) section of the readme.md and other [contributing instructions](https://docs.meshery.io/project/contributing), too. 38 | - See [Meshery site designs](https://www.figma.com/file/SMP3zxOjZztdOLtgN4dS2W/Meshery-UI?node-id=110%3A1) in Figma [(open invite)](https://www.figma.com/team_invite/redeem/qJy1c95qirjgWQODApilR9). 39 | - Fill-in a [Community Member Form](https://meshery.io/newcomer) and join the [Community Slack](http://slack.meshery.io) for access. 40 | 41 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/ci.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 🛠 Continuous Integration / DevOps 3 | about: Improve or update workflows or other automation 4 | title: '[CI]' 5 | labels: 'area/ci' 6 | assignees: '' 7 | --- 8 | #### Current Behavior 9 | 10 | 11 | #### Desired Behavior 12 | 13 | 14 | #### Implementation 15 | 16 | 17 | #### Acceptance Tests 18 | 19 | 20 | --- 21 | #### Contributor [Guides](https://docs.meshery.io/project/contributing) and Resources 22 | - 🛠 [Meshery Build & Release Strategy](https://docs.meshery.io/project/build-and-release) 23 | - 📚 [Instructions for contributing to documentation](https://github.com/meshery/meshery/blob/master/CONTRIBUTING.md#documentation-contribution-flow) 24 | - Meshery documentation [site](https://docs.meshery.io/) and [source](https://github.com/meshery/meshery/tree/master/docs) 25 | - 🎨 Wireframes and [designs for Meshery UI](https://www.figma.com/file/SMP3zxOjZztdOLtgN4dS2W/Meshery-UI) in Figma [(open invite)](https://www.figma.com/team_invite/redeem/qJy1c95qirjgWQODApilR9) 26 | - 🙋🏾🙋🏼 Questions: [Discussion Forum](https://meshery.io/community#community-forums) and [Community Slack](http://slack.meshery.io) 27 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/documentation.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Documentation issue 3 | about: Issues related to documentation. 4 | title: '[Docs]' 5 | labels: 'area/docs' 6 | assignees: '' 7 | --- 8 | **Current State:** 9 | 10 | 11 | **Desired State:** 12 | 13 | 14 | --- 15 | **Contributor Resources** 16 | - [Meshery documentation site](https://docs.meshery.io/) 17 | - [Meshery documentation source](https://github.com/meshery/meshery/tree/master/docs) 18 | - [Instructions for contributing to documentation](https://github.com/meshery/meshery/blob/master/CONTRIBUTING.md#documentation-contribution-flow) 19 | 20 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Request 3 | about: Suggest an enhancement to this project. 4 | title: '' 5 | labels: 'kind/enhancement' 6 | assignees: '' 7 | --- 8 | 9 | **Current Behavior** 10 | 11 | 12 | 13 | **Desired Behavior** 14 | 15 | 16 | 17 | ### Contributor Resources 18 | 19 | The meshery.io website uses Jekyll and GitHub Pages. Site content is found under the [`master` branch](https://github.com/meshery/meshery.io/tree/master). 20 | - See the [Contributing to Meshery.io Website](https://github.com/meshery/meshery.io#contributing-to-the-mesheryio-website) section of the readme.md and other [contributing instructions](https://docs.meshery.io/project/contributing), too. 21 | - See [Meshery site designs](https://www.figma.com/file/SMP3zxOjZztdOLtgN4dS2W/Meshery-UI?node-id=110%3A1) in Figma [(open invite)](https://www.figma.com/team_invite/redeem/qJy1c95qirjgWQODApilR9). 22 | - Fill-in a [Community Member Form](https://meshery.io/newcomer) and join the [Community Slack](http://slack.meshery.io) for access. 23 | 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: General question 3 | about: Request information about the project; clarify behavior of the software 4 | title: '[Question]' 5 | labels: 'kind/question' 6 | assignees: '' 7 | --- 8 | 9 | **How can we help?** 10 | 11 | 12 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | **Description** 2 | 3 | This PR fixes # 4 | 5 | **Notes for Reviewers** 6 | 7 | 8 | **[Signed commits](../CONTRIBUTING.md#signing-off-on-commits-developer-certificate-of-origin)** 9 | - [ ] Yes, I signed my commits. 10 | 11 | 12 | 24 | -------------------------------------------------------------------------------- /.github/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | #------------------------------------------------------------------------------- 3 | # Configuration for new-issue-welcome - https://github.com/behaviorbot/new-issue-welcome 4 | # Comment to be posted to on first time issues 5 | newIssueWelcomeComment: > 6 | Thanks for opening this issue. A contributor will be by to give feedback soon. In the meantime, please review the [Contributors' Welcome Guide](https://docs.meshery.io/project/community), engage in the [discussion forum](https://meshery.io/community#community-forums), and be sure to join the [community Slack](http://slack.meshery.io/). 7 | # Configuration for new-pr-welcome - https://github.com/behaviorbot/new-pr-welcome 8 | # Comment to be posted to on PRs from first time contributors in your repository 9 | newPRWelcomeComment: > 10 | Yay, your first pull request! :thumbsup: A contributor will be by to give feedback soon. In the meantime, please review the [Meshery Contributors' Welcome Guide](https://docs.meshery.io/project/contributing) and sure to join the [community Slack](http://slack.meshery.io/). 11 | 12 | Be sure to double-check that you have signed your commits. Here are instructions for [making signing an implicit activity while peforming a commit](https://github.com/meshery/meshery/blob/master/CONTRIBUTING.md#signing-off-on-commits-developer-certificate-of-origin). 13 | 14 | #------------------------------------------------------------------------------- 15 | # Configuration for first-pr-merge - https://github.com/behaviorbot/first-pr-merge 16 | # Comment to be posted to on pull requests merged by a first time user 17 | firstPRMergeComment: > 18 | Thanks for your contribution to Meshery! :tada: 19 | 20 | 21 | 22 | 23 | Meshery Logo 24 | 25 | 26 |         [Join the community](http://slack.meshery.io), if you haven't yet and please leave a :star: [star on the project](../stargazers). :smile: 27 | 28 | #------------------------------------------------------------------------------- 29 | # Configuration for request-info - https://github.com/behaviorbot/request-info 30 | # Comment to reply with 31 | requestInfoReplyComment: > 32 | Thanks for opening this issue. We welcome all input! If you could provide a little more information, this will greatly aide in its resolution. :thumbsup: 33 | # *OPTIONAL* Add a list of people whose Issues/PRs will not be commented on 34 | # keys must be GitHub usernames 35 | #requestInfoUserstoExclude: 36 | # - meshery/maintainers 37 | 38 | #------------------------------------------------------------------------------- 39 | # Configuration for sentiment-bot - https://github.com/behaviorbot/sentiment-bot 40 | # *Required* toxicity threshold between 0 and .99 with the higher numbers being the most toxic 41 | # Anything higher than this threshold will be marked as toxic and commented on 42 | sentimentBotToxicityThreshold: .9 43 | 44 | # *Required* Comment to reply with 45 | sentimentBotReplyComment: > 46 | Please be sure to review the code of conduct and be respectful of other users. // @meshery/maintainers 47 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gomod 4 | directory: "/" 5 | schedule: 6 | interval: monthly 7 | open-pull-requests-limit: 5 8 | labels: 9 | - pr/dependencies 10 | - kind/chore 11 | ignore: 12 | - dependency-name: k8s.io/api 13 | versions: 14 | - ">= 0.20.a" 15 | - "< 0.21" 16 | - dependency-name: k8s.io/apimachinery 17 | versions: 18 | - ">= 0.20.a" 19 | - "< 0.21" 20 | - dependency-name: k8s.io/client-go 21 | versions: 22 | - ">= 0.20.a" 23 | - "< 0.21" 24 | -------------------------------------------------------------------------------- /.github/label-commenter-config.yml: -------------------------------------------------------------------------------- 1 | comment: 2 | # header: "Please note the following requirement:" 3 | footer: "\ 4 | ---\n\n 5 | >         Be sure to [join the community](https://slack.meshery.io), if you haven't yet and please leave a :star: [star on the project](../stargazers) :smile: 6 | " 7 | 8 | labels: 9 | - name: issue/design required 10 | labeled: 11 | issue: 12 | body: This issue has been labeled with 'design-required'. Note that prior to commencing on implementation, a design specification needs to be created and reviewed for approval. See [Creating a Functional Specification](https://docs.google.com/document/d/1RP3IWLc-MiQS-QYasqCoVuCH7--G87p5ezE5f_nOzB8/edit?usp=sharing) to create a design spec. 13 | action: open 14 | - name: issue/remind 15 | labeled: 16 | issue: 17 | body: Checking in... it has been awhile since we've heard from you on this issue. Are you still working on it? Please let us know and please don't hesitate to contact a [MeshMate](https://meshery.io/community#meshmates) or any other [community member](https://meshery.io/community/members) for assistance. 18 | action: open 19 | pr: 20 | body: Checking in... it has been awhile since we've heard from you on this issue. Are you still working on it? Please let us know and please don't hesitate to contact a [MeshMate](https://meshery.io/community/meshmates/) or any other [community member](https://meshery.io/community/members) for assistance. 21 | action: open 22 | - name: issue/dco 23 | labeled: 24 | issue: 25 | body: "🚨 Alert! Git Police! We couldn’t help but notice that one or more of your commits is missing a sign-off. _A what?_ A commit sign-off (your email address).\n\n 26 | To amend the commits in this PR with your signoff using the instructions provided in the DCO check. \n\n 27 | To configure your dev environment to automatically signoff on your commits in the future, see [these instructions](https://github.com/meshery/meshery/blob/master/CONTRIBUTING.md#signing-off-on-commits-developer-certificate-of-origin)." 28 | action: open 29 | pr: 30 | body: "🚨 Alert! Git Police! We couldn’t help but notice that one or more of your commits is missing a sign-off. _A what?_ A commit sign-off (your email address).\n\n 31 | To amend the commits in this PR with your signoff using the instructions provided in the DCO check. \n\n 32 | To configure your dev environment to automatically signoff on your commits in the future, see [these instructions](https://github.com/meshery/meshery/blob/master/CONTRIBUTING.md#signing-off-on-commits-developer-certificate-of-origin)." 33 | action: open 34 | - name: component/ui 35 | labeled: 36 | issue: 37 | body: This issue has been labeled with 'component/ui'. 🧰 Here are docs on [Contributing to Meshery UI](https://docs.meshery.io/project/contributing/contributing-ui). 🎨 Here is the [Meshery UI Figma File](https://www.figma.com/file/SMP3zxOjZztdOLtgN4dS2W/Meshery-UI?node-id=4%3A0) File. Lastly, here are docs on [Contributing to Meshery's End-to-End Tests Using Cypress](https://docs.meshery.io/project/contributing/contributing-cypress). 38 | action: open 39 | pr: 40 | body: This PR has been labeled with 'component/ui'. 🧰 Here are docs on [Contributing to Meshery UI](https://docs.meshery.io/project/contributing/contributing-ui). 🎨 Here is the [Meshery UI Figma File](https://www.figma.com/file/SMP3zxOjZztdOLtgN4dS2W/Meshery-UI?node-id=4%3A0) File. Lastly, here are docs on [Contributing to Meshery's End-to-End Tests Using Cypress](https://docs.meshery.io/project/contributing/contributing-cypress) 41 | action: open 42 | - name: component/mesheryctl 43 | labeled: 44 | issue: 45 | body: This issue has been labeled with 'component/mesheryctl'. Note that after making changes you need to update it in the [mesheryctl command tracker](https://docs.google.com/spreadsheets/d/1q63sIGAuCnIeDs8PeM-0BAkNj8BBgPUXhLbe1Y-318o/edit#gid=0) spreadsheet. 46 | action: open 47 | pr: 48 | body: This PR has been labeled with 'component/mesheryctl'. Note that after making changes you need to update it in the [mesheryctl command tracker](https://docs.google.com/spreadsheets/d/1q63sIGAuCnIeDs8PeM-0BAkNj8BBgPUXhLbe1Y-318o/edit#gid=0) spreadsheet. 49 | action: open 50 | # pr: 51 | # body: Hi, please note that this issue will need an approved design specification before implementation proceeds. See [Creating a Functional Specification](https://docs.google.com/document/d/1RP3IWLc-MiQS-QYasqCoVuCH7--G87p5ezE5f_nOzB8/edit?usp=sharing) to create a design spec. 52 | # action: open 53 | -------------------------------------------------------------------------------- /.github/labeler.yml: -------------------------------------------------------------------------------- 1 | area/ci: 2 | - changed-files: 3 | - any-glob-to-any-file: 4 | - ".github/**/*" 5 | - "install/*" 6 | area/docs: 7 | - changed-files: 8 | - any-glob-to-any-file: 9 | - "README.md" 10 | - "CONTRIBUTING.md" 11 | - "CONTRIBUTING-gitflow.md" 12 | language/go: 13 | - changed-files: 14 | - any-glob-to-any-file: 15 | - "meshsync/*" 16 | - "internal/**/*" 17 | - "pkg/**/*" -------------------------------------------------------------------------------- /.github/readme/images/community.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/readme/images/layer5-community-sign.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meshery/meshsync/9b393dc7ba653e2e92ed8f2dc414442598e06341/.github/readme/images/layer5-community-sign.png -------------------------------------------------------------------------------- /.github/readme/images/meshery-logo-dark-text-side.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/readme/images/meshery-logo-light-text-side.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/readme/images/meshsync.svg: -------------------------------------------------------------------------------- 1 | meshsync -------------------------------------------------------------------------------- /.github/readme/images/slack-128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meshery/meshsync/9b393dc7ba653e2e92ed8f2dc414442598e06341/.github/readme/images/slack-128.png -------------------------------------------------------------------------------- /.github/readme/images/slack-dark-128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meshery/meshsync/9b393dc7ba653e2e92ed8f2dc414442598e06341/.github/readme/images/slack-dark-128.png -------------------------------------------------------------------------------- /.github/release-drafter.yml: -------------------------------------------------------------------------------- 1 | name-template: 'MeshSync v$NEXT_PATCH_VERSION' 2 | tag-template: 'v$NEXT_PATCH_VERSION' 3 | categories: 4 | - title: '🚀 Features' 5 | labels: 6 | - 'kind/feature' 7 | - 'kind/enhancement' 8 | - title: '🐛 Bug Fixes' 9 | labels: 10 | - 'kind/fix' 11 | - 'kind/bugfix' 12 | - 'kind/bug' 13 | - title: '🧰 Maintenance' 14 | labels: 15 | - 'kind/chore' 16 | - 'area/ci' 17 | - 'area/tests' 18 | - title: 📖 Documentation 19 | label: area/docs 20 | change-template: '- $TITLE @$AUTHOR (#$NUMBER)' 21 | template: | 22 | ## What's New 23 | **General** 24 | $CHANGES 25 | 26 | ## Contributors 27 | 28 | Thank you to our contributors for making this release possible: 29 | $CONTRIBUTORS 30 | -------------------------------------------------------------------------------- /.github/security-insights.yml: -------------------------------------------------------------------------------- 1 | header: 2 | schema-version: 2.0.0 3 | last-updated: '2025-01-01' 4 | last-reviewed: '2025-01-01' 5 | url: https://github.com/meshery/meshsync/blob/master/.github/security-insights.yml 6 | comment: | 7 | This file contains the security insights information for the Meshery project. 8 | 9 | project: 10 | name: meshery 11 | administrators: 12 | - name: Lee Calcote 13 | affiliation: Meshery 14 | email: leecalcote@gmail.com 15 | primary: true 16 | repositories: 17 | - name: meshsync 18 | url: https://github.com/meshery/meshsync 19 | comment: | 20 | Website for Meshery. 21 | vulnerability-reporting: 22 | reports-accepted: true 23 | bug-bounty-available: false 24 | 25 | repository: 26 | url: https://github.com/meshery/meshsync 27 | status: active 28 | accepts-change-request: true 29 | accepts-automated-change-request: false 30 | core-team: 31 | - name: Lee Calcote 32 | affiliation: Meshery 33 | email: leecalcote@gmail.com 34 | primary: true 35 | license: 36 | url: https://github.com/meshery/meshsync/blob/master/LICENSE 37 | expression: https://github.com/meshery/meshsync/blob/master/LICENSE 38 | security: 39 | assessments: 40 | self: 41 | comment: | 42 | Self assessment has not yet been completed. 43 | -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | # Number of days of inactivity before an issue becomes stale 2 | daysUntilStale: 45 3 | # Number of days of inactivity before a stale issue is closed 4 | daysUntilClose: 10 5 | # Issues with these labels will never be considered stale 6 | exemptLabels: 7 | - issue/willfix 8 | # Label to use when marking an issue as stale 9 | staleLabel: issue/stale 10 | # Comment to post when marking an issue as stale. Set to `false` to disable 11 | markComment: > 12 | This issue has been automatically marked as stale because it has not had 13 | recent activity. It will be closed if no further activity occurs. Thank you 14 | for your contributions. 15 | # Comment to post when closing a stale issue. Set to `false` to disable 16 | closeComment: > 17 | This issue is being automatically closed due to inactivity. 18 | However, you may choose to reopen this issue. 19 | -------------------------------------------------------------------------------- /.github/welcome/meshery_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meshery/meshsync/9b393dc7ba653e2e92ed8f2dc414442598e06341/.github/welcome/meshery_dark.png -------------------------------------------------------------------------------- /.github/welcome/meshery_light.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meshery/meshsync/9b393dc7ba653e2e92ed8f2dc414442598e06341/.github/welcome/meshery_light.png -------------------------------------------------------------------------------- /.github/workflows/build-and-release.yml: -------------------------------------------------------------------------------- 1 | name: Meshsync Build and Releaser 2 | on: 3 | push: 4 | branches: 5 | - 'master' 6 | tags: 7 | - 'v*' 8 | paths-ignore: 9 | - 'docs/**' 10 | - '.github/**' 11 | jobs: 12 | build: 13 | name: Build check 14 | runs-on: ubuntu-22.04 15 | steps: 16 | - name: Check out code 17 | uses: actions/checkout@main 18 | with: 19 | fetch-depth: 1 20 | - name: Setup Go 21 | uses: actions/setup-go@main 22 | with: 23 | go-version: '1.23' 24 | check-latest: 'true' 25 | - run: GOPROXY=direct GOSUMDB=off GO111MODULE=on go build -o meshery-meshsync . 26 | docker: 27 | name: Docker build and push 28 | runs-on: ubuntu-22.04 29 | steps: 30 | - name: Check out code 31 | uses: actions/checkout@main 32 | with: 33 | fetch-depth: 1 34 | - name: Setup go 35 | uses: actions/setup-go@main 36 | with: 37 | go-version: '1.23' 38 | check-latest: 'true' 39 | - name: Docker login 40 | uses: docker/login-action@v2 41 | with: 42 | username: ${{ secrets.DOCKER_USERNAME }} 43 | password: ${{ secrets.DOCKER_PASSWORD }} 44 | - name: Docker edge build & tag 45 | if: startsWith(github.ref, 'refs/tags/') != true && success() 46 | run: | 47 | DOCKER_BUILDKIT=1 docker build --no-cache -t ${{ secrets.IMAGE_NAME }}:edge-latest --build-arg GIT_COMMITSHA=${GITHUB_SHA::8} --build-arg GIT_VERSION="edge-latest" . 48 | docker tag ${{ secrets.IMAGE_NAME }}:edge-latest ${{ secrets.IMAGE_NAME }}:edge-${GITHUB_SHA::7} 49 | - name: Docker edge push 50 | if: startsWith(github.ref, 'refs/tags/') != true && success() 51 | run: | 52 | docker push ${{ secrets.IMAGE_NAME }}:edge-latest 53 | docker push ${{ secrets.IMAGE_NAME }}:edge-${GITHUB_SHA::7} 54 | - name: Docker stable build & tag 55 | if: github.event_name != 'pull_request' && startsWith(github.ref, 'refs/tags/') && success() 56 | run: | 57 | DOCKER_BUILDKIT=1 docker build --no-cache -t ${{ secrets.IMAGE_NAME }}:stable-latest --build-arg GIT_COMMITSHA=${GITHUB_SHA::8} --build-arg GIT_VERSION=${GITHUB_REF/refs\/tags\//} . 58 | docker tag ${{ secrets.IMAGE_NAME }}:stable-latest ${{ secrets.IMAGE_NAME }}:stable-${GITHUB_REF/refs\/tags\//} 59 | docker tag ${{ secrets.IMAGE_NAME }}:stable-latest ${{ secrets.IMAGE_NAME }}:stable-${GITHUB_SHA::7} 60 | - name: Docker stable push 61 | if: github.event_name != 'pull_request' && startsWith(github.ref, 'refs/tags/') && success() 62 | run: | 63 | docker push ${{ secrets.IMAGE_NAME }}:stable-latest 64 | docker push ${{ secrets.IMAGE_NAME }}:stable-${GITHUB_REF/refs\/tags\//} 65 | docker push ${{ secrets.IMAGE_NAME }}:stable-${GITHUB_SHA::7} 66 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Meshsync CI 2 | 3 | on: 4 | # push: 5 | # branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | paths-ignore: 9 | - 'docs/**' 10 | - '.github/**' 11 | 12 | jobs: 13 | golangci-lint: 14 | strategy: 15 | matrix: 16 | platform: [ubuntu-24.04] 17 | go-version: [1.23.x] 18 | runs-on: ${{ matrix.platform }} 19 | steps: 20 | - name: Check out code 21 | uses: actions/checkout@master 22 | - name: Setup go 23 | uses: actions/setup-go@main 24 | with: 25 | go-version: ${{ matrix.go-version }} 26 | - name: golangci-lint 27 | uses: golangci/golangci-lint-action@v6 28 | with: 29 | version: v1.64 30 | args: --timeout=5m 31 | codecov: 32 | needs: golangci-lint 33 | name: Code coverage 34 | if: github.repository == 'meshery/meshsync' 35 | runs-on: ubuntu-24.04 36 | steps: 37 | - name: Checkout code 38 | uses: actions/checkout@master 39 | - name: Setup Go 40 | uses: actions/setup-go@v4 41 | with: 42 | go-version: '1.23.x' 43 | - name: Run unit tests 44 | run: go test --short ./... -race -coverprofile=coverage.txt -covermode=atomic 45 | - name: Upload coverage to Codecov 46 | if: github.repository == 'meshery/meshsync' 47 | uses: codecov/codecov-action@v3 48 | with: 49 | file: ./coverage.txt 50 | flags: unittests 51 | build: 52 | needs: [golangci-lint, codecov] 53 | name: Build 54 | runs-on: ubuntu-24.04 55 | steps: 56 | - name: Checkout code 57 | uses: actions/checkout@master 58 | - name: Setup Go 59 | uses: actions/setup-go@v4 60 | with: 61 | go-version: '1.23.x' 62 | - name: Build 63 | run: make build 64 | -------------------------------------------------------------------------------- /.github/workflows/error-codes-updater.yml: -------------------------------------------------------------------------------- 1 | name: Meshsync Error Codes Utility Runner 2 | on: 3 | push: 4 | branches: 5 | - 'master' 6 | paths: 7 | - '**.go' 8 | 9 | jobs: 10 | Update-error-codes: 11 | name: Error codes utility 12 | if: github.repository == 'meshery/meshsync' 13 | runs-on: ubuntu-22.04 14 | steps: 15 | - uses: actions/checkout@main 16 | # token here with write access to meshsync repo 17 | with: 18 | token: ${{ secrets.GH_ACCESS_TOKEN }} 19 | ref: 'master' 20 | 21 | - name: Setup Go 22 | uses: actions/setup-go@main 23 | with: 24 | go-version: '1.23' 25 | 26 | - name: Run utility 27 | run: | 28 | go get github.com/meshery/meshkit/cmd/errorutil 29 | go run github.com/meshery/meshkit/cmd/errorutil -d . update --skip-dirs meshery -i ./helpers -o ./helpers 30 | # to update errorutil* files in meshkit repo 31 | - name: Commit changes 32 | uses: stefanzweifel/git-auto-commit-action@v4 33 | with: 34 | commit_user_name: l5io 35 | commit_user_email: ci@meshery.io 36 | commit_author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> 37 | commit_options: '--signoff' 38 | commit_message: "run error codes utility" 39 | file_pattern: helpers/ **.go 40 | 41 | # to push changes to meshery docs 42 | - name: Checkout meshery 43 | uses: actions/checkout@main 44 | with: 45 | repository: 'meshery/meshery' 46 | # token with write access to meshery repository 47 | token: ${{ secrets.GH_ACCESS_TOKEN }} 48 | path: 'meshery' 49 | ref: 'master' 50 | 51 | - name: Update and push docs 52 | run: | 53 | echo '{ "errors_export": "" }' | jq --slurpfile export ./helpers/errorutil_errors_export.json '.errors_export = $export[0]' > ./meshery/docs/_data/errorref/meshsync_errors_export.json 54 | cd ./meshery 55 | git config user.name l5io 56 | git config user.email ci@meshery.io 57 | if git diff --exit-code --quiet 58 | then 59 | echo "No changes to commit" 60 | exit 61 | fi 62 | git add ./docs/_data/errorref/meshsync_errors_export.json 63 | git commit -m "[Docs] Error Code Reference: Updated codes for MeshSync" --signoff 64 | git push origin master 65 | -------------------------------------------------------------------------------- /.github/workflows/integration-tests-ci.yml: -------------------------------------------------------------------------------- 1 | name: Integration Tests 2 | on: 3 | push: 4 | branches: 5 | - "master" 6 | paths: 7 | - "**.go" 8 | - "**.golden" 9 | - "Makefile" 10 | - ".github/workflows/integration-tests-ci.yml" 11 | pull_request: 12 | branches: 13 | - "master" 14 | paths: 15 | - "**.go" 16 | - "**.golden" 17 | - "Makefile" 18 | - ".github/workflows/integration-tests-ci.yml" 19 | workflow_dispatch: 20 | inputs: 21 | logLevel: 22 | description: "Log level" 23 | required: true 24 | default: "warning" 25 | 26 | jobs: 27 | integration-tests: 28 | name: Integration tests 29 | runs-on: ubuntu-24.04 30 | steps: 31 | - uses: actions/checkout@v4 32 | with: 33 | fetch-depth: 2 34 | - name: Setup Go 35 | uses: actions/setup-go@v5 36 | with: 37 | go-version: "1.23" 38 | - name: Install Docker Compose 39 | run: | 40 | sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose 41 | sudo chmod +x /usr/local/bin/docker-compose 42 | - name: Set up Docker Buildx 43 | uses: docker/setup-buildx-action@v3 44 | - name: Install Kind and kubectl 45 | uses: helm/kind-action@v1.10.0 46 | - name: Integration tests set up 47 | run: make integration-tests-setup 48 | - name: Integration tests run 49 | run: make integration-tests-run 50 | - name: Integration tests clean up 51 | run: make integration-tests-cleanup 52 | - name: Upload meshsync command output 53 | uses: actions/upload-artifact@v4 54 | if: always() 55 | with: 56 | name: meshsync-command-output 57 | path: integration-tests/*.meshsync-output.txt 58 | if-no-files-found: warn 59 | -------------------------------------------------------------------------------- /.github/workflows/label-commenter.yml: -------------------------------------------------------------------------------- 1 | name: Label Commenter 2 | 3 | on: 4 | issues: 5 | types: 6 | - labeled 7 | 8 | pull_request_target: 9 | types: 10 | - labeled 11 | 12 | permissions: 13 | contents: read 14 | issues: write 15 | pull-requests: write 16 | 17 | jobs: 18 | comment: 19 | runs-on: ubuntu-22.04 20 | steps: 21 | - uses: actions/checkout@main 22 | with: 23 | ref: master # Set your default branch 24 | 25 | - name: Label Commenter 26 | uses: peaceiris/actions-label-commenter@v1 27 | -------------------------------------------------------------------------------- /.github/workflows/labeler.yml: -------------------------------------------------------------------------------- 1 | name: "Pull Request Labeler" 2 | on: 3 | - pull_request_target 4 | 5 | jobs: 6 | triage: 7 | permissions: 8 | contents: read 9 | pull-requests: write 10 | runs-on: ubuntu-24.04 11 | steps: 12 | - uses: actions/labeler@v5 13 | with: 14 | repo-token: "${{ secrets.GITHUB_TOKEN }}" 15 | -------------------------------------------------------------------------------- /.github/workflows/multi-platform.yml: -------------------------------------------------------------------------------- 1 | name: Multi-Platform Build and Release 2 | on: 3 | release: 4 | types: [published] 5 | push: 6 | tags: 7 | - 'v*' 8 | branches: 9 | - 'master' 10 | paths-ignore: 11 | - 'docs/**' 12 | - '.github/**' 13 | workflow_dispatch: 14 | inputs: 15 | release-ver: 16 | description: 'Stable Release Version' 17 | required: true 18 | default: 'v' 19 | stripped-release-ver: 20 | description: 'Stripped Stable Release Version' 21 | required: true 22 | default: '' 23 | release-channel: 24 | description: 'Release Channel' 25 | required: true 26 | default: 'edge' 27 | 28 | env: 29 | GIT_VERSION: ${{github.event.inputs.release-ver}} 30 | GIT_STRIPPED_VERSION: ${{github.event.inputs.stripped-release-ver}} 31 | RELEASE_CHANNEL: ${{github.event.inputs.release-channel}} 32 | GIT_TAG: ${{ github.event.release.tag_name }} 33 | 34 | jobs: 35 | print-inputs: 36 | runs-on: ubuntu-22.04 37 | steps: 38 | 39 | - run: | 40 | echo "Dispatched GIT_VERSION: ${{github.event.inputs.release-ver}}" 41 | echo " Dispatched GIT_STRIPPED_VERSION: ${{github.event.inputs.stripped-release-ver}}" 42 | echo "Env RELEASE_CHANNEL: ${{env.RELEASE_CHANNEL}}" 43 | echo "Env GIT_VERSION: ${{env.GIT_VERSION}}" 44 | echo "Env GIT_STRIPPED_VERSION: ${{env.GIT_STRIPPED_VERSION}}" 45 | echo "Env GIT_TAG: ${{ github.event.release.tag_name }}" 46 | 47 | docker-build: 48 | runs-on: ubuntu-22.04 49 | steps: 50 | - 51 | name: Checkout repo 52 | uses: actions/checkout@main 53 | - 54 | name: Identify Release Values 55 | if: "${{ github.event.inputs.release-ver}} != 'v' }}" 56 | run: | 57 | # GIT_REF=`git symbolic-ref HEAD` 58 | if [[ $GIT_TAG = refs/tags* ]] 59 | then 60 | echo RELEASE_CHANNEL=stable >> $GITHUB_ENV 61 | else 62 | echo RELEASE_CHANNEL=edge >> $GITHUB_ENV 63 | fi 64 | echo "Release channel determined to be $RELEASE_CHANNEL" 65 | LATEST_VERSION=$(git ls-remote --tags | tail -1 | cut -f2 | sed 's/refs\/tags\///g') >> $GITHUB_ENV 66 | GIT_VERSION=$(git ls-remote --tags | tail -1 | cut -f2 | sed 's/refs\/tags\///g') >> $GITHUB_ENV 67 | # GIT_VERSION=$(git describe --tags `git rev-list --tags --max-count=1` --always) 68 | GIT_STRIPPED_VERSION=$(git ls-remote --tags | tail -1 | cut -f2 | sed 's/refs\/tags\///g' | cut -c2-) 69 | echo "GIT_LATEST=$LATEST_VERSION" >> $GITHUB_ENV 70 | echo "GIT_VERSION=$GIT_VERSION" >> $GITHUB_ENV 71 | echo "GIT_STRIPPED_VERSION=$GIT_STRIPPED_VERSION" >> $GITHUB_ENV 72 | shell: bash 73 | 74 | - 75 | name: Set up QEMU 76 | uses: docker/setup-qemu-action@v1 77 | - 78 | name: Set up Docker Buildx 79 | uses: docker/setup-buildx-action@v1 80 | - 81 | name: Docker Meta 82 | id: meta 83 | uses: docker/metadata-action@v3 84 | with: 85 | images: ${{ secrets.IMAGE_NAME }} 86 | flavor: | 87 | latest=false 88 | tags: | 89 | type=raw,value=${{env.RELEASE_CHANNEL}}-{{sha}} 90 | type=semver,pattern={{version}},value=${{env.GIT_STRIPPED_VERSION}} 91 | type=raw,pattern={{version}},value=${{env.RELEASE_CHANNEL}}-${{env.GIT_VERSION}} 92 | type=raw,value=${{env.RELEASE_CHANNEL}}-{{tag}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} 93 | type=raw,value=${{env.RELEASE_CHANNEL}}-latest 94 | type=raw,value=${{env.RELEASE_CHANNEL}}-${{env.GIT_VERSION}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} 95 | - 96 | name: Login to DockerHub 97 | uses: docker/login-action@v2 98 | with: 99 | username: ${{ secrets.DOCKER_USERNAME }} 100 | password: ${{ secrets.DOCKER_PASSWORD }} 101 | - 102 | name: Build and Push 103 | uses: docker/build-push-action@v2 104 | with: 105 | context: "{{defaultContext}}" 106 | push: true 107 | build-args: | 108 | GIT_STRIPPED_VERSION=${{env.GIT_STRIPPED_VERSION}} 109 | GIT_VERSION=${{env.GIT_VERSION}} 110 | RELEASE_CHANNEL=${{env.RELEASE_CHANNEL}} 111 | tags: ${{ steps.meta.outputs.tags }} 112 | platforms: linux/amd64,linux/arm64 113 | 114 | # - 115 | # name: Docker Hub Description 116 | # uses: peter-evans/dockerhub-description@v3 117 | # with: 118 | # username: ${{ secrets.DOCKER_USERNAME }} 119 | # password: ${{ secrets.DOCKER_PASSWORD }} 120 | # repository: meshery/meshery-docker-extension 121 | # readme-filepath: /install/docker-extension/README.md 122 | -------------------------------------------------------------------------------- /.github/workflows/newcomers-alert.yml: -------------------------------------------------------------------------------- 1 | name: Newcomers Alert 2 | on: 3 | issues: 4 | types: [labeled] 5 | jobs: 6 | good-first-issue-notify: 7 | if: github.event.label.name == 'good first issue' || github.event.label.name == 'first-timers-only' 8 | name: Notify Slack for new good-first-issue 9 | runs-on: ubuntu-22.04 10 | steps: 11 | - name: Notify slack 12 | env: 13 | SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} 14 | uses: pullreminders/slack-action@master 15 | with: 16 | args: '{\"channel\":\"C019426UBNY\",\"text\":\"A good first issue label was just added to ${{github.event.issue.html_url}}.\"}' 17 | -------------------------------------------------------------------------------- /.github/workflows/release-drafter.yml: -------------------------------------------------------------------------------- 1 | name: Release Drafter 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - master 8 | # paths-ignore: 9 | # - '.github/**' 10 | 11 | permissions: 12 | contents: read 13 | pull-requests: write 14 | 15 | jobs: 16 | update_release_draft: 17 | if: github.repository == 'meshery/meshsync' 18 | runs-on: ubuntu-24.04 19 | steps: 20 | - name: Drafting release 21 | id: release_drafter 22 | uses: release-drafter/release-drafter@v5 23 | with: 24 | config-name: release-drafter.yml 25 | env: 26 | GITHUB_TOKEN: ${{ secrets.RELEASE_NOTES_PAT }} 27 | -------------------------------------------------------------------------------- /.github/workflows/slack.yml: -------------------------------------------------------------------------------- 1 | name: Slack Notify 2 | on: 3 | watch: 4 | types: [started] 5 | jobs: 6 | star-notify: 7 | if: github.event_name == 'watch' 8 | name: Notify Slack on star 9 | runs-on: ubuntu-22.04 10 | steps: 11 | - name: Get current star count 12 | run: | 13 | echo "STARS=$(curl --silent 'https://api.github.com/repos/${{github.repository}}' -H 'Accept: application/vnd.github.preview' | jq '.stargazers_count')" >> $GITHUB_ENV 14 | - name: Notify slack 15 | env: 16 | SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} 17 | uses: pullreminders/slack-action@master 18 | with: 19 | args: '{\"channel\":\"CSK7N9TGX\",\"text\":\"${{ github.actor }} just starred ${{github.repository}}! (https://github.com/${{github.repository}}/stargazers) Total ⭐️: ${{env.STARS}}\"}' 20 | good-first-issue-notify: 21 | if: github.event_name == 'issues' && github.event.label.name == 'good first issue' || github.event.label.name == 'first-timers-only' 22 | name: Notify Slack for new good-first-issue 23 | runs-on: ubuntu-22.04 24 | steps: 25 | - name: Notify slack 26 | env: 27 | SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} 28 | uses: pullreminders/slack-action@master 29 | with: 30 | args: '{\"channel\":\"C019426UBNY\",\"text\":\"A good first issue label was just added to ${{github.event.issue.html_url}}.\"}' 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | .idea 17 | 18 | **errorutil_analyze_errors.json 19 | **errorutil_analyze_summary.json 20 | **errorutil_errors_export.json 21 | 22 | .vscode/ 23 | cover.html 24 | .DS_Store 25 | 26 | bin/meshsync 27 | meshery-cluster-snapshot-* -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters-settings: 2 | cyclop: 3 | package-average: 7 4 | govet: 5 | enable: 6 | - shadow 7 | - tests 8 | 9 | # Configuration for golangci-lint that is suitable for a Kubernetes operator project built with Golang 10 | linters: 11 | enable-all: false 12 | disable-all: false 13 | enable: 14 | - gci 15 | - goconst 16 | - gocritic 17 | - govet 18 | - unused 19 | - cyclop 20 | 21 | issues: 22 | exclude-dirs: 23 | - vendor 24 | - bundle 25 | - config 26 | - hack 27 | - helpers 28 | - img 29 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Meshery Community Code of Conduct 2 | 3 | The Meshery community follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). 4 | 5 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting maintainers@meshery.io. 6 | -------------------------------------------------------------------------------- /CONTRIBUTING-gitflow.md: -------------------------------------------------------------------------------- 1 | # Working by Forking 2 | Just head over to the GitHub page and click the "Fork" button. It's just that simple. Once you've done that, you can use your favorite git client to clone your repo or just head straight to the command line: 3 | 4 | ## Clone your fork to your local machine 5 | ``` 6 | git clone git@github.com:USERNAME/FORKED-PROJECT.git 7 | ``` 8 | Keeping Your Fork Up to Date 9 | While this isn't an absolutely necessary step, if you plan on doing anything more than just a tiny quick fix, you'll want to make sure you keep your fork up to date by tracking the original "upstream" repo that you forked. To do this, you'll need to add a remote: 10 | 11 | ## Add 'upstream' repo to list of remotes 12 | ``` 13 | git remote add upstream https://github.com/meshery/meshery.git 14 | ``` 15 | ("meshery" is used as the example repo. Be sure to reference the _actual_ repo you're contributing to e.g. "meshery-linkerd"). 16 | 17 | ## Verify the new remote named 'upstream' 18 | ``` 19 | git remote -v 20 | ``` 21 | Whenever you want to update your fork with the latest upstream changes, you'll need to first fetch the upstream repo's branches and latest commits to bring them into your repository: 22 | 23 | ## Fetch from upstream remote 24 | ``` 25 | git fetch upstream 26 | ``` 27 | 28 | ## View all branches, including those from upstream 29 | ``` 30 | git branch -va 31 | ``` 32 | Now, checkout your own master branch and merge the upstream repo's master branch: 33 | 34 | ## Checkout your master branch and merge upstream 35 | ``` 36 | git checkout master 37 | git merge upstream/master 38 | ``` 39 | If there are no unique commits on the local master branch, git will simply perform a fast-forward. However, if you have been making changes on master (in the vast majority of cases you probably shouldn't be - see the next section, you may have to deal with conflicts. When doing so, be careful to respect the changes made upstream. 40 | 41 | Now, your local master branch is up-to-date with everything modified upstream. 42 | 43 | **Create a Branch** (doing your work) 44 | Whenever you begin work on a new feature or bugfix, it's important that you create a new branch. Not only is it proper git workflow, but it also keeps your changes organized and separated from the master branch so that you can easily submit and manage multiple pull requests for every task you complete. 45 | 46 | To create a new branch and start working on it, peform the following flow. 47 | 48 | ## Checkout the master branch - you want your new branch to come from master 49 | ``` 50 | git checkout master 51 | ``` 52 | 53 | ## Create a new branch (give your branch its own simple informative name) 54 | For enhancements use `feature/your_username/issue#` or `feature/your_username/name_of_feature` 55 | 56 | For bugs use `bug/your_username/issue#` or `bug/your_username/name_of_bug` 57 | 58 | ``` 59 | git branch feature/jdoe/567 60 | ``` 61 | 62 | ## Switch to your new branch 63 | ``` 64 | git checkout feature/jdoe/567 65 | ``` 66 | Now, go to town hacking away and making whatever changes you want to. 67 | 68 | ## Submitting your changes (a Pull Request) 69 | Prior to submitting your pull request, you might want to do a few things to clean up your branch and make it as simple as possible for the original repo's maintainer to test, accept, and merge your work. 70 | 71 | In the time that you've been working on your changes, if any commits have been made to the upstream master branch, you will need to rebase your development branch so that merging it will be a simple fast-forward that won't require any conflict resolution work. 72 | 73 | ## Fetch upstream master and merge with your repo's master branch 74 | ``` 75 | git fetch upstream 76 | git checkout master 77 | git merge upstream/master 78 | ``` 79 | 80 | ## If there were any new commits, rebase your development branch 81 | ``` 82 | git checkout feature/jdoe/567 83 | git rebase master 84 | ``` 85 | Now, it may be desirable to squash some of your smaller commits down into a small number of larger more cohesive commits. You can do this with an interactive rebase: 86 | 87 | ## Rebase all commits on your development branch 88 | ``` 89 | git checkout 90 | git rebase -i master 91 | ``` 92 | This will open up a text editor where you can specify which commits to squash. 93 | 94 | ## Submitting 95 | Once you've committed and pushed all of your changes to GitHub, go to the page for your fork on GitHub, select your development branch, and click the pull request button. If you need to make any adjustments to your pull request, just push the updates to GitHub. Your pull request will automatically track the changes on your development branch and update. 96 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Overview 2 | 3 | Please do! Thank you for your help in improving Meshery! :balloon: 4 | 5 | --- 6 | 7 |

Find the complete set of contributor guides at https://docs.meshery.io/project/contributing

8 | 9 | --- 10 | 11 | All contributors are welcome. Not sure where to start? Please see the [newcomers welcome guide](https://meshery.io/community/newcomers) for how, where, and why to contribute. This project is community-built and welcomes collaboration. Contributors are expected to adhere to our [Code of Conduct](CODE_OF_CONDUCT.md). 12 | 13 | All set to contribute? Grab an open issue with the [help-wanted label](../../labels/help%20wanted) and jump in. Join our [Slack channel](https://slack.meshery.io) and engage in conversation. Create a [new issue](/../../issues/new/choose) if needed. All [pull requests](/../../pulls) should ideally reference an open [issue](/../../issues). Include keywords in your pull request descriptions, as well as commit messages, to [automatically close related issues in GitHub](https://help.github.com/en/github/managing-your-work-on-github/closing-issues-using-keywords). 14 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.23 AS builder 2 | ARG GIT_VERSION 3 | ARG GIT_COMMITSHA 4 | 5 | WORKDIR /build 6 | # Copy the Go Modules manifests 7 | COPY go.mod go.mod 8 | COPY go.sum go.sum 9 | # cache deps before building and copying source so that we don't need to re-download as much 10 | # and so that source changes don't invalidate our downloaded layer 11 | RUN go mod download 12 | # Copy the go source 13 | COPY . . 14 | # Build 15 | RUN CGO_ENABLED=0 GO111MODULE=on go build -ldflags="-w -s -X main.version=$GIT_VERSION -X main.commitsha=$GIT_COMMITSHA" -a -o meshery-meshsync main.go 16 | 17 | # Use distroless as minimal base image to package the manager binary 18 | # Refer to https://github.com/GoogleContainerTools/distroless for more details 19 | FROM gcr.io/distroless/base-debian10 20 | WORKDIR / 21 | ENV GODISTRO="debian" 22 | COPY --from=builder /build/meshery-meshsync . 23 | ENTRYPOINT ["/meshery-meshsync"] 24 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright Meshery Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) 16 | 17 | include install/Makefile.core.mk 18 | include install/Makefile.show-help.mk 19 | 20 | CURRENT_DIR:=$(shell pwd) 21 | MESHSYNC_BINARY_TARGET_RELATIVE:=bin/meshsync 22 | MESHSYNC_BINARY_TARGET_ABSOLUTE:=$(CURRENT_DIR)/$(MESHSYNC_BINARY_TARGET_RELATIVE) 23 | INTEGRATION_TESTS_DIR:=$(CURRENT_DIR)/integration-tests 24 | 25 | ifeq (,$(shell go env GOBIN)) 26 | GOBIN=$(shell go env GOPATH)/bin 27 | else 28 | GOBIN=$(shell go env GOBIN) 29 | endif 30 | 31 | #----------------------------------------------------------------------------- 32 | # Docker-based Builds 33 | #----------------------------------------------------------------------------- 34 | .PHONY: docker-check 35 | ## Build Meshsync's docker image 36 | docker: check 37 | docker build -t meshery/meshery-meshsync . 38 | 39 | .PHONY: docker-run 40 | ## Runs Meshsync in docker 41 | docker-run: 42 | (docker rm -f meshery-meshsync) || true 43 | docker run --name meshery-meshsync -d \ 44 | -p 10007:10007 \ 45 | -e DEBUG=true \ 46 | meshery/meshery-meshsync 47 | 48 | PHONY: nats 49 | ## Runs a local instance of NATS server in detached mode 50 | nats: 51 | (docker rm -f nats) || true 52 | docker run --name nats --rm -p 4222:4222 -p 8222:8222 -d nats --http_port 8222 53 | 54 | #----------------------------------------------------------------------------- 55 | # Local Builds 56 | #----------------------------------------------------------------------------- 57 | .PHONY: build 58 | ## Build Meshsync binary to $(MESHSYNC_BINARY_TARGET_RELATIVE) 59 | build: 60 | go build -o $(MESHSYNC_BINARY_TARGET_RELATIVE) main.go 61 | 62 | .PHONY: run-check 63 | ## Runs local instance of Meshsync: can be used during local development 64 | run: nats 65 | go$(v) mod tidy; \ 66 | DEBUG=true GOPROXY=direct GOSUMDB=off go run main.go 67 | 68 | .PHONY: check 69 | ## Lint check Meshsync. 70 | check: 71 | $(GOBIN)/golangci-lint run ./... 72 | 73 | .PHONY: go-mod-tidy 74 | ## Run go mod tidy for dependency management 75 | go-mod-tidy: 76 | go mod tidy 77 | 78 | #----------------------------------------------------------------------------- 79 | # Tests 80 | #----------------------------------------------------------------------------- 81 | 82 | # Test covergae 83 | .PHONY: coverage 84 | ## Runs coverage tests for Meshsync 85 | coverage: 86 | go test -v ./... -coverprofile cover.out 87 | go tool cover -html=cover.out -o cover.html 88 | ## Runs unit tests 89 | test: check 90 | go test -failfast --short ./... -race 91 | ## Lint check Golang 92 | lint: 93 | golangci-lint run ./... 94 | 95 | ## Runs integration tests check dependencies (if docker, kind, kubectl are present) 96 | integration-tests-check-dependencies: 97 | ./integration-tests/setup.sh check_dependencies 98 | 99 | ## Runs integration tests set up (runs docker compose with nats and creates a test kind cluster) 100 | ## docker compose exposes nats on default ports to host, so they must be available 101 | integration-tests-setup: 102 | ./integration-tests/infrastructure/setup.sh setup 103 | 104 | ## Runs integration tests clean up (stops docker compose and deletes test cluster) 105 | integration-tests-cleanup: 106 | ./integration-tests/infrastructure/setup.sh cleanup 107 | 108 | ## Runs integration tests 109 | integration-tests-run: build 110 | RUN_INTEGRATION_TESTS=true \ 111 | MESHSYNC_BINARY_PATH=$(MESHSYNC_BINARY_TARGET_ABSOLUTE) \ 112 | SAVE_MESHSYNC_OUTPUT=true \ 113 | go test -v -count=1 -run Integration $(INTEGRATION_TESTS_DIR) 114 | 115 | ## Runs integration tests full cycle (setup, run, cleanup) 116 | integration-tests: integration-tests-setup integration-tests-run integration-tests-cleanup 117 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 | 4 | Meshery Logo

6 |

7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 34 |

35 | 36 | # MeshSync 37 | 38 | MeshSync, an event-driven, continuous discovery and synchronization engine performs the task of ensuring that the state of configuration and status of operation of any supported Meshery platform (e.g. Kubernetes) and environment are known to Meshery Server. When deployed into Kubernetes enviroments, MeshSync runs as a Kubernetes custom controller under the control of Meshery Operator. 39 | 40 | See [MeshSync in Meshery Docs](https://docs.meshery.io/concepts/architecture/meshsync) for additional information. 41 | 42 | ---- 43 | 44 | Could be run in two modes: 45 | - nats (default) 46 | - file 47 | 48 | See details on input params in command help output: 49 | ```sh 50 | meshsync --help 51 | ``` 52 | 53 | 54 | ## NATS mode 55 | NATS mode is the default mode. 56 | 57 | In that mode, MeshSync expects a NATS connection and outputs Kubernetes resources updates into NATS queue, which is how MeshSync runs when deployed in Kubernetes cluster in conjuction with Meshery Broker. 58 | 59 | ## File mode 60 | File mode is an option to run meshsync without dependency on nats and CRD. 61 | 62 | In that mode meshsync outputs k8s resources updates into file in kubernetes manifest yaml format. 63 | 64 | The result of run is two files: 65 | - meshery-cluster-snapshot-YYYYMMDD-00.yaml 66 | - meshery-cluster-snapshot-YYYYMMDD-00-extended.yaml 67 | 68 | meshery-cluster-snapshot-YYYYMMDD-00-extended.yaml contains all events meshsync produces as output; 69 | 70 | meshery-cluster-snapshot-YYYYMMDD-00.yaml contains a deduplicated version where each resource is presented with one entity. 71 | Deduplication is done by `metadata.uid` field. 72 | 73 | 74 | ### Notes (on file mode) 75 | Right now the format of the generated files is very close to kubernetes manifest yaml format, but not exactly the same. 76 | 77 | Generated files contain `metadata.labels` as array while in kubernetes manifest it should be an object. 78 | 79 | It is due to the format of [KubernetesResourceObjectMeta](pkg/model/model.go#52). 80 | 81 | `kubectl apply --dry-run` returns corresponding error: 82 | 83 | ```sh 84 | kubectl apply --dry-run=client -f meshery-cluster-snapshot-YYYYMMDD-00.yaml 85 | 86 | unable to decode "meshery-cluster-snapshot-YYYYMMDD-00.yaml": json: cannot unmarshal array into Go struct field ObjectMeta.metadata.labels of type map[string]string 87 | ``` 88 | 89 |
 
90 | 91 | ## Join the Meshery community 92 | 93 | 94 | Our projects are community-built and welcome collaboration. 👍 Be sure to see the Contributor Journey Map and Community Handbook for a tour of resources available to you and the Repository Overview for a cursory description of repository by technology and programming language. Jump into community Slack or discussion forum to participate. 95 | 96 |

97 |

Find your MeshMate

98 | 99 |

MeshMates are experienced Meshery community members, who will help you learn your way around, discover live projects, and expand your community network. Connect with a MeshMate today!

100 | 101 | Learn more about the MeshMates program.
102 | 103 |

104 |

105 |
106 |
107 | Meshery Community 108 |
109 |
110 |

111 | ✔️ Join any or all of the weekly meetings on community calendar.
112 | ✔️ Watch community meeting recordings.
113 | ✔️ Fill-in a member form and gain access to community resources. 114 |
115 | ✔️ Discuss in the community forum.
116 | ✔️ Explore more in the community handbook.
117 |

118 |


119 |
120 | 121 | 122 | 123 | 124 | Shows an illustrated light mode meshery logo in light color mode and a dark mode meshery logo dark color mode. 125 | 126 | 127 |
128 |
129 |

130 |

131 |      Not sure where to start? Grab an open issue with the help-wanted label. 132 |

133 |

134 | 135 |
 
136 | 137 | ## Contributing 138 | 139 | Please do! We're a warm and welcoming community of open source contributors. Please join. All types of contributions are welcome. Be sure to read the [Contributor Guides](https://docs.meshery.io/project/contributing) for a tour of resources available to you and how to get started. 140 | 141 | 142 | 143 |
 
144 | 145 | ### License 146 | 147 | This repository and site are available as open-source under the terms of the [Apache 2.0 License](https://opensource.org/licenses/Apache-2.0). -------------------------------------------------------------------------------- /all-certmgs.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meshery/meshsync/9b393dc7ba653e2e92ed8f2dc414442598e06341/all-certmgs.yaml -------------------------------------------------------------------------------- /cert-manager-cainjector-c7d4dbdd9-jlstt.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | creationTimestamp: "2024-04-01T04:29:32Z" 5 | generateName: cert-manager-cainjector-c7d4dbdd9- 6 | labels: 7 | app: cainjector 8 | app.kubernetes.io/component: cainjector 9 | app.kubernetes.io/instance: cert-manager 10 | app.kubernetes.io/managed-by: Helm 11 | app.kubernetes.io/name: cainjector 12 | app.kubernetes.io/version: v1.14.4 13 | helm.sh/chart: cert-manager-v1.14.4 14 | pod-template-hash: c7d4dbdd9 15 | name: cert-manager-cainjector-c7d4dbdd9-jlstt 16 | namespace: cert-manager 17 | ownerReferences: 18 | - apiVersion: apps/v1 19 | blockOwnerDeletion: true 20 | controller: true 21 | kind: ReplicaSet 22 | name: cert-manager-cainjector-c7d4dbdd9 23 | uid: 4ad5f4b4-6ddc-4bad-8208-418435b3b8d7 24 | resourceVersion: "4476" 25 | uid: b905aeb8-3633-4ca9-b350-df3a87cdd60e 26 | spec: 27 | containers: 28 | - args: 29 | - --v=2 30 | - --leader-election-namespace=kube-system 31 | env: 32 | - name: POD_NAMESPACE 33 | valueFrom: 34 | fieldRef: 35 | apiVersion: v1 36 | fieldPath: metadata.namespace 37 | image: quay.io/jetstack/cert-manager-cainjector:v1.14.4 38 | imagePullPolicy: IfNotPresent 39 | name: cert-manager-cainjector 40 | resources: {} 41 | securityContext: 42 | allowPrivilegeEscalation: false 43 | capabilities: 44 | drop: 45 | - ALL 46 | readOnlyRootFilesystem: true 47 | terminationMessagePath: /dev/termination-log 48 | terminationMessagePolicy: File 49 | volumeMounts: 50 | - mountPath: /var/run/secrets/kubernetes.io/serviceaccount 51 | name: kube-api-access-sktmn 52 | readOnly: true 53 | dnsPolicy: ClusterFirst 54 | enableServiceLinks: false 55 | nodeName: c3-medium-x86-03-meshery 56 | nodeSelector: 57 | kubernetes.io/os: linux 58 | preemptionPolicy: PreemptLowerPriority 59 | priority: 0 60 | restartPolicy: Always 61 | schedulerName: default-scheduler 62 | securityContext: 63 | runAsNonRoot: true 64 | seccompProfile: 65 | type: RuntimeDefault 66 | serviceAccount: cert-manager-cainjector 67 | serviceAccountName: cert-manager-cainjector 68 | terminationGracePeriodSeconds: 30 69 | tolerations: 70 | - effect: NoExecute 71 | key: node.kubernetes.io/not-ready 72 | operator: Exists 73 | tolerationSeconds: 300 74 | - effect: NoExecute 75 | key: node.kubernetes.io/unreachable 76 | operator: Exists 77 | tolerationSeconds: 300 78 | volumes: 79 | - name: kube-api-access-sktmn 80 | projected: 81 | defaultMode: 420 82 | sources: 83 | - serviceAccountToken: 84 | expirationSeconds: 3607 85 | path: token 86 | - configMap: 87 | items: 88 | - key: ca.crt 89 | path: ca.crt 90 | name: kube-root-ca.crt 91 | - downwardAPI: 92 | items: 93 | - fieldRef: 94 | apiVersion: v1 95 | fieldPath: metadata.namespace 96 | path: namespace 97 | status: 98 | conditions: 99 | - lastProbeTime: null 100 | lastTransitionTime: "2024-04-01T04:29:34Z" 101 | status: "True" 102 | type: PodReadyToStartContainers 103 | - lastProbeTime: null 104 | lastTransitionTime: "2024-04-01T04:29:32Z" 105 | status: "True" 106 | type: Initialized 107 | - lastProbeTime: null 108 | lastTransitionTime: "2024-04-01T04:29:34Z" 109 | status: "True" 110 | type: Ready 111 | - lastProbeTime: null 112 | lastTransitionTime: "2024-04-01T04:29:34Z" 113 | status: "True" 114 | type: ContainersReady 115 | - lastProbeTime: null 116 | lastTransitionTime: "2024-04-01T04:29:32Z" 117 | status: "True" 118 | type: PodScheduled 119 | containerStatuses: 120 | - containerID: containerd://1c7b0a11d4c38a0fce9c3abf2b69358eae486bf383344a18b730db6bf5e135a4 121 | image: quay.io/jetstack/cert-manager-cainjector:v1.14.4 122 | imageID: quay.io/jetstack/cert-manager-cainjector@sha256:30286297e5b4b71a86759d297a8109c6a1649fdc68d28f618d87edf12a2da417 123 | lastState: {} 124 | name: cert-manager-cainjector 125 | ready: true 126 | restartCount: 0 127 | started: true 128 | state: 129 | running: 130 | startedAt: "2024-04-01T04:29:33Z" 131 | hostIP: 139.178.83.85 132 | hostIPs: 133 | - ip: 139.178.83.85 134 | phase: Running 135 | podIP: 192.168.0.14 136 | podIPs: 137 | - ip: 192.168.0.14 138 | qosClass: BestEffort 139 | startTime: "2024-04-01T04:29:32Z" 140 | -------------------------------------------------------------------------------- /cert-manager-webhook-847d7676c9-89rtq.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | creationTimestamp: "2024-04-01T04:29:32Z" 5 | generateName: cert-manager-webhook-847d7676c9- 6 | labels: 7 | app: webhook 8 | app.kubernetes.io/component: webhook 9 | app.kubernetes.io/instance: cert-manager 10 | app.kubernetes.io/managed-by: Helm 11 | app.kubernetes.io/name: webhook 12 | app.kubernetes.io/version: v1.14.4 13 | helm.sh/chart: cert-manager-v1.14.4 14 | pod-template-hash: 847d7676c9 15 | name: cert-manager-webhook-847d7676c9-89rtq 16 | namespace: cert-manager 17 | ownerReferences: 18 | - apiVersion: apps/v1 19 | blockOwnerDeletion: true 20 | controller: true 21 | kind: ReplicaSet 22 | name: cert-manager-webhook-847d7676c9 23 | uid: 9ca18977-52e3-474e-93e7-cbcaba9a13fa 24 | resourceVersion: "47870864" 25 | uid: 4b127dff-feeb-4962-81b2-27b87d2a1418 26 | spec: 27 | containers: 28 | - args: 29 | - --v=2 30 | - --secure-port=10250 31 | - --dynamic-serving-ca-secret-namespace=$(POD_NAMESPACE) 32 | - --dynamic-serving-ca-secret-name=cert-manager-webhook-ca 33 | - --dynamic-serving-dns-names=cert-manager-webhook 34 | - --dynamic-serving-dns-names=cert-manager-webhook.$(POD_NAMESPACE) 35 | - --dynamic-serving-dns-names=cert-manager-webhook.$(POD_NAMESPACE).svc 36 | env: 37 | - name: POD_NAMESPACE 38 | valueFrom: 39 | fieldRef: 40 | apiVersion: v1 41 | fieldPath: metadata.namespace 42 | image: quay.io/jetstack/cert-manager-webhook:v1.14.4 43 | imagePullPolicy: IfNotPresent 44 | livenessProbe: 45 | failureThreshold: 3 46 | httpGet: 47 | path: /livez 48 | port: 6080 49 | scheme: HTTP 50 | initialDelaySeconds: 60 51 | periodSeconds: 10 52 | successThreshold: 1 53 | timeoutSeconds: 1 54 | name: cert-manager-webhook 55 | ports: 56 | - containerPort: 10250 57 | name: https 58 | protocol: TCP 59 | - containerPort: 6080 60 | name: healthcheck 61 | protocol: TCP 62 | readinessProbe: 63 | failureThreshold: 3 64 | httpGet: 65 | path: /healthz 66 | port: 6080 67 | scheme: HTTP 68 | initialDelaySeconds: 5 69 | periodSeconds: 5 70 | successThreshold: 1 71 | timeoutSeconds: 1 72 | resources: {} 73 | securityContext: 74 | allowPrivilegeEscalation: false 75 | capabilities: 76 | drop: 77 | - ALL 78 | readOnlyRootFilesystem: true 79 | terminationMessagePath: /dev/termination-log 80 | terminationMessagePolicy: File 81 | volumeMounts: 82 | - mountPath: /var/run/secrets/kubernetes.io/serviceaccount 83 | name: kube-api-access-9657m 84 | readOnly: true 85 | dnsPolicy: ClusterFirst 86 | enableServiceLinks: false 87 | nodeName: c3-medium-x86-03-meshery 88 | nodeSelector: 89 | kubernetes.io/os: linux 90 | preemptionPolicy: PreemptLowerPriority 91 | priority: 0 92 | restartPolicy: Always 93 | schedulerName: default-scheduler 94 | securityContext: 95 | runAsNonRoot: true 96 | seccompProfile: 97 | type: RuntimeDefault 98 | serviceAccount: cert-manager-webhook 99 | serviceAccountName: cert-manager-webhook 100 | terminationGracePeriodSeconds: 30 101 | tolerations: 102 | - effect: NoExecute 103 | key: node.kubernetes.io/not-ready 104 | operator: Exists 105 | tolerationSeconds: 300 106 | - effect: NoExecute 107 | key: node.kubernetes.io/unreachable 108 | operator: Exists 109 | tolerationSeconds: 300 110 | volumes: 111 | - name: kube-api-access-9657m 112 | projected: 113 | defaultMode: 420 114 | sources: 115 | - serviceAccountToken: 116 | expirationSeconds: 3607 117 | path: token 118 | - configMap: 119 | items: 120 | - key: ca.crt 121 | path: ca.crt 122 | name: kube-root-ca.crt 123 | - downwardAPI: 124 | items: 125 | - fieldRef: 126 | apiVersion: v1 127 | fieldPath: metadata.namespace 128 | path: namespace 129 | status: 130 | conditions: 131 | - lastProbeTime: null 132 | lastTransitionTime: "2024-04-01T04:29:34Z" 133 | status: "True" 134 | type: PodReadyToStartContainers 135 | - lastProbeTime: null 136 | lastTransitionTime: "2024-04-01T04:29:32Z" 137 | status: "True" 138 | type: Initialized 139 | - lastProbeTime: null 140 | lastTransitionTime: "2024-11-06T05:50:58Z" 141 | status: "True" 142 | type: Ready 143 | - lastProbeTime: null 144 | lastTransitionTime: "2024-11-06T05:50:58Z" 145 | status: "True" 146 | type: ContainersReady 147 | - lastProbeTime: null 148 | lastTransitionTime: "2024-04-01T04:29:32Z" 149 | status: "True" 150 | type: PodScheduled 151 | containerStatuses: 152 | - containerID: containerd://f60e25f3ca9dddc430ade42179a757868cf2264e52682a003fb2fd52df7b8efc 153 | image: quay.io/jetstack/cert-manager-webhook:v1.14.4 154 | imageID: quay.io/jetstack/cert-manager-webhook@sha256:11f7e7c462da3c0329e0a1e695a7bd37d6b3c28312d4edd4cc8d36f70ecbfa63 155 | lastState: {} 156 | name: cert-manager-webhook 157 | ready: true 158 | restartCount: 0 159 | started: true 160 | state: 161 | running: 162 | startedAt: "2024-04-01T04:29:33Z" 163 | hostIP: 139.178.83.85 164 | hostIPs: 165 | - ip: 139.178.83.85 166 | phase: Running 167 | podIP: 192.168.0.15 168 | podIPs: 169 | - ip: 192.168.0.15 170 | qosClass: BestEffort 171 | startTime: "2024-04-01T04:29:32Z" 172 | -------------------------------------------------------------------------------- /config.yaml: -------------------------------------------------------------------------------- 1 | have_fun: true 2 | code_review: 3 | disable: true 4 | comment_severity_threshold: MEDIUM 5 | max_review_comments: -1 6 | pull_request_opened: 7 | help: false 8 | summary: true 9 | code_review: true 10 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/meshery/meshsync 2 | 3 | replace vbom.ml/util => github.com/fvbommel/util v0.0.0-20180919145318-efcd4e0f9787 4 | 5 | go 1.23.4 6 | 7 | require ( 8 | github.com/buger/jsonparser v1.1.1 9 | github.com/google/uuid v1.6.0 10 | github.com/meshery/meshery-operator v0.8.7 11 | github.com/meshery/meshkit v0.8.32 12 | github.com/myntra/pipeline v0.0.0-20180618182531-2babf4864ce8 13 | github.com/sirupsen/logrus v1.9.3 14 | github.com/spf13/viper v1.19.0 15 | github.com/stretchr/testify v1.10.0 16 | golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 17 | golang.org/x/net v0.38.0 18 | gorm.io/gorm v1.25.12 19 | k8s.io/api v0.32.2 20 | k8s.io/apimachinery v0.32.2 21 | k8s.io/client-go v0.32.2 22 | k8s.io/kubectl v0.32.2 23 | sigs.k8s.io/yaml v1.4.0 24 | ) 25 | 26 | require ( 27 | cloud.google.com/go/auth v0.14.0 // indirect 28 | cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect 29 | cloud.google.com/go/compute/metadata v0.6.0 // indirect 30 | cuelang.org/go v0.11.2 // indirect 31 | dario.cat/mergo v1.0.1 // indirect 32 | github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect 33 | github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect 34 | github.com/BurntSushi/toml v1.4.0 // indirect 35 | github.com/MakeNowJust/heredoc v1.0.0 // indirect 36 | github.com/Masterminds/goutils v1.1.1 // indirect 37 | github.com/Masterminds/semver/v3 v3.3.1 // indirect 38 | github.com/Masterminds/sprig/v3 v3.3.0 // indirect 39 | github.com/Masterminds/squirrel v1.5.4 // indirect 40 | github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect 41 | github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect 42 | github.com/beorn7/perks v1.0.1 // indirect 43 | github.com/blang/semver/v4 v4.0.0 // indirect 44 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 45 | github.com/chai2010/gettext-go v1.0.3 // indirect 46 | github.com/cockroachdb/apd/v3 v3.2.1 // indirect 47 | github.com/containerd/containerd v1.7.25 // indirect 48 | github.com/containerd/errdefs v1.0.0 // indirect 49 | github.com/containerd/log v0.1.0 // indirect 50 | github.com/containerd/platforms v0.2.1 // indirect 51 | github.com/cyphar/filepath-securejoin v0.4.0 // indirect 52 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 53 | github.com/distribution/reference v0.6.0 // indirect 54 | github.com/docker/cli v27.5.1+incompatible // indirect 55 | github.com/docker/distribution v2.8.3+incompatible // indirect 56 | github.com/docker/docker v27.5.1+incompatible // indirect 57 | github.com/docker/docker-credential-helpers v0.8.2 // indirect 58 | github.com/docker/go-connections v0.5.0 // indirect 59 | github.com/docker/go-metrics v0.0.1 // indirect 60 | github.com/emicklei/go-restful/v3 v3.12.1 // indirect 61 | github.com/evanphx/json-patch v5.9.0+incompatible // indirect 62 | github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect 63 | github.com/fatih/color v1.18.0 // indirect 64 | github.com/felixge/httpsnoop v1.0.4 // indirect 65 | github.com/fsnotify/fsnotify v1.8.0 // indirect 66 | github.com/fxamacker/cbor/v2 v2.7.0 // indirect 67 | github.com/go-errors/errors v1.5.1 // indirect 68 | github.com/go-gorp/gorp/v3 v3.1.0 // indirect 69 | github.com/go-logr/logr v1.4.2 // indirect 70 | github.com/go-logr/stdr v1.2.2 // indirect 71 | github.com/go-openapi/jsonpointer v0.21.0 // indirect 72 | github.com/go-openapi/jsonreference v0.21.0 // indirect 73 | github.com/go-openapi/swag v0.23.0 // indirect 74 | github.com/gobwas/glob v0.2.3 // indirect 75 | github.com/gofrs/uuid v4.4.0+incompatible // indirect 76 | github.com/gogo/protobuf v1.3.2 // indirect 77 | github.com/golang/protobuf v1.5.4 // indirect 78 | github.com/google/btree v1.1.3 // indirect 79 | github.com/google/gnostic-models v0.6.9 // indirect 80 | github.com/google/go-cmp v0.6.0 // indirect 81 | github.com/google/gofuzz v1.2.0 // indirect 82 | github.com/google/s2a-go v0.1.9 // indirect 83 | github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect 84 | github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect 85 | github.com/googleapis/gax-go/v2 v2.14.1 // indirect 86 | github.com/gorilla/mux v1.8.1 // indirect 87 | github.com/gorilla/websocket v1.5.3 // indirect 88 | github.com/gosuri/uitable v0.0.4 // indirect 89 | github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect 90 | github.com/hashicorp/errwrap v1.1.0 // indirect 91 | github.com/hashicorp/go-multierror v1.1.1 // indirect 92 | github.com/hashicorp/hcl v1.0.0 // indirect 93 | github.com/huandu/xstrings v1.5.0 // indirect 94 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 95 | github.com/jackc/pgpassfile v1.0.0 // indirect 96 | github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect 97 | github.com/jackc/pgx/v5 v5.7.2 // indirect 98 | github.com/jackc/puddle/v2 v2.2.2 // indirect 99 | github.com/jinzhu/inflection v1.0.0 // indirect 100 | github.com/jinzhu/now v1.1.5 // indirect 101 | github.com/jmoiron/sqlx v1.4.0 // indirect 102 | github.com/josharian/intern v1.0.0 // indirect 103 | github.com/json-iterator/go v1.1.12 // indirect 104 | github.com/klauspost/compress v1.17.11 // indirect 105 | github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect 106 | github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect 107 | github.com/lib/pq v1.10.9 // indirect 108 | github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect 109 | github.com/magiconair/properties v1.8.9 // indirect 110 | github.com/mailru/easyjson v0.9.0 // indirect 111 | github.com/mattn/go-colorable v0.1.14 // indirect 112 | github.com/mattn/go-isatty v0.0.20 // indirect 113 | github.com/mattn/go-runewidth v0.0.16 // indirect 114 | github.com/mattn/go-sqlite3 v1.14.24 // indirect 115 | github.com/meshery/schemas v0.8.22 // indirect 116 | github.com/mitchellh/copystructure v1.2.0 // indirect 117 | github.com/mitchellh/go-wordwrap v1.0.1 // indirect 118 | github.com/mitchellh/mapstructure v1.5.0 // indirect 119 | github.com/mitchellh/reflectwalk v1.0.2 // indirect 120 | github.com/moby/locker v1.0.1 // indirect 121 | github.com/moby/spdystream v0.5.0 // indirect 122 | github.com/moby/term v0.5.2 // indirect 123 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 124 | github.com/modern-go/reflect2 v1.0.2 // indirect 125 | github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect 126 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 127 | github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect 128 | github.com/nats-io/nats.go v1.38.0 // indirect 129 | github.com/nats-io/nkeys v0.4.9 // indirect 130 | github.com/nats-io/nuid v1.0.1 // indirect 131 | github.com/oapi-codegen/runtime v1.1.1 // indirect 132 | github.com/opencontainers/go-digest v1.0.0 // indirect 133 | github.com/opencontainers/image-spec v1.1.0 // indirect 134 | github.com/pelletier/go-toml/v2 v2.2.3 // indirect 135 | github.com/peterbourgon/diskv v2.0.1+incompatible // indirect 136 | github.com/pkg/errors v0.9.1 // indirect 137 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 138 | github.com/prometheus/client_golang v1.20.5 // indirect 139 | github.com/prometheus/client_model v0.6.1 // indirect 140 | github.com/prometheus/common v0.62.0 // indirect 141 | github.com/prometheus/procfs v0.15.1 // indirect 142 | github.com/rivo/uniseg v0.4.7 // indirect 143 | github.com/rubenv/sql-migrate v1.7.1 // indirect 144 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 145 | github.com/sagikazarmark/locafero v0.7.0 // indirect 146 | github.com/sagikazarmark/slog-shim v0.1.0 // indirect 147 | github.com/shopspring/decimal v1.4.0 // indirect 148 | github.com/sourcegraph/conc v0.3.0 // indirect 149 | github.com/spf13/afero v1.12.0 // indirect 150 | github.com/spf13/cast v1.7.1 // indirect 151 | github.com/spf13/cobra v1.8.1 // indirect 152 | github.com/spf13/pflag v1.0.5 // indirect 153 | github.com/subosito/gotenv v1.6.0 // indirect 154 | github.com/x448/float16 v0.8.4 // indirect 155 | github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect 156 | github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect 157 | github.com/xeipuuv/gojsonschema v1.2.0 // indirect 158 | github.com/xlab/treeprint v1.2.0 // indirect 159 | go.opentelemetry.io/auto/sdk v1.1.0 // indirect 160 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect 161 | go.opentelemetry.io/otel v1.34.0 // indirect 162 | go.opentelemetry.io/otel/metric v1.34.0 // indirect 163 | go.opentelemetry.io/otel/trace v1.34.0 // indirect 164 | go.uber.org/multierr v1.11.0 // indirect 165 | golang.org/x/crypto v0.36.0 // indirect 166 | golang.org/x/mod v0.22.0 // indirect 167 | golang.org/x/oauth2 v0.25.0 // indirect 168 | golang.org/x/sync v0.12.0 // indirect 169 | golang.org/x/sys v0.31.0 // indirect 170 | golang.org/x/term v0.30.0 // indirect 171 | golang.org/x/text v0.23.0 // indirect 172 | golang.org/x/time v0.9.0 // indirect 173 | google.golang.org/api v0.218.0 // indirect 174 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47 // indirect 175 | google.golang.org/grpc v1.70.0 // indirect 176 | google.golang.org/protobuf v1.36.4 // indirect 177 | gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect 178 | gopkg.in/inf.v0 v0.9.1 // indirect 179 | gopkg.in/ini.v1 v1.67.0 // indirect 180 | gopkg.in/yaml.v3 v3.0.1 // indirect 181 | gorm.io/driver/postgres v1.5.11 // indirect 182 | gorm.io/driver/sqlite v1.5.7 // indirect 183 | helm.sh/helm/v3 v3.17.3 // indirect 184 | k8s.io/apiextensions-apiserver v0.32.2 // indirect 185 | k8s.io/apiserver v0.32.2 // indirect 186 | k8s.io/cli-runtime v0.32.2 // indirect 187 | k8s.io/component-base v0.32.2 // indirect 188 | k8s.io/klog/v2 v2.130.1 // indirect 189 | k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 // indirect 190 | k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect 191 | oras.land/oras-go v1.2.6 // indirect 192 | sigs.k8s.io/controller-runtime v0.20.1 // indirect 193 | sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect 194 | sigs.k8s.io/kustomize/api v0.19.0 // indirect 195 | sigs.k8s.io/kustomize/kyaml v0.19.0 // indirect 196 | sigs.k8s.io/structured-merge-diff/v4 v4.5.0 // indirect 197 | ) 198 | -------------------------------------------------------------------------------- /helpers/component_info.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "meshsync", 3 | "type": "controller", 4 | "next_error_code": 1015 5 | } -------------------------------------------------------------------------------- /install/Makefile.core.mk: -------------------------------------------------------------------------------- 1 | # Copyright Meshery Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | #----------------------------------------------------------------------------- 16 | # Global Variables 17 | #----------------------------------------------------------------------------- 18 | GIT_VERSION = $(shell git describe --tags `git rev-list --tags --max-count=1`) 19 | GIT_COMMITSHA = $(shell git rev-list -1 HEAD) 20 | GIT_STRIPPED_VERSION=$(shell git describe --tags `git rev-list --tags --max-count=1` | cut -c 2-) 21 | REMOTE_PROVIDER="Meshery" 22 | LOCAL_PROVIDER="None" 23 | GOVERSION = 1.19 24 | GOPATH = $(shell go env GOPATH) 25 | GOBIN = $(GOPATH)/bin 26 | 27 | SHELL :=/bin/bash -o pipefail 28 | 29 | #----------------------------------------------------------------------------- 30 | # Server 31 | #----------------------------------------------------------------------------- 32 | MESHERY_K8S_SKIP_COMP_GEN ?= TRUE 33 | APPLICATIONCONFIGPATH="./apps.json" 34 | 35 | #----------------------------------------------------------------------------- 36 | # Build 37 | #----------------------------------------------------------------------------- 38 | RELEASE_CHANNEL="edge" 39 | -------------------------------------------------------------------------------- /install/Makefile.show-help.mk: -------------------------------------------------------------------------------- 1 | .DEFAULT_GOAL := show-help 2 | # See for explanation. 3 | .PHONY: show-help 4 | show-help: 5 | @echo "$$(tput bold)Please specify a build target. The choices are:$$(tput sgr0)";echo;sed -ne"/^## /{h;s/.*//;:d" -e"H;n;s/^## //;td" -e"s/:.*//;G;s/\\n## /---/;s/\\n/ /g;p;}" ${MAKEFILE_LIST}|LC_ALL='C' sort -f|awk -F --- -v n=$$(tput cols) -v i=19 -v a="$$(tput setaf 6)" -v z="$$(tput sgr0)" '{printf"%s%*s%s ",a,-i,$$1,z;m=split($$2,w," ");l=n-i;for(j=1;j<=m;j++){l-=length(w[j])+1;if(l<= 0){l=n-i-length(w[j])-1;printf"\n%*s ",-i," ";}printf"%s ",w[j];}printf"\n";}'|more $(shell test $(shell uname) == Darwin && echo '-Xr') -------------------------------------------------------------------------------- /integration-tests/.gitignore: -------------------------------------------------------------------------------- 1 | *meshsync-output.txt -------------------------------------------------------------------------------- /integration-tests/default_cluster_integration_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | "syscall" 8 | "testing" 9 | "time" 10 | 11 | "github.com/meshery/meshkit/broker" 12 | "github.com/meshery/meshkit/broker/nats" 13 | ) 14 | 15 | var runIntegrationTest bool 16 | var meshsyncBinaryPath string 17 | var saveMeshsyncOutput bool // if true, saves outputof meshsync binary to file 18 | var testMeshsyncTopic = "meshery.meshsync.core" 19 | var testMeshsyncNatsURL = "localhost:4222" 20 | 21 | func init() { 22 | runIntegrationTest = os.Getenv("RUN_INTEGRATION_TESTS") == "true" 23 | meshsyncBinaryPath = os.Getenv("MESHSYNC_BINARY_PATH") 24 | saveMeshsyncOutput = os.Getenv("SAVE_MESHSYNC_OUTPUT") == "true" 25 | } 26 | 27 | /** 28 | * to run locally this test requires: 29 | * - docker 30 | * - kind 31 | * - kubectl 32 | * -- 33 | * use Makefile to run 34 | * -- 35 | * this test runs all test cases on the same k8s cluster, but with different input params for meshsync; 36 | * if you need a specific cluster setup you (probably) need to write a separate test, 37 | * or fit in the current cluster set up without failing existing tests; 38 | * -- 39 | * test flow of every test case is as follow: 40 | * - subscribe to nats (each test case has a separate queue group, so it receives every message); 41 | * - run meshsync binary; 42 | * - receive messages from nats and perform assertions; 43 | */ 44 | func TestWithNatsDefaultK8SClusterIntegration(t *testing.T) { 45 | if !runIntegrationTest { 46 | t.Skip("skipping integration test") 47 | } 48 | 49 | br, err := nats.New(nats.Options{ 50 | URLS: []string{testMeshsyncNatsURL}, 51 | ConnectionName: "meshsync", 52 | Username: "", 53 | Password: "", 54 | ReconnectWait: 2 * time.Second, 55 | MaxReconnect: 60, 56 | }) 57 | if err != nil { 58 | t.Fatal("error connecting to nats", err) 59 | } 60 | 61 | for i, tc := range defaultClusterTestCasesData { 62 | t.Run( 63 | tc.name, 64 | runWithNatsDefaultK8SClusterTestCase( 65 | br, 66 | i, 67 | tc, 68 | ), 69 | ) 70 | } 71 | } 72 | 73 | // need this as separate function to bring down cyclomatic complexity 74 | // this one itself is to complicated :) 75 | // 76 | // TODO fix cyclop error 77 | // integration-tests/default_cluster_integration_test.go:74:1: calculated cyclomatic complexity for function runWithNatsDefaultK8SClusterTestCase is 11, max is 10 (cyclop) 78 | // 79 | //nolint:cyclop 80 | func runWithNatsDefaultK8SClusterTestCase( 81 | br broker.Handler, 82 | tcIndex int, 83 | tc defaultClusterTestCaseStruct, 84 | ) func(t *testing.T) { 85 | return func(t *testing.T) { 86 | for _, cleanupHook := range tc.cleanupHooks { 87 | defer cleanupHook() 88 | } 89 | 90 | for _, setupHook := range tc.setupHooks { 91 | setupHook() 92 | } 93 | 94 | out := make(chan *broker.Message) 95 | // Step 1: subscribe to the queue 96 | if err := br.SubscribeWithChannel( 97 | testMeshsyncTopic, 98 | fmt.Sprintf("default-cluster-queue-group-%d", tcIndex), 99 | out, 100 | ); err != nil { 101 | t.Fatalf("error subscribing to topic: %v", err) 102 | } 103 | 104 | // Step 2: process messages 105 | resultData := make(map[string]any, 1) 106 | if tc.natsMessageHandler != nil { 107 | go tc.natsMessageHandler(t, out, resultData) 108 | } 109 | 110 | os.Setenv("BROKER_URL", testMeshsyncNatsURL) 111 | 112 | // Step 3: run the meshsync command 113 | cmd, deferFunc := prepareMeshsyncCMD(t, tcIndex, tc) 114 | defer deferFunc() 115 | 116 | if err := cmd.Start(); err != nil { 117 | t.Fatalf("error starting binary: %v", err) 118 | } 119 | errCh := make(chan error) 120 | go func(cmd0 *exec.Cmd, errCh0 chan<- error) { 121 | errCh0 <- cmd0.Wait() 122 | }(cmd, errCh) 123 | 124 | // intentionally big timeout to wait till the cmd execution ended 125 | timeout := time.Duration(time.Hour * 24) 126 | if tc.waitMeshsyncTimeout > 0 { 127 | timeout = tc.waitMeshsyncTimeout 128 | } 129 | 130 | select { 131 | case err := <-errCh: 132 | if err != nil { 133 | t.Fatalf("error running binary: %v", err) 134 | } 135 | case <-time.After(timeout): 136 | if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { 137 | t.Fatalf("error terminating meshsync command: %v", err) 138 | } 139 | t.Logf("processing after timeout %d", timeout) 140 | } 141 | 142 | // Step 4: do final assertion, if any 143 | tc.finalHandler(t, resultData) 144 | 145 | t.Logf("done %s", tc.name) 146 | } 147 | } 148 | 149 | // introduced this function to decrease cyclomatic complexity 150 | func prepareMeshsyncCMD( 151 | t *testing.T, 152 | tcIndex int, 153 | tc defaultClusterTestCaseStruct, 154 | ) (*exec.Cmd, func()) { 155 | cmd := exec.Command(meshsyncBinaryPath, tc.meshsyncCMDArgs...) 156 | deferFunc := func() {} 157 | // there is quite rich output from meshsync 158 | // save to file instead of stdout 159 | if saveMeshsyncOutput { 160 | meshsyncOutputFileName := fmt.Sprintf("default-cluster-test-case-%02d.meshsync-output.txt", tcIndex) 161 | meshsyncOutputFile, err := os.Create(meshsyncOutputFileName) 162 | if err != nil { 163 | t.Logf("Could not create meshsync output file %s", meshsyncOutputFileName) 164 | // if not possible to create output file, print to the stdout 165 | cmd.Stdout = os.Stdout 166 | } 167 | deferFunc = func() { 168 | meshsyncOutputFile.Close() 169 | } 170 | cmd.Stdout = meshsyncOutputFile 171 | } else { 172 | cmd.Stdout = nil 173 | } 174 | cmd.Stderr = os.Stderr 175 | cmd.Stdin = os.Stdin 176 | 177 | return cmd, deferFunc 178 | } 179 | -------------------------------------------------------------------------------- /integration-tests/default_cluster_test_cases_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | "testing" 8 | "time" 9 | 10 | "github.com/meshery/meshkit/broker" 11 | "github.com/stretchr/testify/assert" 12 | "sigs.k8s.io/yaml" 13 | ) 14 | 15 | type defaultClusterTestCaseStruct struct { 16 | setupHooks []func() 17 | cleanupHooks []func() 18 | name string 19 | meshsyncCMDArgs []string // args to pass to meshsync binary 20 | waitMeshsyncTimeout time.Duration // if <= 0: waits till meshsync ends execution, otherwise moves further after specified duration 21 | // the reason for resultData map is that natsMessageHandler is processing chan indefinitely 22 | // and there is no graceful exit from function; 23 | natsMessageHandler func( 24 | t *testing.T, 25 | out chan *broker.Message, 26 | resultData map[string]any, 27 | ) 28 | finalHandler func(t *testing.T, resultData map[string]any) 29 | } 30 | 31 | var defaultClusterTestCasesData []defaultClusterTestCaseStruct = []defaultClusterTestCaseStruct{ 32 | { 33 | name: "number of messages received from nats is greater than zero", 34 | meshsyncCMDArgs: []string{"--stopAfterSeconds", "8"}, 35 | natsMessageHandler: func( 36 | t *testing.T, 37 | out chan *broker.Message, 38 | resultData map[string]any, 39 | ) { 40 | count := 0 41 | resultData["count"] = count 42 | go func() { 43 | for range out { 44 | count++ 45 | resultData["count"] = count 46 | } 47 | }() 48 | }, 49 | finalHandler: func(t *testing.T, resultData map[string]any) { 50 | count, ok := resultData["count"].(int) 51 | assert.True(t, ok, "must get count from result map") 52 | if ok { 53 | t.Logf("received %d messages from broker", count) 54 | assert.True(t, count > 0, "must receive messages from queue") 55 | } 56 | 57 | }, 58 | }, 59 | { 60 | name: "receive from nats only specified resources", 61 | meshsyncCMDArgs: []string{ 62 | "--stopAfterSeconds", 63 | "8", 64 | "--outputResources", 65 | "pod,replicaset", 66 | }, 67 | natsMessageHandler: func( 68 | t *testing.T, 69 | out chan *broker.Message, 70 | resultData map[string]any, 71 | ) { 72 | resourcesCount := make(map[string]int) 73 | resultData["count"] = resourcesCount 74 | errCh := make(chan error) 75 | go func(errCh0 chan<- error) { 76 | for message := range out { 77 | k8sResource, err := unmarshalObject(message.Object) 78 | if err != nil { 79 | errCh0 <- fmt.Errorf( 80 | "error convering message.Object to model.KubernetesResource for %T", 81 | message.Object, 82 | ) 83 | return 84 | } 85 | resourcesCount[strings.ToLower(k8sResource.Kind)]++ 86 | resultData["count"] = resourcesCount 87 | } 88 | }(errCh) 89 | 90 | err := <-errCh 91 | if err != nil { 92 | t.Fatal(err) 93 | } 94 | }, 95 | finalHandler: func(t *testing.T, resultData map[string]any) { 96 | count, ok := resultData["count"].(map[string]int) 97 | assert.True(t, ok, "must get count from result map") 98 | if ok { 99 | allowedKeys := map[string]bool{"pod": true, "replicaset": true} 100 | otherKeys := make([]string, 0) 101 | for k, v := range count { 102 | t.Logf("received %d messages from Kind %s", v, k) 103 | if !allowedKeys[k] { 104 | otherKeys = append( 105 | otherKeys, 106 | fmt.Sprintf("[%s = %v]", k, v), 107 | ) 108 | } 109 | } 110 | assert.True(t, count["pod"] > 0, "must receive kind pod messages from queue") 111 | assert.True(t, count["replicaset"] > 0, "must receive kind replicaset messages from queue") 112 | if len(otherKeys) > 0 { 113 | t.Fatalf("received not allowed kind keys %s", strings.Join(otherKeys, ",")) 114 | } 115 | } 116 | 117 | }, 118 | }, 119 | { 120 | name: "receive from nats only resources from specified namespace", 121 | meshsyncCMDArgs: []string{ 122 | "--stopAfterSeconds", 123 | "8", 124 | "--outputNamespace", 125 | "agile-otter", 126 | }, 127 | natsMessageHandler: func( 128 | t *testing.T, 129 | out chan *broker.Message, 130 | resultData map[string]any, 131 | ) { 132 | resourcesPerNamespaceCount := make(map[string]int) 133 | resultData["count"] = resourcesPerNamespaceCount 134 | errCh := make(chan error) 135 | go func(errCh0 chan<- error) { 136 | for message := range out { 137 | k8sResource, err := unmarshalObject(message.Object) 138 | if err != nil { 139 | errCh0 <- fmt.Errorf( 140 | "error convering message.Object to model.KubernetesResource for %T", 141 | message.Object, 142 | ) 143 | return 144 | } 145 | resourcesPerNamespaceCount[strings.ToLower(k8sResource.KubernetesResourceMeta.Namespace)]++ 146 | resultData["count"] = resourcesPerNamespaceCount 147 | } 148 | }(errCh) 149 | 150 | err := <-errCh 151 | if err != nil { 152 | t.Fatal(err) 153 | } 154 | }, 155 | finalHandler: func(t *testing.T, resultData map[string]any) { 156 | count, ok := resultData["count"].(map[string]int) 157 | assert.True(t, ok, "must get count from result map") 158 | if ok { 159 | allowedKeys := map[string]bool{"agile-otter": true} 160 | otherKeys := make([]string, 0) 161 | for k, v := range count { 162 | t.Logf("received %d messages for namespace %s", v, k) 163 | if !allowedKeys[k] { 164 | otherKeys = append( 165 | otherKeys, 166 | fmt.Sprintf("[%s = %v]", k, v), 167 | ) 168 | } 169 | } 170 | assert.True(t, count["agile-otter"] > 0, "must receive messages from resources in agile-otter namespace") 171 | 172 | if len(otherKeys) > 0 { 173 | t.Fatalf("received not allowed namespace keys %s", strings.Join(otherKeys, ",")) 174 | } 175 | } 176 | 177 | }, 178 | }, 179 | { 180 | name: "output mode file: must not receive message from queue", 181 | meshsyncCMDArgs: []string{ 182 | "--stopAfterSeconds", "8", 183 | "--output", "file", 184 | "--outputFile", "meshery-cluster-snapshot-integration-test-03.yaml", 185 | }, 186 | setupHooks: []func(){ 187 | func() { 188 | // put this in setupHooks and not in cleanupHooks 189 | // as it is convinient to have files stay after test run 190 | // but need to clear them before the test run 191 | os.RemoveAll("meshery-cluster-snapshot-integration-test-03-extended.yaml") 192 | os.RemoveAll("meshery-cluster-snapshot-integration-test-03.yaml") 193 | }, 194 | }, 195 | natsMessageHandler: func( 196 | t *testing.T, 197 | out chan *broker.Message, 198 | resultData map[string]any, 199 | ) { 200 | count := 0 201 | resultData["count"] = count 202 | go func() { 203 | for range out { 204 | count++ 205 | resultData["count"] = count 206 | } 207 | }() 208 | }, 209 | finalHandler: func(t *testing.T, resultData map[string]any) { 210 | count, ok := resultData["count"].(int) 211 | assert.True(t, ok, "must get count from result map") 212 | if ok { 213 | t.Logf("received %d messages from broker", count) 214 | assert.Equal(t, 0, count, "must not receive messages from queue") 215 | } 216 | 217 | }, 218 | }, 219 | { 220 | // TODO check that yaml must be a valid k8s manifest 221 | // f.e. with kubectl apply --dry-run=client 222 | // now the yaml has an issues, when dry-run receive an error: 223 | // unable to decode "integration-tests/meshery-cluster-snapshot-integration-test-04.yaml": json: cannot unmarshal array into Go struct field ObjectMeta.metadata.labels of type map[string]string 224 | name: "output mode file: must have yaml with kind pod and kind deployment only and from dedicated namespace", 225 | meshsyncCMDArgs: []string{ 226 | "--stopAfterSeconds", "8", 227 | "--output", "file", 228 | "--outputResources", "pod,deployment", 229 | "--outputNamespace", "agile-otter", 230 | "--outputFile", "meshery-cluster-snapshot-integration-test-04.yaml", 231 | }, 232 | setupHooks: []func(){ 233 | func() { 234 | // put this in setupHooks and not in cleanupHooks 235 | // as it is convinient to have files stay after test run 236 | // but need to clear them before the test run 237 | os.RemoveAll("meshery-cluster-snapshot-integration-test-04-extended.yaml") 238 | os.RemoveAll("meshery-cluster-snapshot-integration-test-04.yaml") 239 | }, 240 | }, 241 | finalHandler: func(t *testing.T, resultData map[string]any) { 242 | expectedKinds := []string{"pod", "deployment"} 243 | expectedNamespace := "agile-otter" 244 | 245 | type K8sMetadata struct { 246 | Kind string `yaml:"kind"` 247 | Metadata struct { 248 | Namespace string `yaml:"namespace"` 249 | } `yaml:"metadata"` 250 | } 251 | 252 | // Load file content 253 | data, err := os.ReadFile("meshery-cluster-snapshot-integration-test-04.yaml") 254 | if err != nil { 255 | panic(err) 256 | } 257 | 258 | // Split on '---' to handle multiple resources 259 | docs := strings.Split(string(data), "---") 260 | var objects []K8sMetadata 261 | 262 | for _, doc := range docs { 263 | doc = strings.TrimSpace(doc) 264 | if doc == "" { 265 | continue 266 | } 267 | 268 | var obj K8sMetadata 269 | if err := yaml.Unmarshal([]byte(doc), &obj); err != nil { 270 | fmt.Printf("Error parsing document: %v\n", err) 271 | continue 272 | } 273 | objects = append(objects, obj) 274 | 275 | kindCount := make(map[string]int) 276 | namespaceCount := make(map[string]int) 277 | for _, obj := range objects { 278 | kindCount[strings.ToLower(obj.Kind)]++ 279 | namespaceCount[obj.Metadata.Namespace]++ 280 | } 281 | 282 | for kind, count := range kindCount { 283 | t.Logf("read %d objects of Kind %s", count, kind) 284 | assert.Contains(t, expectedKinds, kind, "kind must be one from the expected list") 285 | } 286 | 287 | for namespace, count := range namespaceCount { 288 | t.Logf("read %d objects with namespace %s", count, namespace) 289 | assert.Equal(t, expectedNamespace, namespace, "namespace must match expected namespace") 290 | } 291 | } 292 | }, 293 | }, 294 | { 295 | name: "must not fail with a --broker-url param", 296 | meshsyncCMDArgs: []string{ 297 | "--broker-url", "10.96.235.19:4222", 298 | "--stopAfterSeconds", "8", 299 | }, 300 | natsMessageHandler: func( 301 | t *testing.T, 302 | out chan *broker.Message, 303 | resultData map[string]any, 304 | ) { 305 | count := 0 306 | resultData["count"] = count 307 | go func() { 308 | for range out { 309 | count++ 310 | resultData["count"] = count 311 | } 312 | }() 313 | }, 314 | finalHandler: func(t *testing.T, resultData map[string]any) { 315 | count, ok := resultData["count"].(int) 316 | assert.True(t, ok, "must get count from result map") 317 | if ok { 318 | t.Logf("received %d messages from broker", count) 319 | assert.True(t, count > 0, "must receive messages from queue") 320 | } 321 | 322 | }, 323 | }, 324 | } 325 | -------------------------------------------------------------------------------- /integration-tests/infrastructure/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | nats: 5 | image: nats:2.11-alpine3.21 6 | container_name: nats 7 | ports: 8 | - "4222:4222" # client connections 9 | - "8222:8222" # HTTP monitoring (optional) 10 | healthcheck: 11 | test: ["CMD", "curl", "-f", "http://localhost:8222/varz"] 12 | interval: 5s 13 | timeout: 3s 14 | retries: 5 15 | -------------------------------------------------------------------------------- /integration-tests/infrastructure/meshsync.yaml: -------------------------------------------------------------------------------- 1 | # Source: meshery-operator/charts/meshery-meshsync/templates/meshery_v1alpha1_broker_cr.tpl 2 | apiVersion: meshery.io/v1alpha1 3 | kind: MeshSync 4 | metadata: 5 | name: meshery-meshsync 6 | namespace: meshery 7 | spec: 8 | size: 1 9 | broker: 10 | native: 11 | name: meshery-broker 12 | namespace: meshery 13 | watch-list: 14 | data: 15 | whitelist: "[{\"Resource\":\"grafanas.v1beta1.grafana.integreatly.org\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"prometheuses.v1.monitoring.coreos.com\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"namespaces.v1.\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"configmaps.v1.\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"nodes.v1.\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"secrets.v1.\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"persistentvolumes.v1.\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"persistentvolumeclaims.v1.\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"replicasets.v1.apps\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"pods.v1.\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"services.v1.\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"deployments.v1.apps\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"statefulsets.v1.apps\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"daemonsets.v1.apps\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"ingresses.v1.networking.k8s.io\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"endpoints.v1.\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"endpointslices.v1.discovery.k8s.io\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"cronjobs.v1.batch\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"replicationcontrollers.v1.\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"storageclasses.v1.storage.k8s.io\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"clusterroles.v1.rbac.authorization.k8s.io\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"volumeattachments.v1.storage.k8s.io\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]},{\"Resource\":\"apiservices.v1.apiregistration.k8s.io\",\"Events\":[\"ADDED\",\"MODIFIED\",\"DELETED\"]}]" 16 | -------------------------------------------------------------------------------- /integration-tests/infrastructure/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | CLUSTER_NAME="meshsync-integration-test-cluster" 7 | CUSTOM_NAMESPACE="agile-otter" 8 | 9 | 10 | check_dependencies() { 11 | # Check for docker 12 | if ! command -v docker &> /dev/null; then 13 | echo "❌ docker is not installed. Please install docker first." 14 | exit 1 15 | fi 16 | echo "✅ docker is installed;" 17 | 18 | # Check for kind 19 | if ! command -v kind &> /dev/null; then 20 | echo "❌ kind is not installed. Please install KinD first." 21 | exit 1 22 | fi 23 | echo "✅ kind is installed;" 24 | 25 | # Check for kubectl 26 | if ! command -v kubectl &> /dev/null; then 27 | echo "❌ kubectl is not installed. Please install kubectl first." 28 | exit 1 29 | fi 30 | echo "✅ kubectl is installed;" 31 | } 32 | 33 | setup() { 34 | check_dependencies 35 | echo "🔧 Setting up..." 36 | 37 | echo "Running docker compose..." 38 | docker compose -f $SCRIPT_DIR/docker-compose.yaml up -d || exit 1 39 | 40 | echo "Creating KinD cluster..." 41 | kind create cluster --name $CLUSTER_NAME 42 | 43 | echo "Creating meshery namespace..." 44 | kubectl create namespace meshery 45 | 46 | echo "Applying meshery resources..." 47 | kubectl apply -f https://raw.githubusercontent.com/meshery/meshery/refs/heads/master/install/kubernetes/helm/meshery-operator/crds/crds.yaml 48 | kubectl --namespace meshery apply -f $SCRIPT_DIR/meshsync.yaml 49 | 50 | echo "Creating $CUSTOM_NAMESPACE namespace..." 51 | kubectl create namespace $CUSTOM_NAMESPACE 52 | 53 | echo "Applying $CUSTOM_NAMESPACE resources..." 54 | kubectl --namespace $CUSTOM_NAMESPACE apply -f $SCRIPT_DIR/test-deployment.yaml 55 | 56 | echo "Outputing cluster resources..." 57 | kubectl --namespace default get deployment 58 | kubectl --namespace default get rs 59 | kubectl --namespace default get po 60 | kubectl --namespace default get service 61 | kubectl --namespace default get configmap 62 | kubectl --namespace $CUSTOM_NAMESPACE get deployment 63 | kubectl --namespace $CUSTOM_NAMESPACE get rs 64 | kubectl --namespace $CUSTOM_NAMESPACE get po 65 | kubectl --namespace $CUSTOM_NAMESPACE get service 66 | kubectl --namespace $CUSTOM_NAMESPACE get configmap 67 | } 68 | 69 | cleanup() { 70 | echo "🧹 Cleaning up..." 71 | 72 | echo "Stopping docker compose..." 73 | docker compose -f $SCRIPT_DIR/docker-compose.yaml down 74 | 75 | echo "Deleting KinD cluster..." 76 | kind delete cluster --name $CLUSTER_NAME 77 | } 78 | 79 | print_help() { 80 | echo "Usage: $0 {check_dependencies|setup|cleanup|help}" 81 | } 82 | 83 | # Main dispatcher 84 | case "$1" in 85 | check_dependencies) 86 | check_dependencies 87 | ;; 88 | setup) 89 | setup 90 | ;; 91 | cleanup) 92 | cleanup 93 | ;; 94 | help) 95 | print_help 96 | ;; 97 | *) 98 | echo "❌ Unknown command: $1" 99 | print_help 100 | exit 1 101 | ;; 102 | esac -------------------------------------------------------------------------------- /integration-tests/infrastructure/test-deployment.yaml: -------------------------------------------------------------------------------- 1 | # based on the https://k8s.io/examples/application/deployment.yaml 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: nginx-deployment 6 | spec: 7 | selector: 8 | matchLabels: 9 | app: nginx 10 | replicas: 3 # tells deployment to run 3 pods matching the template 11 | template: 12 | metadata: 13 | labels: 14 | app: nginx 15 | spec: 16 | containers: 17 | - name: nginx 18 | image: nginx:1.14.2 19 | ports: 20 | - containerPort: 80 21 | -------------------------------------------------------------------------------- /integration-tests/unmarshal_object_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "github.com/meshery/meshkit/encoding" 5 | "github.com/meshery/meshkit/utils" 6 | "github.com/meshery/meshsync/pkg/model" 7 | ) 8 | 9 | func unmarshalObject(object interface{}) (model.KubernetesResource, error) { 10 | objectJSON, _ := utils.Marshal(object) 11 | obj := model.KubernetesResource{} 12 | err := encoding.Unmarshal([]byte(objectJSON), &obj) 13 | return obj, err 14 | } 15 | -------------------------------------------------------------------------------- /internal/channels/broker.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | import "github.com/meshery/meshkit/broker" 4 | 5 | var ( 6 | BrokerPublish = "broker-publish" 7 | BrokerSubscribe = "broker-subscribe" 8 | ) 9 | 10 | type BrokerPublishPayload struct { 11 | Subject string 12 | Data *broker.Message 13 | } 14 | 15 | func NewBrokerSubscribeChannel() BrokerSubscribeChannel { 16 | return make(chan *broker.Message) 17 | } 18 | 19 | type BrokerSubscribeChannel chan *broker.Message 20 | 21 | func (ch BrokerSubscribeChannel) Stop() { 22 | <-ch 23 | } 24 | 25 | func NewBrokerPublishChannel() BrokerPublishChannel { 26 | return make(chan *BrokerPublishPayload) 27 | } 28 | 29 | type BrokerPublishChannel chan *BrokerPublishPayload 30 | 31 | func (ch BrokerPublishChannel) Stop() { 32 | <-ch 33 | } 34 | -------------------------------------------------------------------------------- /internal/channels/channel.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | type GenericChannel interface { 4 | Stop() 5 | } 6 | 7 | func NewChannelPool() map[string]GenericChannel { 8 | return map[string]GenericChannel{ 9 | Stop: NewStopChannel(), 10 | OS: NewOSChannel(), 11 | ReSync: NewReSyncChannel(), 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /internal/channels/generic.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | const ( 4 | Struct = "struct" 5 | ) 6 | 7 | func NewStructChannel() StructChannel { 8 | return make(chan struct{}) 9 | } 10 | 11 | type StructChannel chan struct{} 12 | 13 | func (ch StructChannel) Stop() { 14 | <-ch 15 | } 16 | -------------------------------------------------------------------------------- /internal/channels/system.go: -------------------------------------------------------------------------------- 1 | package channels 2 | 3 | import "os" 4 | 5 | var ( 6 | OS = "os" 7 | Stop = "stop" 8 | ReSync = "resync" 9 | ) 10 | 11 | func NewStopChannel() StopChannel { 12 | return make(chan struct{}) 13 | } 14 | 15 | type StopChannel chan struct{} 16 | 17 | func (ch StopChannel) Stop() { 18 | <-ch 19 | } 20 | 21 | func NewOSChannel() OSChannel { 22 | return make(chan os.Signal, 1) 23 | } 24 | 25 | type OSChannel chan os.Signal 26 | 27 | func (ch OSChannel) Stop() { 28 | <-ch 29 | } 30 | 31 | func NewReSyncChannel() ReSyncChannel { 32 | return make(chan struct{}) 33 | } 34 | 35 | type ReSyncChannel chan struct{} 36 | 37 | func (ch ReSyncChannel) Stop() { 38 | <-ch 39 | } 40 | 41 | func (ch ReSyncChannel) ReSyncInformer() { 42 | ch <- struct{}{} 43 | } 44 | -------------------------------------------------------------------------------- /internal/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/meshery/meshkit/config" 5 | configprovider "github.com/meshery/meshkit/config/provider" 6 | "github.com/meshery/meshkit/utils" 7 | ) 8 | 9 | // New creates a new config instance 10 | func New(provider string) (config.Handler, error) { 11 | var ( 12 | handler config.Handler 13 | err error 14 | ) 15 | opts := configprovider.Options{ 16 | // TODO do we need this always or only when running from binary? 17 | FilePath: utils.GetHome() + "/.meshery", 18 | FileType: "yaml", 19 | FileName: "meshsync_config", 20 | } 21 | 22 | // this is required, because if folder opts.Filepath does not exist 23 | // meshsync run ends up with error, f.e. 24 | // Error while initializing MeshSync configuration. .Config File \"meshsync_config\" Not Found in \"[/home/runner/.meshery] 25 | // if folder exists, there is no error 26 | if err := utils.CreateDirectory(opts.FilePath); err != nil { 27 | return nil, ErrInitConfig(err) 28 | } 29 | 30 | // Config provider 31 | switch provider { 32 | case configprovider.ViperKey: 33 | handler, err = configprovider.NewViper(opts) 34 | if err != nil { 35 | return nil, ErrInitConfig(err) 36 | } 37 | case configprovider.InMemKey: 38 | handler, err = configprovider.NewInMem(opts) 39 | if err != nil { 40 | return nil, ErrInitConfig(err) 41 | } 42 | } 43 | 44 | return handler, nil 45 | } 46 | -------------------------------------------------------------------------------- /internal/config/config_local.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // how it is defined in 4 | // https://github.com/meshery/meshery/blob/master/install/kubernetes/helm/meshery-operator/charts/meshery-meshsync/values.yaml#L8 5 | var LocalMeshsyncConfig map[string]string = map[string]string{ 6 | "whitelist": `[{"Resource":"grafanas.v1beta1.grafana.integreatly.org","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"prometheuses.v1.monitoring.coreos.com","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"namespaces.v1.","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"configmaps.v1.","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"nodes.v1.","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"secrets.v1.","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"persistentvolumes.v1.","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"persistentvolumeclaims.v1.","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"replicasets.v1.apps","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"pods.v1.","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"services.v1.","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"deployments.v1.apps","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"statefulsets.v1.apps","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"daemonsets.v1.apps","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"ingresses.v1.networking.k8s.io","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"endpoints.v1.","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"endpointslices.v1.discovery.k8s.io","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"cronjobs.v1.batch","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"replicationcontrollers.v1.","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"storageclasses.v1.storage.k8s.io","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"clusterroles.v1.rbac.authorization.k8s.io","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"volumeattachments.v1.storage.k8s.io","Events":["ADDED","MODIFIED","DELETED"]},{"Resource":"apiservices.v1.apiregistration.k8s.io","Events":["ADDED","MODIFIED","DELETED"]}]`, 7 | } 8 | -------------------------------------------------------------------------------- /internal/config/crd_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | 8 | "github.com/meshery/meshery-operator/pkg/client" 9 | "github.com/meshery/meshkit/utils" 10 | "golang.org/x/exp/slices" 11 | corev1 "k8s.io/api/core/v1" 12 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 13 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 14 | "k8s.io/apimachinery/pkg/runtime/schema" 15 | "k8s.io/apimachinery/pkg/types" 16 | "k8s.io/client-go/dynamic" 17 | "k8s.io/client-go/rest" 18 | ) 19 | 20 | var ( 21 | namespace = "meshery" // Namespace for the Custom Resource 22 | crName = "meshery-meshsync" // Name of the custom resource 23 | version = "v1alpha1" // Version of the Custom Resource 24 | group = "meshery.io" //Group for the Custom Resource 25 | resource = "meshsyncs" //Name of the Resource 26 | ) 27 | 28 | func GetMeshsyncCRDConfigs(dyClient dynamic.Interface) (*MeshsyncConfig, error) { 29 | // make a call to get the custom resource 30 | crd, err := GetMeshsyncCRD(dyClient) 31 | 32 | if err != nil { 33 | return nil, ErrInitConfig(err) 34 | } 35 | 36 | if crd == nil { 37 | return nil, ErrInitConfig(errors.New("Custom Resource is nil")) 38 | } 39 | 40 | spec := crd.Object["spec"] 41 | specMap, ok := spec.(map[string]interface{}) 42 | if !ok { 43 | return nil, ErrInitConfig(errors.New("Unable to convert spec to map")) 44 | } 45 | configObj := specMap["watch-list"] 46 | if configObj == nil { 47 | return nil, ErrInitConfig(errors.New("Custom Resource does not have Meshsync Configs")) 48 | } 49 | configStr, err := utils.Marshal(configObj) 50 | if err != nil { 51 | return nil, ErrInitConfig(err) 52 | } 53 | 54 | configMap := corev1.ConfigMap{} 55 | err = utils.Unmarshal(string(configStr), &configMap) 56 | 57 | if err != nil { 58 | return nil, ErrInitConfig(err) 59 | } 60 | 61 | // populate the required configs 62 | meshsyncConfig, err := PopulateConfigs(configMap) 63 | 64 | if err != nil { 65 | return nil, ErrInitConfig(err) 66 | } 67 | return meshsyncConfig, nil 68 | } 69 | 70 | func GetMeshsyncCRD(dyClient dynamic.Interface) (*unstructured.Unstructured, error) { 71 | // initialize the group version resource to access the custom resource 72 | gvr := schema.GroupVersionResource{Version: version, Group: group, Resource: resource} 73 | return dyClient.Resource(gvr).Namespace(namespace).Get(context.TODO(), crName, metav1.GetOptions{}) 74 | } 75 | 76 | func GetMeshsyncCRDConfigsLocal() (*MeshsyncConfig, error) { 77 | // populate the required configs 78 | meshsyncConfig, err := PopulateConfigsFromMap(LocalMeshsyncConfig) 79 | 80 | if err != nil { 81 | // // this hides actual error message 82 | // return nil, ErrInitConfig(err) 83 | return nil, err 84 | } 85 | return meshsyncConfig, nil 86 | } 87 | 88 | // PopulateConfigs compares the default configs and the whitelist and blacklist 89 | func PopulateConfigs(configMap corev1.ConfigMap) (*MeshsyncConfig, error) { 90 | return PopulateConfigsFromMap(configMap.Data) 91 | } 92 | 93 | func PopulateConfigsFromMap(data map[string]string) (*MeshsyncConfig, error) { 94 | meshsyncConfig := &MeshsyncConfig{} 95 | 96 | if _, ok := data["blacklist"]; ok { 97 | if len(data["blacklist"]) > 0 { 98 | err := utils.Unmarshal(data["blacklist"], &meshsyncConfig.BlackList) 99 | if err != nil { 100 | return nil, ErrInitConfig(err) 101 | } 102 | } 103 | } 104 | 105 | if _, ok := data["whitelist"]; ok { 106 | if len(data["whitelist"]) > 0 { 107 | err := utils.Unmarshal(data["whitelist"], &meshsyncConfig.WhiteList) 108 | if err != nil { 109 | return nil, ErrInitConfig(err) 110 | } 111 | } 112 | } 113 | 114 | // ensure that atleast one of whitelist or blacklist has been supplied 115 | if len(meshsyncConfig.BlackList) == 0 && len(meshsyncConfig.WhiteList) == 0 { 116 | return nil, ErrInitConfig(errors.New("Both whitelisted and blacklisted resources missing")) 117 | } 118 | 119 | // ensure that only one of whitelist or blacklist has been supplied 120 | if len(meshsyncConfig.BlackList) != 0 && len(meshsyncConfig.WhiteList) != 0 { 121 | return nil, ErrInitConfig(errors.New("Both whitelisted and blacklisted resources not currently supported")) 122 | } 123 | 124 | // Handle global resources 125 | globalPipelines := make(PipelineConfigs, 0) 126 | localPipelines := make(PipelineConfigs, 0) 127 | 128 | if len(meshsyncConfig.WhiteList) != 0 { 129 | for _, v := range Pipelines[GlobalResourceKey] { 130 | if idx := slices.IndexFunc(meshsyncConfig.WhiteList, func(c ResourceConfig) bool { return c.Resource == v.Name }); idx != -1 { 131 | config := meshsyncConfig.WhiteList[idx] 132 | v.Events = config.Events 133 | globalPipelines = append(globalPipelines, v) 134 | } 135 | } 136 | if len(globalPipelines) > 0 { 137 | meshsyncConfig.Pipelines = map[string]PipelineConfigs{} 138 | meshsyncConfig.Pipelines[GlobalResourceKey] = globalPipelines 139 | } 140 | 141 | // Handle local resources 142 | for _, v := range Pipelines[LocalResourceKey] { 143 | if idx := slices.IndexFunc(meshsyncConfig.WhiteList, func(c ResourceConfig) bool { return c.Resource == v.Name }); idx != -1 { 144 | config := meshsyncConfig.WhiteList[idx] 145 | v.Events = config.Events 146 | localPipelines = append(localPipelines, v) 147 | } 148 | } 149 | 150 | if len(localPipelines) > 0 { 151 | if meshsyncConfig.Pipelines == nil { 152 | meshsyncConfig.Pipelines = make(map[string]PipelineConfigs) 153 | } 154 | meshsyncConfig.Pipelines[LocalResourceKey] = localPipelines 155 | } 156 | 157 | } else { 158 | 159 | for _, v := range Pipelines[GlobalResourceKey] { 160 | if idx := slices.IndexFunc(meshsyncConfig.BlackList, func(c string) bool { return c == v.Name }); idx == -1 { 161 | v.Events = DefaultEvents 162 | globalPipelines = append(globalPipelines, v) 163 | } 164 | } 165 | if len(globalPipelines) > 0 { 166 | meshsyncConfig.Pipelines = map[string]PipelineConfigs{} 167 | meshsyncConfig.Pipelines[GlobalResourceKey] = globalPipelines 168 | } 169 | 170 | // Handle local resources 171 | for _, v := range Pipelines[LocalResourceKey] { 172 | if idx := slices.IndexFunc(meshsyncConfig.BlackList, func(c string) bool { return c == v.Name }); idx == -1 { 173 | v.Events = DefaultEvents 174 | localPipelines = append(localPipelines, v) 175 | } 176 | } 177 | 178 | if len(localPipelines) > 0 { 179 | if meshsyncConfig.Pipelines == nil { 180 | meshsyncConfig.Pipelines = make(map[string]PipelineConfigs) 181 | } 182 | meshsyncConfig.Pipelines[LocalResourceKey] = localPipelines 183 | } 184 | } 185 | 186 | return meshsyncConfig, nil 187 | } 188 | 189 | func PatchCRVersion(config *rest.Config) error { 190 | meshsyncClient, err := client.New(config) 191 | if err != nil { 192 | return ErrInitConfig(fmt.Errorf("unable to update MeshSync configuration")) 193 | } 194 | 195 | patchedResource := map[string]interface{}{ 196 | "spec": map[string]interface{}{ 197 | "version": Server["version"], 198 | }, 199 | } 200 | byt, err := utils.Marshal(patchedResource) 201 | if err != nil { 202 | return ErrInitConfig(fmt.Errorf("unable to update MeshSync configuration")) 203 | } 204 | _, err = meshsyncClient.CoreV1Alpha1().MeshSyncs("meshery").Patch(context.TODO(), crName, types.MergePatchType, []byte(byt), metav1.PatchOptions{}) 205 | if err != nil { 206 | return ErrInitConfig(fmt.Errorf("unable to update MeshSync configuration")) 207 | } 208 | return nil 209 | } 210 | -------------------------------------------------------------------------------- /internal/config/crd_config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // test for empty blacklist/whitelist 4 | import ( 5 | "context" 6 | "reflect" 7 | "testing" 8 | 9 | corev1 "k8s.io/api/core/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/client-go/dynamic/fake" 12 | ) 13 | 14 | var ( 15 | Kind string = "MeshSync" 16 | APIVersion string = "meshery.io/v1alpha1" 17 | URL string = "https://meshery.io" 18 | fakeDyClient *fake.FakeDynamicClient 19 | ctx = context.Background() 20 | ) 21 | 22 | func TestWhiteListResources(t *testing.T) { 23 | 24 | // Create an instance of the custom resource. 25 | watchList := corev1.ConfigMap{ 26 | TypeMeta: metav1.TypeMeta{ 27 | APIVersion: "v1apha1", 28 | Kind: "ConfigMap", 29 | }, 30 | ObjectMeta: metav1.ObjectMeta{ 31 | Name: "watch-list", 32 | Namespace: "default", 33 | }, 34 | Data: map[string]string{ 35 | "blacklist": "", 36 | "whitelist": "[{\"Resource\":\"namespaces.v1.\",\"Events\":[\"ADDED\",\"DELETE\"]},{\"Resource\":\"replicasets.v1.apps\",\"Events\":[\"ADDED\",\"DELETE\"]},{\"Resource\":\"pods.v1.\",\"Events\":[\"MODIFIED\"]}]", 37 | }, 38 | } 39 | 40 | meshsyncConfig, err := PopulateConfigs(watchList) 41 | 42 | if err != nil { 43 | t.Errorf("Meshsync config not well deserialized got %s", err.Error()) 44 | } 45 | 46 | if len(meshsyncConfig.WhiteList) == 0 { 47 | t.Errorf("WhiteListed resources not correctly deserialized") 48 | } 49 | expectedWhiteList := []ResourceConfig{ 50 | {Resource: "namespaces.v1.", Events: []string{"ADDED", "DELETE"}}, 51 | {Resource: "replicasets.v1.apps", Events: []string{"ADDED", "DELETE"}}, 52 | {Resource: "pods.v1.", Events: []string{"MODIFIED"}}, 53 | } 54 | 55 | if !reflect.DeepEqual(meshsyncConfig.WhiteList, expectedWhiteList) { 56 | t.Error("WhiteListed resources not equal") 57 | } 58 | 59 | // now we assertain the global and local pipelines have been correctly configured 60 | // global pipelines: namespaces 61 | // local pipelines: pods, replicasets 62 | 63 | if len(meshsyncConfig.Pipelines["global"]) != 1 { 64 | t.Error("global pipelines not well configured expected 1") 65 | } 66 | 67 | if len(meshsyncConfig.Pipelines["local"]) != 2 { 68 | t.Error("global pipelines not well configured expected 2") 69 | } 70 | } 71 | 72 | func TestBlackListResources(t *testing.T) { 73 | 74 | // Create an instance of the custom resource. 75 | watchList := corev1.ConfigMap{ 76 | TypeMeta: metav1.TypeMeta{ 77 | APIVersion: "v1apha1", 78 | Kind: "ConfigMap", 79 | }, 80 | ObjectMeta: metav1.ObjectMeta{ 81 | Name: "watch-list", 82 | Namespace: "default", 83 | }, 84 | Data: map[string]string{ 85 | "blacklist": "[\"namespaces.v1.\",\"replicasets.v1.apps\",\"pods.v1.\"]", 86 | "whitelist": "", 87 | }, 88 | } 89 | 90 | meshsyncConfig, err := PopulateConfigs(watchList) 91 | 92 | if err != nil { 93 | t.Errorf("Meshsync config not well deserialized got %s", err.Error()) 94 | } 95 | 96 | if len(meshsyncConfig.BlackList) == 0 { 97 | t.Errorf("Blacklisted resources not correctly deserialized") 98 | } 99 | 100 | expectedBlackList := []string{"namespaces.v1.", "replicasets.v1.apps", "pods.v1."} 101 | if !reflect.DeepEqual(meshsyncConfig.BlackList, expectedBlackList) { 102 | t.Error("BlackListed resources not equal") 103 | } 104 | 105 | // now we assertain the global and local pipelines have been correctly configured 106 | // excempted global pipelines: namespaces 107 | // excempted local pipelines: pods, replicasets 108 | 109 | // counting expected pipelines after blacklist 110 | expectedGlobalCount := len(Pipelines[GlobalResourceKey]) - 1 //excluding namespaces 111 | expectedLocalCount := len(Pipelines[LocalResourceKey]) - 2 //excluding pods, replicasets 112 | 113 | if len(meshsyncConfig.Pipelines["global"]) != expectedGlobalCount { 114 | t.Errorf("global pipelines not well configured expected %d", expectedGlobalCount) 115 | } 116 | 117 | if len(meshsyncConfig.Pipelines["local"]) != expectedLocalCount { 118 | t.Errorf("local pipelines not well configured expected %d", expectedLocalCount) 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /internal/config/default_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | var ( 8 | Server = map[string]string{ 9 | "name": "meshery-meshsync", 10 | "port": "11000", 11 | "version": "latest", 12 | "startedat": time.Now().String(), 13 | } 14 | 15 | DefaultPublishingSubject = "meshery.meshsync.core" 16 | 17 | Pipelines = map[string]PipelineConfigs{ 18 | GlobalResourceKey: []PipelineConfig{ 19 | // Core Resources 20 | { 21 | Name: "namespaces.v1.", 22 | PublishTo: DefaultPublishingSubject, 23 | }, 24 | { 25 | Name: "configmaps.v1.", 26 | PublishTo: "meshery.meshsync.core", 27 | }, 28 | { 29 | Name: "nodes.v1.", 30 | PublishTo: "meshery.meshsync.core", 31 | }, 32 | { 33 | Name: "secrets.v1.", 34 | PublishTo: "meshery.meshsync.core", 35 | }, 36 | { 37 | Name: "persistentvolumes.v1.", 38 | PublishTo: "meshery.meshsync.core", 39 | }, 40 | { 41 | Name: "persistentvolumeclaims.v1.", 42 | PublishTo: "meshery.meshsync.core", 43 | }, 44 | }, 45 | LocalResourceKey: []PipelineConfig{ 46 | // Core Resources 47 | { 48 | Name: "replicasets.v1.apps", 49 | PublishTo: DefaultPublishingSubject, 50 | }, 51 | { 52 | Name: "pods.v1.", 53 | PublishTo: DefaultPublishingSubject, 54 | }, 55 | { 56 | Name: "services.v1.", 57 | PublishTo: DefaultPublishingSubject, 58 | }, 59 | { 60 | Name: "deployments.v1.apps", 61 | PublishTo: DefaultPublishingSubject, 62 | }, 63 | { 64 | Name: "statefulsets.v1.apps", 65 | PublishTo: DefaultPublishingSubject, 66 | }, 67 | { 68 | Name: "daemonsets.v1.apps", 69 | PublishTo: DefaultPublishingSubject, 70 | }, 71 | //Added Ingress support 72 | { 73 | Name: "ingresses.v1.networking.k8s.io", 74 | PublishTo: DefaultPublishingSubject, 75 | }, 76 | //Added 77 | { 78 | Name: "ingressclass.v1.networking.k8s.io", 79 | PublishTo: DefaultPublishingSubject, 80 | }, 81 | // Added endpoint support 82 | { 83 | Name: "endpoints.v1.", 84 | PublishTo: DefaultPublishingSubject, 85 | }, 86 | //Added endpointslice support 87 | { 88 | Name: "endpointslices.v1.discovery.k8s.io", 89 | PublishTo: DefaultPublishingSubject, 90 | }, 91 | //Added container support 92 | { 93 | Name: "container.v1.core", 94 | PublishTo: DefaultPublishingSubject, 95 | }, 96 | //Added job support 97 | { 98 | Name: "job.v1.batch", 99 | PublishTo: DefaultPublishingSubject, 100 | }, 101 | //Added service APIs support 102 | { 103 | Name: "service.apis", 104 | PublishTo: DefaultPublishingSubject, 105 | }, 106 | //Added csidriver support 107 | { 108 | Name: "csidriver.v1.storage.k8s.io", 109 | PublishTo: DefaultPublishingSubject, 110 | }, 111 | //Added csinode support 112 | { 113 | Name: "csinode.v1.storage.k8s.io", 114 | PublishTo: DefaultPublishingSubject, 115 | }, 116 | //Added csistoragecapacity support 117 | { 118 | Name: "csistoragecapacity.v1.storage.k8s.io", 119 | PublishTo: DefaultPublishingSubject, 120 | }, 121 | //Added volume support 122 | { 123 | Name: "volume.v1.", 124 | PublishTo: DefaultPublishingSubject, 125 | }, 126 | //Added volumeattributesclass support 127 | { 128 | Name: "volumeattributesclass.v1alpha1.storage.k8s.io", 129 | PublishTo: DefaultPublishingSubject, 130 | }, 131 | //Added clustertrustbundle support 132 | { 133 | Name: "clustertrustbundle.v1alpha1.certificates.k8s.io", 134 | PublishTo: DefaultPublishingSubject, 135 | }, 136 | //Added controllerversion support 137 | { 138 | Name: "controllerrevision.v1.apps", 139 | PublishTo: DefaultPublishingSubject, 140 | }, 141 | //Added customresourcedefinition support 142 | { 143 | Name: "customresourcedefinition.v1.apiextensions.k8s.io", 144 | PublishTo: DefaultPublishingSubject, 145 | }, 146 | //Added event support 147 | { 148 | Name: "event.v1.events.k8s.io", 149 | PublishTo: DefaultPublishingSubject, 150 | }, 151 | //Added limitrange support 152 | { 153 | Name: "limitrange.v1.", 154 | PublishTo: DefaultPublishingSubject, 155 | }, 156 | //Added horizontalpodautoscaler support 157 | { 158 | Name: "horizontalpodautoscaler.v2.autoscaling", 159 | PublishTo: DefaultPublishingSubject, 160 | }, 161 | //Added mutatingwebhookconfiguration support 162 | { 163 | Name: "mutatingwebhookconfiguration.v1.admissionregistration.k8s.io", 164 | PublishTo: DefaultPublishingSubject, 165 | }, 166 | //Added podschedulingcontext support 167 | { 168 | Name: "podschedulingcontext.v1alpha2.resource.k8s.io", 169 | PublishTo: DefaultPublishingSubject, 170 | }, 171 | //Added podtemplate support 172 | { 173 | Name: "podtemplate.v1.", 174 | PublishTo: DefaultPublishingSubject, 175 | }, 176 | //Added poddistruptionbudget support 177 | { 178 | Name: "poddisruptionbudget.v1.policy", 179 | PublishTo: DefaultPublishingSubject, 180 | }, 181 | //Added priorityclass support 182 | { 183 | Name: "priorityclass.v1.scheduling.k8s.io", 184 | PublishTo: DefaultPublishingSubject, 185 | }, 186 | //Added resourceclaim support 187 | { 188 | Name: "resourceclaim.v1alpha2.resource.k8s.io", 189 | PublishTo: DefaultPublishingSubject, 190 | }, 191 | //Added resourceclaimtemplate support 192 | { 193 | Name: "resourceclaimtemplate.v1alpha2.resource.k8s.io", 194 | PublishTo: DefaultPublishingSubject, 195 | }, 196 | //Added resourceclass support 197 | { 198 | Name: "resourceclass.v1alpha2.resource.k8s.io", 199 | PublishTo: DefaultPublishingSubject, 200 | }, 201 | //Added ValidatingWebhookConfiguration support 202 | { 203 | Name: "validatingwebhookconfiguration.v1.admissionregistration.k8s.io", 204 | PublishTo: DefaultPublishingSubject, 205 | }, 206 | //Added ValidatingAdmissionPolicy support 207 | { 208 | Name: "validatingadmissionpolicy.v1beta1.admissionregistration.k8s.io", 209 | PublishTo: DefaultPublishingSubject, 210 | }, 211 | //Added ValidatingAdmissionPolicyBinding support 212 | { 213 | Name: "validatingadmissionpolicybinding.v1beta1.admissionregistration.k8s.io", 214 | PublishTo: DefaultPublishingSubject, 215 | }, 216 | //Added binding support 217 | { 218 | Name: "binding.v1.", 219 | PublishTo: DefaultPublishingSubject, 220 | }, 221 | //Added certificatesigningrequest support 222 | { 223 | Name: "certificatesigningrequest.v1.certificates.k8s.io", 224 | PublishTo: DefaultPublishingSubject, 225 | }, 226 | 227 | //Added clusterrolebinding support 228 | { 229 | Name: "clusterrolebinding.v1.rbac.authorization.k8s.io", 230 | PublishTo: DefaultPublishingSubject, 231 | }, 232 | //Added componentstatus support 233 | { 234 | Name: "componentstatus.v1.", 235 | PublishTo: DefaultPublishingSubject, 236 | }, 237 | //Added flowschema support 238 | { 239 | Name: "flowschema.v1.flowcontrol.apiserver.k8s.io", 240 | PublishTo: DefaultPublishingSubject, 241 | }, 242 | //Added IPAddress support 243 | { 244 | Name: "ipaddress.v1alpha1.networking.k8s.io", 245 | PublishTo: DefaultPublishingSubject, 246 | }, 247 | //Added Lease support 248 | { 249 | Name: "lease.v1.coordination.k8s.io", 250 | PublishTo: DefaultPublishingSubject, 251 | }, 252 | //Added LocalSubjectAccessReview support 253 | { 254 | Name: "localsubjectaccessreview.v1.authorization.k8s.io", 255 | PublishTo: DefaultPublishingSubject, 256 | }, 257 | //Added Node support 258 | { 259 | Name: "node.v1.", 260 | PublishTo: DefaultPublishingSubject, 261 | }, 262 | //Added NetworkPolicy support 263 | { 264 | Name: "networkpolicy.v1.networking.k8s.io", 265 | PublishTo: DefaultPublishingSubject, 266 | }, 267 | //Added PriorityLevelConfiguration support 268 | { 269 | Name: "prioritylevelconfiguration.v1.flowcontrol.apiserver.k8s.io", 270 | PublishTo: DefaultPublishingSubject, 271 | }, 272 | //Added resourcequota support 273 | { 274 | Name: "resourcequota.v1.", 275 | PublishTo: DefaultPublishingSubject, 276 | }, 277 | //Added Role support 278 | { 279 | Name: "role.v1.rbac.authorization.k8s.io", 280 | PublishTo: DefaultPublishingSubject, 281 | }, 282 | //Added RoleBinding support 283 | { 284 | Name: "rolebinding.v1.rbac.authorization.k8s.io", 285 | PublishTo: DefaultPublishingSubject, 286 | }, 287 | //Added runtimeclass support 288 | { 289 | Name: "runtimeclass.v1.node.k8s.io", 290 | PublishTo: DefaultPublishingSubject, 291 | }, 292 | //Added SelfSubjectAccessReview support 293 | { 294 | Name: "selfsubjectaccessreview.v1.authorization.k8s.io", 295 | PublishTo: DefaultPublishingSubject, 296 | }, 297 | //Added SelfSubjectReview support 298 | { 299 | Name: "selfsubjectreview.v1.authentication.k8s.io", 300 | PublishTo: DefaultPublishingSubject, 301 | }, 302 | //Added selfsubjectrulesreview support 303 | { 304 | Name: "selfsubjectrulesreview.v1.authorization.k8s.io", 305 | PublishTo: DefaultPublishingSubject, 306 | }, 307 | //Added ServiceAccount support 308 | { 309 | Name: "serviceaccount.v1.", 310 | PublishTo: DefaultPublishingSubject, 311 | }, 312 | //Added ServiceCIDR support 313 | { 314 | Name: "servicecidr.v1alpha1.networking.k8s.io", 315 | PublishTo: DefaultPublishingSubject, 316 | }, 317 | //Added StorageVersion support 318 | { 319 | Name: "storageversion.v1alpha1.internal.apiserver.k8s.io", 320 | PublishTo: DefaultPublishingSubject, 321 | }, 322 | //Added subjectaccessreview support 323 | { 324 | Name: "subjectaccessreview.v1.authorization.k8s.io", 325 | PublishTo: DefaultPublishingSubject, 326 | }, 327 | //Added TokenRequest support 328 | { 329 | Name: "tokenrequest.v1.authentication.k8s.io", 330 | PublishTo: DefaultPublishingSubject, 331 | }, 332 | //Added TokenReview support 333 | { 334 | Name: "tokenreview.v1.authentication.k8s.io", 335 | PublishTo: DefaultPublishingSubject, 336 | }, 337 | // Added cronJob support 338 | { 339 | Name: "cronjobs.v1.batch", 340 | PublishTo: DefaultPublishingSubject, 341 | }, 342 | //Added ReplicationController support 343 | { 344 | Name: "replicationcontrollers.v1.", 345 | PublishTo: DefaultPublishingSubject, 346 | }, 347 | //Added storageClass support 348 | { 349 | Name: "storageclasses.v1.storage.k8s.io", 350 | PublishTo: DefaultPublishingSubject, 351 | }, 352 | //Added ClusterRole support 353 | { 354 | Name: "clusterroles.v1.rbac.authorization.k8s.io", 355 | PublishTo: DefaultPublishingSubject, 356 | }, 357 | //Added VolumeAttachment support 358 | { 359 | Name: "volumeattachments.v1.storage.k8s.io", 360 | PublishTo: DefaultPublishingSubject, 361 | }, 362 | //Added apiservice support 363 | { 364 | Name: "apiservices.v1.apiregistration.k8s.io", 365 | PublishTo: DefaultPublishingSubject, 366 | }, 367 | }, 368 | } 369 | 370 | Listeners = map[string]ListenerConfig{ 371 | LogStream: { 372 | Name: LogStream, 373 | ConnectionName: "meshsync-logstream", 374 | PublishTo: "meshery.meshsync.logs", 375 | }, 376 | ExecShell: { 377 | Name: ExecShell, 378 | ConnectionName: "meshsync-exec", 379 | PublishTo: "meshery.meshsync.exec", 380 | }, 381 | RequestStream: { 382 | Name: RequestStream, 383 | ConnectionName: "meshsync-request-stream", 384 | SubscribeTo: "meshery.meshsync.request", 385 | }, 386 | } 387 | 388 | DefaultEvents = []string{"ADD", "UPDATE", "DELETE"} 389 | ) 390 | -------------------------------------------------------------------------------- /internal/config/error.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/meshery/meshkit/errors" 5 | ) 6 | 7 | const ( 8 | ErrInitConfigCode = "1000" 9 | ) 10 | 11 | func ErrInitConfig(err error) error { 12 | return errors.New(ErrInitConfigCode, errors.Alert, []string{"Error while initializing MeshSync configuration. ", err.Error()}, []string{"Missing or outdated CRD. "}, []string{"Missing or outdated CRD."}, []string{"Confirm that meshsyncs custom resource is present in the cluster."}) 13 | } 14 | -------------------------------------------------------------------------------- /internal/config/types.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "golang.org/x/exp/slices" 5 | ) 6 | 7 | const ( 8 | ServerKey = "server-config" 9 | PipelineNameKey = "meshsync-pipeline" 10 | ResourcesKey = "resources" 11 | GlobalResourceKey = "global" 12 | LocalResourceKey = "local" 13 | ListenersKey = "listeners" 14 | LogStreamsKey = "log-streams" 15 | BrokerURL = "broker-url" 16 | RequestStream = "request-stream" 17 | LogStream = "log-stream" 18 | ExecShell = "exec-shell" 19 | InformerStore = "informer-store" 20 | OutputModeNats = "nats" 21 | OutputModeFile = "file" 22 | ) 23 | 24 | // Command line flag to determine the output mode 25 | var ( 26 | OutputMode string 27 | OutputFileName string 28 | OutputNamespace string 29 | StopAfterSeconds int 30 | OutputResourcesSet map[string]bool 31 | OutputOnlySpecifiedResources bool 32 | ) 33 | 34 | type PipelineConfigs []PipelineConfig 35 | 36 | func (p PipelineConfigs) Add(pc PipelineConfig) PipelineConfigs { 37 | p = append(p, pc) 38 | return p 39 | } 40 | 41 | func (p PipelineConfigs) Delete(pc PipelineConfig) PipelineConfigs { 42 | for index, pipelineConfig := range p { 43 | if pipelineConfig.Name == pc.Name { 44 | p = slices.Delete[PipelineConfigs](p, index, index+1) 45 | break 46 | } 47 | } 48 | return p 49 | } 50 | 51 | type PipelineConfig struct { 52 | Name string `json:"name" yaml:"name"` 53 | PublishTo string `json:"publish-to" yaml:"publish-to"` 54 | Events []string `json:"events" yaml:"events"` 55 | } 56 | 57 | type ListenerConfigs []ListenerConfig 58 | 59 | type ListenerConfig struct { 60 | Name string `json:"name" yaml:"name"` 61 | ConnectionName string `json:"connection-name" yaml:"connection-name"` 62 | PublishTo string `json:"publish-to" yaml:"publish-to"` 63 | SubscribeTo string `json:"subscribe-to" yaml:"subscribe-to"` 64 | Events []string `json:"events" yaml:"events"` 65 | } 66 | 67 | // Meshsync configuration controls the resources meshsync produces and consumes 68 | type MeshsyncConfig struct { 69 | BlackList []string `json:"blacklist" yaml:"blacklist"` 70 | Pipelines map[string]PipelineConfigs `json:"pipeline-configs,omitempty" yaml:"pipeline-configs,omitempty"` 71 | Listeners map[string]ListenerConfig `json:"listener-config,omitempty" yaml:"listener-config,omitempty"` 72 | WhiteList []ResourceConfig `json:"resource-configs" yaml:"resource-configs"` 73 | } 74 | 75 | // Watched Resource configuration 76 | type ResourceConfig struct { 77 | Resource string 78 | Events []string 79 | } 80 | -------------------------------------------------------------------------------- /internal/file/file.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "time" 8 | ) 9 | 10 | func GenerateUniqueFileNameForSnapshot(format string) (string, error) { 11 | // Get the current date and time 12 | currentTime := time.Now() 13 | 14 | // Format the current date in YYYYMMDD format 15 | currentDate := currentTime.Format("20060102") 16 | 17 | slug := fmt.Sprintf("meshery-cluster-snapshot-%s", currentDate) 18 | 19 | name := "" 20 | gotTheName := false 21 | for i := 0; i < 1024; i++ { 22 | name = fmt.Sprintf("%s-%02d.%s", slug, i, format) 23 | // Use os.Stat to check if the file exists 24 | _, err := os.Stat(name) 25 | if os.IsNotExist(err) { 26 | gotTheName = true 27 | break 28 | } 29 | } 30 | 31 | if !gotTheName { 32 | return "", errors.New("no unique name available") 33 | } 34 | return name, nil 35 | } 36 | -------------------------------------------------------------------------------- /internal/file/writer.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | type Writer interface { 4 | Write(data any) (int, error) 5 | } 6 | -------------------------------------------------------------------------------- /internal/file/yaml_writer.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "io" 5 | "os" 6 | 7 | "sigs.k8s.io/yaml" 8 | ) 9 | 10 | type YAMLWriter struct { 11 | filename string 12 | writeCloser io.WriteCloser 13 | } 14 | 15 | func NewYAMLWriter(filename string) (*YAMLWriter, error) { 16 | descriptor, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 17 | if err != nil { 18 | return nil, err 19 | } 20 | return &YAMLWriter{ 21 | filename: filename, 22 | writeCloser: io.WriteCloser(descriptor), 23 | }, nil 24 | } 25 | 26 | var delimiter = []byte{'-', '-', '-', '\n'} 27 | 28 | func (w *YAMLWriter) Write(data any) (int, error) { 29 | bytes, err := yaml.Marshal(data) 30 | if err != nil { 31 | return 0, err 32 | } 33 | 34 | bytes = append(bytes, delimiter...) 35 | return w.writeCloser.Write(bytes) 36 | } 37 | 38 | func (w *YAMLWriter) Close() error { 39 | return w.writeCloser.Close() 40 | } 41 | -------------------------------------------------------------------------------- /internal/output/composite.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/meshery/meshkit/broker" 7 | "github.com/meshery/meshsync/internal/config" 8 | "github.com/meshery/meshsync/pkg/model" 9 | ) 10 | 11 | // a wrapper which allows to have multiple writers under one entity 12 | type CompositeWriter struct { 13 | writersPool []Writer 14 | } 15 | 16 | func NewCompositeWriter(writer ...Writer) *CompositeWriter { 17 | return &CompositeWriter{ 18 | writersPool: writer, 19 | } 20 | } 21 | 22 | func (w *CompositeWriter) Write( 23 | obj model.KubernetesResource, 24 | evtype broker.EventType, 25 | config config.PipelineConfig, 26 | ) error { 27 | errs := make([]error, 0, len(w.writersPool)) 28 | 29 | for _, writer := range w.writersPool { 30 | if err := writer.Write(obj, evtype, config); err != nil { 31 | errs = append(errs, err) 32 | } 33 | } 34 | 35 | if len(errs) > 0 { 36 | return errors.Join(errs...) 37 | } 38 | 39 | return nil 40 | } 41 | -------------------------------------------------------------------------------- /internal/output/file.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "github.com/meshery/meshkit/broker" 5 | "github.com/meshery/meshsync/internal/config" 6 | "github.com/meshery/meshsync/internal/file" 7 | "github.com/meshery/meshsync/pkg/model" 8 | ) 9 | 10 | type FileWriter struct { 11 | fw file.Writer 12 | } 13 | 14 | func NewFileWriter(fw file.Writer) *FileWriter { 15 | return &FileWriter{ 16 | fw: fw, 17 | } 18 | } 19 | 20 | func (s *FileWriter) Write( 21 | obj model.KubernetesResource, 22 | evtype broker.EventType, 23 | config config.PipelineConfig, 24 | ) error { 25 | _, err := s.fw.Write(obj) 26 | if err != nil { 27 | return err 28 | } 29 | return nil 30 | } 31 | -------------------------------------------------------------------------------- /internal/output/inmemory_deduplicator.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/meshery/meshkit/broker" 7 | "github.com/meshery/meshsync/internal/config" 8 | "github.com/meshery/meshsync/pkg/model" 9 | ) 10 | 11 | // instead of direct write to output destination 12 | // InMemoryDeduplicatorWriter collects data in memory identifying entity by metadata.uid 13 | // and write to output only on program exit 14 | type InMemoryDeduplicatorWriter struct { 15 | realWritter Writer 16 | // for this entities for which model.KubernetesResource.KubernetesResourceMeta != nil 17 | storage map[string]*inMemoryDeduplicatorContainer 18 | // as model.KubernetesResource.KubernetesResourceMeta could be nil 19 | // treat such entities as unique 20 | // and just put them in this slice 21 | storageIfNoMetaUid []*inMemoryDeduplicatorContainer 22 | } 23 | 24 | func NewInMemoryDeduplicatorWriter(realWritter Writer) *InMemoryDeduplicatorWriter { 25 | return &InMemoryDeduplicatorWriter{ 26 | realWritter: realWritter, 27 | storage: make(map[string]*inMemoryDeduplicatorContainer), 28 | storageIfNoMetaUid: make([]*inMemoryDeduplicatorContainer, 0, 128), 29 | } 30 | } 31 | 32 | func (w *InMemoryDeduplicatorWriter) Write( 33 | obj model.KubernetesResource, 34 | evtype broker.EventType, 35 | config config.PipelineConfig, 36 | ) error { 37 | uid := "" 38 | if obj.KubernetesResourceMeta != nil { 39 | uid = obj.KubernetesResourceMeta.UID 40 | } 41 | 42 | entity := &inMemoryDeduplicatorContainer{ 43 | obj: obj, 44 | evtype: evtype, 45 | config: config, 46 | } 47 | 48 | if uid != "" { 49 | w.storage[uid] = entity 50 | } else { 51 | w.storageIfNoMetaUid = append(w.storageIfNoMetaUid, entity) 52 | } 53 | 54 | return nil 55 | } 56 | 57 | func (w *InMemoryDeduplicatorWriter) Flush() error { 58 | errs := make([]error, 0, len(w.storage)+len(w.storageIfNoMetaUid)) 59 | 60 | for _, v := range w.storage { 61 | if err := w.realWritter.Write(v.obj, v.evtype, v.config); err != nil { 62 | errs = append(errs, err) 63 | } 64 | } 65 | 66 | for _, v := range w.storageIfNoMetaUid { 67 | if err := w.realWritter.Write(v.obj, v.evtype, v.config); err != nil { 68 | errs = append(errs, err) 69 | } 70 | } 71 | 72 | if len(errs) > 0 { 73 | return errors.Join(errs...) 74 | } 75 | 76 | return nil 77 | } 78 | 79 | type inMemoryDeduplicatorContainer struct { 80 | obj model.KubernetesResource 81 | evtype broker.EventType 82 | config config.PipelineConfig 83 | } 84 | -------------------------------------------------------------------------------- /internal/output/nats.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "github.com/meshery/meshkit/broker" 5 | "github.com/meshery/meshsync/internal/config" 6 | "github.com/meshery/meshsync/pkg/model" 7 | ) 8 | 9 | type NatsWriter struct { 10 | br broker.Handler 11 | } 12 | 13 | func NewNatsWriter(br broker.Handler) *NatsWriter { 14 | return &NatsWriter{ 15 | br: br, 16 | } 17 | } 18 | 19 | func (s *NatsWriter) Write( 20 | obj model.KubernetesResource, 21 | evtype broker.EventType, 22 | config config.PipelineConfig, 23 | ) error { 24 | return s.br.Publish( 25 | config.PublishTo, 26 | &broker.Message{ 27 | ObjectType: broker.MeshSync, 28 | EventType: evtype, 29 | Object: obj, 30 | }, 31 | ) 32 | } 33 | -------------------------------------------------------------------------------- /internal/output/output.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "github.com/meshery/meshkit/broker" 5 | "github.com/meshery/meshsync/internal/config" 6 | "github.com/meshery/meshsync/pkg/model" 7 | ) 8 | 9 | type Writer interface { 10 | Write( 11 | obj model.KubernetesResource, 12 | evtype broker.EventType, 13 | config config.PipelineConfig, 14 | ) error 15 | } 16 | -------------------------------------------------------------------------------- /internal/output/processor.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "github.com/meshery/meshkit/broker" 5 | "github.com/meshery/meshsync/internal/config" 6 | "github.com/meshery/meshsync/pkg/model" 7 | ) 8 | 9 | type Processor struct { 10 | output Writer 11 | } 12 | 13 | func NewProcessor() *Processor { 14 | return &Processor{} 15 | } 16 | 17 | func (p *Processor) SetOutput(output Writer) { 18 | p.output = output 19 | } 20 | 21 | func (p *Processor) Write( 22 | obj model.KubernetesResource, 23 | evtype broker.EventType, 24 | config config.PipelineConfig, 25 | ) error { 26 | return p.output.Write(obj, evtype, config) 27 | } 28 | -------------------------------------------------------------------------------- /internal/pipeline/error.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "github.com/meshery/meshkit/errors" 5 | ) 6 | 7 | const ( 8 | ErrListCode = "1001" 9 | ErrPublishCode = "1002" 10 | ErrDynamicClientCode = "1003" 11 | ErrCacheSyncCode = "1014" 12 | ErrWriteOutputCode = "1015" 13 | ) 14 | 15 | func ErrDynamicClient(name string, err error) error { 16 | return errors.New(ErrDynamicClientCode, errors.Alert, []string{"Error creating dynamic client for: " + name, err.Error()}, []string{}, []string{}, []string{}) 17 | } 18 | 19 | func ErrList(name string, err error) error { 20 | return errors.New(ErrListCode, errors.Alert, []string{"Error while listing: " + name, err.Error()}, []string{}, []string{}, []string{}) 21 | } 22 | 23 | func ErrPublish(name string, err error) error { 24 | return errors.New(ErrPublishCode, errors.Alert, []string{"Error while publishing for: " + name, err.Error()}, []string{}, []string{}, []string{}) 25 | } 26 | 27 | func ErrCacheSync(name string, err error) error { 28 | return errors.New(ErrCacheSyncCode, errors.Alert, []string{"Error while syncing the informer store for: " + name, err.Error()}, []string{}, []string{}, []string{}) 29 | } 30 | 31 | func ErrWriteOutput(name string, err error) error { 32 | return errors.New(ErrWriteOutputCode, errors.Alert, []string{"Error while writing output for: " + name, err.Error()}, []string{}, []string{}, []string{}) 33 | } 34 | -------------------------------------------------------------------------------- /internal/pipeline/handlers.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | 8 | "github.com/meshery/meshkit/broker" 9 | internalconfig "github.com/meshery/meshsync/internal/config" 10 | "github.com/meshery/meshsync/pkg/model" 11 | "golang.org/x/exp/slices" 12 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 13 | "k8s.io/client-go/tools/cache" 14 | ) 15 | 16 | func (ri *RegisterInformer) GetEventHandlers() cache.ResourceEventHandlerFuncs { 17 | return cache.ResourceEventHandlerFuncs{ 18 | AddFunc: func(obj interface{}) { 19 | err := ri.publishItem(obj.(*unstructured.Unstructured), broker.Add, ri.config) 20 | if err != nil { 21 | ri.log.Error(err) 22 | } 23 | ri.log.Info("Received ADD event for: ", obj.(*unstructured.Unstructured).GetName(), "/", obj.(*unstructured.Unstructured).GetNamespace(), " of kind: ", obj.(*unstructured.Unstructured).GroupVersionKind().Kind) 24 | }, 25 | UpdateFunc: func(oldObj, obj interface{}) { 26 | oldObjCasted := oldObj.(*unstructured.Unstructured) 27 | objCasted := obj.(*unstructured.Unstructured) 28 | 29 | oldRV, _ := strconv.ParseInt(oldObjCasted.GetResourceVersion(), 0, 64) 30 | newRV, _ := strconv.ParseInt(objCasted.GetResourceVersion(), 0, 64) 31 | 32 | if oldRV < newRV { 33 | err := ri.publishItem(obj.(*unstructured.Unstructured), broker.Update, ri.config) 34 | 35 | if err != nil { 36 | ri.log.Error(err) 37 | } 38 | ri.log.Info("Received UPDATE event for: ", obj.(*unstructured.Unstructured).GetName(), "/", obj.(*unstructured.Unstructured).GetNamespace(), " of kind: ", obj.(*unstructured.Unstructured).GroupVersionKind().Kind) 39 | } else { 40 | ri.log.Debug(fmt.Sprintf( 41 | "Skipping UPDATE event for: %s => [No changes detected]: %d %d", 42 | objCasted.GetName(), 43 | oldRV, 44 | newRV, 45 | )) 46 | } 47 | }, 48 | DeleteFunc: func(obj interface{}) { 49 | // the obj can only be of two types, Unstructured or DeletedFinalStateUnknown. 50 | // DeletedFinalStateUnknown means that the object that we receive may be `stale` 51 | // because of the way informer behaves 52 | 53 | // refer 'https://pkg.go.dev/k8s.io/client-go/tools/cache#ResourceEventHandler.OnDelete' 54 | 55 | var objCasted *unstructured.Unstructured 56 | objCasted = obj.(*unstructured.Unstructured) 57 | 58 | possiblyStaleObj, ok := obj.(cache.DeletedFinalStateUnknown) 59 | if ok { 60 | objCasted = possiblyStaleObj.Obj.(*unstructured.Unstructured) 61 | } 62 | err := ri.publishItem(objCasted, broker.Delete, ri.config) 63 | 64 | if err != nil { 65 | ri.log.Error(err) 66 | } 67 | ri.log.Info("Received DELETE event for: ", obj.(*unstructured.Unstructured).GetName(), "/", obj.(*unstructured.Unstructured).GetNamespace(), " of kind: ", obj.(*unstructured.Unstructured).GroupVersionKind().Kind) 68 | }, 69 | } 70 | } 71 | 72 | func (ri *RegisterInformer) registerHandlers(s cache.SharedIndexInformer) { 73 | s.AddEventHandler(ri.GetEventHandlers()) // nolint 74 | } 75 | 76 | func (ri *RegisterInformer) publishItem(obj *unstructured.Unstructured, evtype broker.EventType, config internalconfig.PipelineConfig) error { 77 | 78 | // if the event is not supported skip 79 | if !slices.Contains(ri.config.Events, string(evtype)) { 80 | return nil 81 | } 82 | k8sResource := model.ParseList(*obj, evtype) 83 | 84 | mustSkip := false 85 | 86 | if internalconfig.OutputNamespace != "" && 87 | obj.GetNamespace() != internalconfig.OutputNamespace { 88 | mustSkip = true 89 | } 90 | 91 | if internalconfig.OutputOnlySpecifiedResources && 92 | !internalconfig.OutputResourcesSet[strings.ToLower(k8sResource.Kind)] { 93 | mustSkip = true 94 | } 95 | 96 | if mustSkip { 97 | // skip this resource 98 | ri.log.Info("Skipping resource: ", obj.GetName(), "/", obj.GetNamespace(), " of kind: ", k8sResource.Kind) 99 | return nil 100 | 101 | } 102 | 103 | if err := ri.outputWriter.Write( 104 | k8sResource, 105 | evtype, 106 | config, 107 | ); err != nil { 108 | ri.log.Error(ErrWriteOutput(config.Name, err)) 109 | return err 110 | } 111 | 112 | return nil 113 | } 114 | -------------------------------------------------------------------------------- /internal/pipeline/pipeline.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "github.com/meshery/meshkit/logger" 5 | internalconfig "github.com/meshery/meshsync/internal/config" 6 | "github.com/meshery/meshsync/internal/output" 7 | "github.com/myntra/pipeline" 8 | "k8s.io/client-go/dynamic/dynamicinformer" 9 | ) 10 | 11 | var ( 12 | Name = internalconfig.PipelineNameKey 13 | GlobalDiscoveryStage = &pipeline.Stage{ 14 | Name: internalconfig.GlobalResourceKey, 15 | Concurrent: false, 16 | Steps: []pipeline.Step{}, 17 | } 18 | 19 | LocalDiscoveryStage = &pipeline.Stage{ 20 | Name: internalconfig.LocalResourceKey, 21 | Concurrent: false, 22 | Steps: []pipeline.Step{}, 23 | } 24 | 25 | StartInformersStage = &pipeline.Stage{ 26 | Name: "StartInformers", 27 | Concurrent: false, 28 | Steps: []pipeline.Step{}, 29 | } 30 | ) 31 | 32 | func New(log logger.Handler, informer dynamicinformer.DynamicSharedInformerFactory, ow output.Writer, plConfigs map[string]internalconfig.PipelineConfigs, stopChan chan struct{}) *pipeline.Pipeline { 33 | // Global discovery 34 | gdstage := GlobalDiscoveryStage 35 | configs := plConfigs[gdstage.Name] 36 | for _, config := range configs { 37 | gdstage.AddStep(newRegisterInformerStep(log, informer, config, ow)) // Register the informers for different resources 38 | } 39 | 40 | // Local discovery 41 | ldstage := LocalDiscoveryStage 42 | configs = plConfigs[ldstage.Name] 43 | for _, config := range configs { 44 | ldstage.AddStep(newRegisterInformerStep(log, informer, config, ow)) // Register the informers for different resources 45 | } 46 | 47 | // Start informers 48 | strtInfmrs := StartInformersStage 49 | strtInfmrs.AddStep(newStartInformersStep(stopChan, log, informer)) // Start the registered informers 50 | 51 | // Create Pipeline 52 | clusterPipeline := pipeline.New(Name, 1000) 53 | clusterPipeline.AddStage(gdstage) 54 | clusterPipeline.AddStage(ldstage) 55 | clusterPipeline.AddStage(strtInfmrs) 56 | 57 | return clusterPipeline 58 | } 59 | -------------------------------------------------------------------------------- /internal/pipeline/step.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/meshery/meshkit/logger" 7 | internalconfig "github.com/meshery/meshsync/internal/config" 8 | "github.com/meshery/meshsync/internal/output" 9 | "github.com/myntra/pipeline" 10 | "k8s.io/apimachinery/pkg/runtime/schema" 11 | "k8s.io/client-go/dynamic/dynamicinformer" 12 | "k8s.io/client-go/tools/cache" 13 | ) 14 | 15 | type RegisterInformer struct { 16 | pipeline.StepContext 17 | log logger.Handler 18 | informer dynamicinformer.DynamicSharedInformerFactory 19 | config internalconfig.PipelineConfig 20 | outputWriter output.Writer 21 | } 22 | 23 | func newRegisterInformerStep(log logger.Handler, informer dynamicinformer.DynamicSharedInformerFactory, config internalconfig.PipelineConfig, ow output.Writer) *RegisterInformer { 24 | return &RegisterInformer{ 25 | log: log, 26 | informer: informer, 27 | config: config, 28 | outputWriter: ow, 29 | } 30 | } 31 | 32 | // TODO: Find a way to respond when an informer has stopped for some reason unknown 33 | // Exec - step interface 34 | func (ri *RegisterInformer) Exec(request *pipeline.Request) *pipeline.Result { 35 | gvr, _ := schema.ParseResourceArg(ri.config.Name) 36 | if gvr == nil { 37 | return &pipeline.Result{ 38 | Error: internalconfig.ErrInitConfig(fmt.Errorf("error parsing resource arg, gvr not found")), 39 | Data: nil, 40 | } 41 | } 42 | 43 | iclient := ri.informer.ForResource(*gvr) 44 | 45 | ri.registerHandlers(iclient.Informer()) 46 | 47 | // add the instance of store to the Result 48 | data := make(map[string]cache.Store) 49 | if request.Data != nil { 50 | data = request.Data.(map[string]cache.Store) 51 | } 52 | data[ri.config.Name] = iclient.Informer().GetStore() 53 | return &pipeline.Result{ 54 | Error: nil, 55 | Data: data, 56 | } 57 | } 58 | 59 | // Cancel - step interface 60 | func (ri *RegisterInformer) Cancel() error { 61 | ri.Status("cancel step") 62 | return nil 63 | } 64 | 65 | // StartInformers Step 66 | 67 | type StartInformers struct { 68 | pipeline.StepContext 69 | stopChan chan struct{} 70 | informer dynamicinformer.DynamicSharedInformerFactory 71 | log logger.Handler 72 | } 73 | 74 | func newStartInformersStep(stopChan chan struct{}, log logger.Handler, informer dynamicinformer.DynamicSharedInformerFactory) *StartInformers { 75 | return &StartInformers{ 76 | log: log, 77 | informer: informer, 78 | stopChan: stopChan, 79 | } 80 | } 81 | 82 | func (si *StartInformers) Exec(request *pipeline.Request) *pipeline.Result { 83 | si.informer.WaitForCacheSync(si.stopChan) 84 | si.informer.Start(si.stopChan) 85 | return &pipeline.Result{ 86 | Error: nil, 87 | Data: request.Data, 88 | } 89 | } 90 | 91 | // Cancel - step interface 92 | func (si *StartInformers) Cancel() error { 93 | si.Status("cancel step") 94 | return nil 95 | } 96 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // TODO fix cyclop error 2 | // Error: main.go:1:1: the average complexity for the package main is 7.166667, max is 7.000000 (cyclop) 3 | // 4 | //nolint:cyclop 5 | package main 6 | 7 | import ( 8 | "errors" 9 | "flag" 10 | "fmt" 11 | "net/http" 12 | "os" 13 | "os/signal" 14 | "path" 15 | "strings" 16 | "syscall" 17 | "time" 18 | 19 | "github.com/meshery/meshkit/broker" 20 | "github.com/meshery/meshkit/broker/nats" 21 | configprovider "github.com/meshery/meshkit/config/provider" 22 | "github.com/meshery/meshkit/logger" 23 | mesherykube "github.com/meshery/meshkit/utils/kubernetes" 24 | "github.com/meshery/meshsync/internal/channels" 25 | "github.com/meshery/meshsync/internal/config" 26 | "github.com/meshery/meshsync/internal/file" 27 | "github.com/meshery/meshsync/internal/output" 28 | "github.com/meshery/meshsync/meshsync" 29 | "github.com/sirupsen/logrus" 30 | "github.com/spf13/viper" 31 | ) 32 | 33 | var ( 34 | serviceName = "meshsync" 35 | provider = configprovider.ViperKey 36 | version = "Not Set" 37 | commitsha = "Not Set" 38 | pingEndpoint = ":8222/connz" 39 | ) 40 | 41 | func main() { 42 | parseFlags() 43 | viper.SetDefault("BUILD", version) 44 | viper.SetDefault("COMMITSHA", commitsha) 45 | 46 | // Initialize Logger instance 47 | log, errLoggerNew := logger.New(serviceName, logger.Options{ 48 | Format: logger.SyslogLogFormat, 49 | LogLevel: int(logrus.InfoLevel), 50 | }) 51 | if errLoggerNew != nil { 52 | fmt.Println(errLoggerNew) 53 | os.Exit(1) 54 | } 55 | 56 | if err := mainWithError(log); err != nil { 57 | log.Error(err) 58 | os.Exit(1) 59 | } 60 | } 61 | 62 | // TODO fix cyclop error 63 | // Error: main.go:46:1: calculated cyclomatic complexity for function mainWithExitCode is 25, max is 10 (cyclop) 64 | // 65 | //nolint:cyclop 66 | func mainWithError(log logger.Handler) error { 67 | // Initialize kubeclient 68 | kubeClient, err := mesherykube.New(nil) 69 | if err != nil { 70 | return err 71 | } 72 | 73 | useCRDFlag := determineUseCRDFlag(log, kubeClient) 74 | 75 | crdConfigs, errGetMeshsyncCRDConfigs := getMeshsyncCRDConfigs(useCRDFlag, kubeClient) 76 | if errGetMeshsyncCRDConfigs != nil { 77 | // no configs found from meshsync CRD log warning 78 | log.Warn(err) 79 | } 80 | // Config init and seed 81 | cfg, err := config.New(provider) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | config.Server["version"] = version 87 | err = cfg.SetObject(config.ServerKey, config.Server) 88 | if err != nil { 89 | return err 90 | } 91 | 92 | if useCRDFlag { 93 | // this patch only make sense when CRD is present in cluster 94 | if errPatchCRVersion := config.PatchCRVersion(&kubeClient.RestConfig); errPatchCRVersion != nil { 95 | log.Warn(errPatchCRVersion) 96 | } 97 | } 98 | 99 | // pass configs from crd to default configs 100 | if crdConfigs != nil { 101 | if len(crdConfigs.Pipelines) > 0 { 102 | config.Pipelines = crdConfigs.Pipelines 103 | } 104 | 105 | if len(crdConfigs.Listeners) > 0 { 106 | config.Listeners = crdConfigs.Listeners 107 | } 108 | } 109 | 110 | cfg.SetKey(config.BrokerURL, os.Getenv("BROKER_URL")) 111 | 112 | err = cfg.SetObject(config.ResourcesKey, config.Pipelines) 113 | if err != nil { 114 | return err 115 | } 116 | 117 | err = cfg.SetObject(config.ListenersKey, config.Listeners) 118 | if err != nil { 119 | return err 120 | } 121 | 122 | outputProcessor := output.NewProcessor() 123 | var br broker.Handler 124 | if config.OutputMode == config.OutputModeNats { 125 | // Skip/Comment the below connectivity test in local environment 126 | if errConnectivityTest := connectivityTest(cfg.GetKey(config.BrokerURL), log); errConnectivityTest != nil { 127 | return errConnectivityTest 128 | } 129 | // Initialize Broker instance 130 | broker, errNatsNew := nats.New(nats.Options{ 131 | URLS: []string{cfg.GetKey(config.BrokerURL)}, 132 | ConnectionName: "meshsync", 133 | Username: "", 134 | Password: "", 135 | ReconnectWait: 2 * time.Second, 136 | MaxReconnect: 60, 137 | }) 138 | if errNatsNew != nil { 139 | return errNatsNew 140 | } 141 | br = broker 142 | outputProcessor.SetOutput( 143 | output.NewNatsWriter( 144 | br, 145 | ), 146 | ) 147 | } 148 | 149 | if config.OutputMode == config.OutputModeFile { 150 | filename := config.OutputFileName 151 | defaultFormat := "yaml" 152 | if filename == "" { 153 | fname, errGenerateUniqueFileNameForSnapshot := file.GenerateUniqueFileNameForSnapshot(defaultFormat) 154 | if errGenerateUniqueFileNameForSnapshot != nil { 155 | return errGenerateUniqueFileNameForSnapshot 156 | } 157 | filename = fname 158 | } 159 | ext := path.Ext(filename) 160 | if ext == "" { 161 | ext = "." + defaultFormat 162 | } 163 | // this is a file which contains all messages from nats 164 | // (hence it also contains more than one yaml manifest for the same entity) 165 | fw, errNewYAMLWriter := file.NewYAMLWriter( 166 | fmt.Sprintf( 167 | "%s-extended%s", 168 | strings.TrimSuffix(filename, ext), 169 | ext, 170 | ), 171 | ) 172 | if errNewYAMLWriter != nil { 173 | return errNewYAMLWriter 174 | } 175 | defer fw.Close() 176 | 177 | // this is a file which contains only unique resource's messages from nats 178 | // it filters out duplicates and writes only latest message from nats per resource 179 | fw2, errNewYAMLWriter2 := file.NewYAMLWriter(filename) 180 | if errNewYAMLWriter2 != nil { 181 | return errNewYAMLWriter2 182 | } 183 | // this one not written immediately, 184 | // but collects in memory and flushes in the end 185 | outputInMemoryDeduplicatorWriter := output.NewInMemoryDeduplicatorWriter( 186 | output.NewFileWriter(fw2), 187 | ) 188 | // ensure to flush 189 | defer outputInMemoryDeduplicatorWriter.Flush() 190 | 191 | outputProcessor.SetOutput( 192 | output.NewCompositeWriter( 193 | output.NewFileWriter(fw), 194 | outputInMemoryDeduplicatorWriter, 195 | ), 196 | ) 197 | } 198 | 199 | chPool := channels.NewChannelPool() 200 | meshsyncHandler, err := meshsync.New(cfg, log, br, outputProcessor, chPool) 201 | if err != nil { 202 | return err 203 | } 204 | 205 | go meshsyncHandler.WatchCRDs() 206 | 207 | go meshsyncHandler.Run() 208 | if config.OutputMode == config.OutputModeNats { 209 | // even so the config param name is OutputMode 210 | // it is not only output but also input 211 | // in that case if OutputMode is not OutputModeNats 212 | // there is no nats at all, so we do not subscribe to any topic 213 | go meshsyncHandler.ListenToRequests() 214 | } 215 | 216 | if config.StopAfterSeconds > -1 { 217 | go func(stopCh channels.StopChannel) { 218 | <-time.After(time.Second * time.Duration(config.StopAfterSeconds)) 219 | log.Infof("Stopping after %d seconds", config.StopAfterSeconds) 220 | stopCh <- struct{}{} 221 | // close(stopCh) 222 | }(chPool[channels.Stop].(channels.StopChannel)) 223 | } 224 | 225 | log.Info("Server started") 226 | // Handle graceful shutdown 227 | signal.Notify(chPool[channels.OS].(channels.OSChannel), syscall.SIGTERM, os.Interrupt) 228 | select { 229 | case <-chPool[channels.OS].(channels.OSChannel): 230 | close(chPool[channels.Stop].(channels.StopChannel)) 231 | case <-chPool[channels.Stop].(channels.StopChannel): 232 | // // NOTE: 233 | // // does not make sense to close the StopChannel here, 234 | // // as the general approach with stop channel to close it rather then put smth in it, 235 | // // and hence next close will create panic if stop channel is already closed 236 | // // so commented this out: 237 | // close(chPool[channels.Stop].(channels.StopChannel)) 238 | } 239 | 240 | log.Info("Shutting down") 241 | 242 | return nil 243 | } 244 | 245 | func connectivityTest(url string, log logger.Handler) error { 246 | // Make sure Broker has started before starting NATS client 247 | urls := strings.Split(url, ":") 248 | if len(urls) == 0 { 249 | return errors.New("invalid URL") 250 | } 251 | pingURL := "http://" + urls[0] + pingEndpoint 252 | for { 253 | resp, err := http.Get(pingURL) //nolint 254 | if err != nil { 255 | log.Info("could not connect to broker: " + err.Error() + " retrying...") 256 | time.Sleep(1 * time.Second) 257 | continue 258 | } 259 | if resp.StatusCode == http.StatusOK { 260 | break 261 | } 262 | log.Info("could not receive OK response from broker: "+pingURL, " retrying...") 263 | time.Sleep(1 * time.Second) 264 | } 265 | 266 | return nil 267 | } 268 | 269 | func parseFlags() { 270 | notUsedBrokerURL := "" 271 | flag.StringVar( 272 | ¬UsedBrokerURL, 273 | "broker-url", 274 | "", 275 | "Broker URL (note: primarily configured via BROKER_URL env var; this flag is for compatibility and its value is ignored).", 276 | ) 277 | flag.StringVar( 278 | &config.OutputMode, 279 | "output", 280 | config.OutputModeNats, 281 | fmt.Sprintf("output mode: \"%s\" or \"%s\"", config.OutputModeNats, config.OutputModeFile), 282 | ) 283 | flag.StringVar( 284 | &config.OutputFileName, 285 | "outputFile", 286 | "", 287 | "output file where to put the meshsync events (cluster snapshot), only applicable for file output mode (default \"./meshery-cluster-snapshot-YYYYMMDD-00.yaml\")", 288 | ) 289 | flag.StringVar( 290 | &config.OutputNamespace, 291 | "outputNamespace", 292 | "", 293 | "k8s namespace for which limit the output, f.e. \"default\", applicable for both nats and file output mode", 294 | ) 295 | var outputResourcesString string 296 | flag.StringVar( 297 | &outputResourcesString, 298 | "outputResources", 299 | "", 300 | "k8s resources for which limit the output, coma separated case insensitive list of k8s resources, f.e. \"pod,deployment,service\", applicable for both nats and file output mode", 301 | ) 302 | flag.IntVar( 303 | &config.StopAfterSeconds, 304 | "stopAfterSeconds", 305 | -1, 306 | "stop meshsync execution after specified amount of seconds", 307 | ) 308 | 309 | // Parse the command=line flags to get the output mode 310 | flag.Parse() 311 | 312 | config.OutputResourcesSet = make(map[string]bool) 313 | if outputResourcesString != "" { 314 | config.OutputOnlySpecifiedResources = true 315 | outputResourcesList := strings.Split(outputResourcesString, ",") 316 | for _, item := range outputResourcesList { 317 | config.OutputResourcesSet[strings.ToLower(item)] = true 318 | } 319 | } 320 | } 321 | 322 | func determineUseCRDFlag(log logger.Handler, kubeClient *mesherykube.Client) bool { 323 | useCRDFlag := true 324 | if config.OutputMode == config.OutputModeFile { 325 | // if output mode is file -> generally it is not expected to have CRD present in cluster. 326 | // theoretically CRDs could be present even in file output mode. 327 | // hence check if CRD is present in the cluster, 328 | // and only skip them in file output mode if it is not present. 329 | crd, errGetMeshsyncCRD := config.GetMeshsyncCRD(kubeClient.DynamicKubeClient) 330 | if crd != nil && errGetMeshsyncCRD == nil { 331 | // this is rare, but valid case 332 | log.Info("running in file output mode and meshsync CRD is present in the cluster") 333 | } else { 334 | useCRDFlag = false 335 | // this is the most common case, file mode and no CRD 336 | log.Info("running in file output mode and NO meshsync CRD is present in the cluster (expected behaviour)") 337 | } 338 | } 339 | return useCRDFlag 340 | } 341 | 342 | func getMeshsyncCRDConfigs(useCRDFlag bool, kubeClient *mesherykube.Client) (*config.MeshsyncConfig, error) { 343 | if useCRDFlag { 344 | // get configs from meshsync crd if available 345 | return config.GetMeshsyncCRDConfigs(kubeClient.DynamicKubeClient) 346 | } 347 | // get configs from local variable 348 | return config.GetMeshsyncCRDConfigsLocal() 349 | } 350 | -------------------------------------------------------------------------------- /meshsync/discovery.go: -------------------------------------------------------------------------------- 1 | // Copyright Meshery Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package meshsync 16 | 17 | import ( 18 | "github.com/meshery/meshsync/internal/config" 19 | "github.com/meshery/meshsync/internal/pipeline" 20 | "k8s.io/client-go/tools/cache" 21 | ) 22 | 23 | func (h *Handler) startDiscovery(pipelineCh chan struct{}) { 24 | pipelineConfigs := make(map[string]config.PipelineConfigs, 10) 25 | err := h.Config.GetObject(config.ResourcesKey, &pipelineConfigs) 26 | if err != nil { 27 | h.Log.Error(ErrGetObject(err)) 28 | return 29 | } 30 | 31 | h.Log.Info("Pipeline started") 32 | pl := pipeline.New(h.Log, h.informer, h.outputWriter, pipelineConfigs, pipelineCh) 33 | result := pl.Run() 34 | h.stores = result.Data.(map[string]cache.Store) 35 | if result.Error != nil { 36 | h.Log.Error(ErrNewPipeline(result.Error)) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /meshsync/error.go: -------------------------------------------------------------------------------- 1 | // Copyright Meshery Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package meshsync 16 | 17 | import ( 18 | "github.com/meshery/meshkit/errors" 19 | ) 20 | 21 | var ( 22 | ErrGetObjectCode = "1004" 23 | ErrNewPipelineCode = "1005" 24 | ErrNewInformerCode = "1006" 25 | ErrKubeConfigCode = "1007" 26 | ErrInitRequestCode = "1008" 27 | ErrSubscribeRequestCode = "1009" 28 | ErrLogStreamCode = "1010" 29 | ErrCopyBufferCode = "1011" 30 | ErrInvalidRequestCode = "1012" 31 | ErrExecTerminalCode = "1013" 32 | 33 | ErrInvalidRequest = errors.New(ErrInvalidRequestCode, errors.Alert, []string{"Request is invalid"}, []string{}, []string{"Stale request on the broker"}, []string{"Make sure the request format is correctly configured"}) 34 | ) 35 | 36 | func ErrGetObject(err error) error { 37 | return errors.New(ErrGetObjectCode, errors.Alert, []string{"Error getting config object"}, []string{err.Error()}, []string{"Config doesnt exist"}, []string{"Check application config is configured correct or restart the server"}) 38 | } 39 | 40 | func ErrNewPipeline(err error) error { 41 | return errors.New(ErrNewPipelineCode, errors.Alert, []string{"Error creating pipeline"}, []string{err.Error()}, []string{"Pipeline step failed"}, []string{"Investigate on the respective pipeline step that has failed to figure the cause"}) 42 | } 43 | 44 | func ErrNewInformer(err error) error { 45 | return errors.New(ErrNewInformerCode, errors.Alert, []string{"Error initializing informer"}, []string{err.Error()}, []string{"Resource is invalid or doesnt exist"}, []string{"Make sure to input the existing valid resource"}) 46 | } 47 | 48 | func ErrKubeConfig(err error) error { 49 | return errors.New(ErrKubeConfigCode, errors.Alert, []string{"Error creating kubernetes client"}, []string{err.Error()}, []string{"Kubernetes config is invalid or APi server is not reachable"}, []string{"Make sure to upload a valid kubernetes config", "Make sure kubernetes API server is reachable"}) 50 | } 51 | 52 | func ErrInitRequest(err error) error { 53 | return errors.New(ErrInitRequestCode, errors.Alert, []string{"Error while initializing requests channel"}, []string{err.Error()}, []string{"Application resource deficit"}, []string{"Make sure meshsync has enough resources to create channels"}) 54 | } 55 | 56 | func ErrSubscribeRequest(err error) error { 57 | return errors.New(ErrSubscribeRequestCode, errors.Alert, []string{"Error while subscribing to requests"}, []string{err.Error()}, []string{"Broker resource deficit"}, []string{"Make sure Broker has enough resources to create channels"}) 58 | } 59 | 60 | func ErrLogStream(err error) error { 61 | return errors.New(ErrLogStreamCode, errors.Alert, []string{"Error while open log stream connection"}, []string{err.Error()}, []string{"requested Resource could be invalid"}, []string{"Make sure the requested resource is valid and existing"}) 62 | } 63 | 64 | func ErrExecTerminal(err error) error { 65 | return errors.New(ErrExecTerminalCode, errors.Alert, []string{"Error while opening a terminal session"}, []string{err.Error()}, []string{"requested Resource could be invalid"}, []string{"Make sure the requested resource is valid and existing"}) 66 | } 67 | 68 | func ErrCopyBuffer(err error) error { 69 | return errors.New(ErrCopyBufferCode, errors.Alert, []string{"Error while copying log buffer"}, []string{err.Error()}, []string{}, []string{}) 70 | } 71 | -------------------------------------------------------------------------------- /meshsync/exec.go: -------------------------------------------------------------------------------- 1 | // Copyright Meshery Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package meshsync 16 | 17 | import ( 18 | "bufio" 19 | "encoding/json" 20 | "fmt" 21 | "io" 22 | "os" 23 | "strings" 24 | "time" 25 | 26 | "github.com/google/uuid" 27 | "github.com/meshery/meshkit/broker" 28 | "github.com/meshery/meshkit/utils" 29 | "github.com/meshery/meshsync/internal/channels" 30 | "github.com/meshery/meshsync/internal/config" 31 | "github.com/meshery/meshsync/pkg/model" 32 | "golang.org/x/net/context" 33 | corev1 "k8s.io/api/core/v1" 34 | "k8s.io/client-go/kubernetes/scheme" 35 | "k8s.io/client-go/tools/remotecommand" 36 | "k8s.io/kubectl/pkg/util/interrupt" 37 | "k8s.io/kubectl/pkg/util/term" 38 | ) 39 | 40 | // KB stands for KiloByte 41 | const KB = 1024 42 | 43 | func (h *Handler) processExecRequest(obj interface{}, cfg config.ListenerConfig) error { 44 | reqs := make(model.ExecRequests) 45 | d, err := utils.Marshal(obj) 46 | if err != nil { 47 | return err 48 | } 49 | 50 | err = json.Unmarshal([]byte(d), &reqs) 51 | if err != nil { 52 | return err 53 | } 54 | 55 | for _, req := range reqs { 56 | id := fmt.Sprintf("exec.%s.%s.%s.%s", req.Namespace, req.Name, req.Container, req.ID) 57 | if _, ok := h.channelPool[id]; !ok { 58 | // Subscribing the first time 59 | if !bool(req.Stop) { 60 | h.channelPool[id] = channels.NewStructChannel() 61 | h.Log.Info("Starting session") 62 | 63 | err := h.Broker.Publish("active_sessions.exec", &broker.Message{ 64 | ObjectType: broker.ActiveExecObject, 65 | Object: h.getActiveChannels(), 66 | }) 67 | if err != nil { 68 | h.Log.Error(ErrGetObject(err)) 69 | } 70 | go h.streamSession(id, req, cfg) 71 | } 72 | } else { 73 | // Already running subscription 74 | if bool(req.Stop) { 75 | // TODO: once we have a unsubscribe functionality, need to publish message to active sessions subject 76 | execCleanup(h, id) 77 | } 78 | } 79 | } 80 | 81 | return nil 82 | } 83 | func (h *Handler) processActiveExecRequest() error { 84 | go h.streamChannelPool() 85 | 86 | return nil 87 | } 88 | func (h *Handler) getActiveChannels() []*string { 89 | activeChannels := make([]*string, 0, len(h.channelPool)) 90 | for k := range h.channelPool { 91 | activeChannels = append(activeChannels, &k) 92 | } 93 | 94 | return activeChannels 95 | } 96 | 97 | func (h *Handler) streamChannelPool() { 98 | go func() { 99 | for { 100 | err := h.Broker.Publish("active_sessions.exec", &broker.Message{ 101 | ObjectType: broker.ActiveExecObject, 102 | Object: h.getActiveChannels(), 103 | }) 104 | if err != nil { 105 | h.Log.Error(ErrGetObject(err)) 106 | } 107 | 108 | time.Sleep(10 * time.Second) 109 | } 110 | }() 111 | } 112 | 113 | // TODO fix cyclop error 114 | // Error: meshsync/exec.go:113:1: calculated cyclomatic complexity for function streamSession is 15, max is 10 (cyclop) 115 | // 116 | //nolint:cyclop 117 | func (h *Handler) streamSession(id string, req model.ExecRequest, cfg config.ListenerConfig) { 118 | subCh := make(chan *broker.Message) 119 | tstdin, putStdin := io.Pipe() 120 | stdin := io.NopCloser(tstdin) 121 | getStdout, stdout := io.Pipe() 122 | 123 | err := h.Broker.SubscribeWithChannel(fmt.Sprintf("input.%s", id), generateID(), subCh) 124 | if err != nil { 125 | h.Log.Error(ErrExecTerminal(err)) 126 | } 127 | 128 | // Put the terminal into raw mode to prevent it echoing characters twice. 129 | t := term.TTY{ 130 | Parent: interrupt.New(func(s os.Signal) {}), 131 | Out: stdout, 132 | In: stdin, 133 | Raw: true, 134 | } 135 | sizeQueue := t.MonitorSize(t.GetSize()) 136 | 137 | // TTY request GoRoutine 138 | go func() { 139 | fn := func() error { 140 | request := h.staticClient.CoreV1().RESTClient().Post(). 141 | Namespace(req.Namespace). 142 | Resource("pods"). 143 | Name(req.Name). 144 | SubResource("exec") 145 | request.VersionedParams(&corev1.PodExecOptions{ 146 | Container: req.Container, 147 | Command: []string{"/bin/sh"}, 148 | Stdin: true, 149 | Stdout: true, 150 | Stderr: true, 151 | TTY: true, 152 | }, scheme.ParameterCodec) 153 | 154 | exec, postErr := remotecommand.NewSPDYExecutor(&h.restConfig, "POST", request.URL()) 155 | if postErr != nil { 156 | return err 157 | } 158 | 159 | err = exec.StreamWithContext(context.TODO(), remotecommand.StreamOptions{ 160 | Stdin: stdin, 161 | Stdout: stdout, 162 | Stderr: stdout, 163 | Tty: true, 164 | TerminalSizeQueue: sizeQueue}) 165 | if err != nil { 166 | return err 167 | } 168 | 169 | // Cleanup the resources when the streaming process terminates 170 | execCleanup(h, id) 171 | return nil 172 | } 173 | 174 | if err = t.Safe(fn); err != nil { 175 | h.Log.Error(ErrExecTerminal(err)) 176 | execCleanup(h, id) 177 | 178 | // If the TTY fails then send the error message to the client 179 | if err = h.Broker.Publish(id, &broker.Message{ 180 | ObjectType: broker.ErrorObject, 181 | Object: err.Error(), 182 | }); err != nil { 183 | h.Log.Error(ErrExecTerminal(err)) 184 | } 185 | 186 | return 187 | } 188 | }() 189 | 190 | // TTY stdout streaming Goroutine 191 | go func() { 192 | rdr := bufio.NewReader(getStdout) 193 | for { 194 | data := make([]byte, 1*KB) 195 | _, err = rdr.Read(data) 196 | if err == io.EOF { 197 | break // No clean up here as this can generate a false positive 198 | } 199 | 200 | err = h.Broker.Publish(id, &broker.Message{ 201 | ObjectType: broker.ExecOutputObject, 202 | Object: string(data), 203 | }) 204 | if err != nil { 205 | h.Log.Error(ErrExecTerminal(err)) 206 | } 207 | } 208 | }() 209 | 210 | for { 211 | if _, ok := h.channelPool[id]; !ok { 212 | h.Log.Info("Session closed for: ", id) 213 | return 214 | } 215 | 216 | select { 217 | case msg := <-subCh: 218 | if msg.ObjectType == broker.ExecInputObject { 219 | _, err = io.CopyBuffer(putStdin, strings.NewReader(msg.Object.(string)+"\n"), nil) 220 | if err != nil { 221 | h.Log.Error(ErrExecTerminal(err)) 222 | } 223 | } 224 | case <-h.channelPool[id].(channels.StructChannel): 225 | h.Log.Info("Closing", id) 226 | delete(h.channelPool, id) 227 | } 228 | } 229 | } 230 | 231 | func execCleanup(h *Handler, id string) { 232 | ch, ok := h.channelPool[id] 233 | if !ok { 234 | return 235 | } 236 | 237 | structChan, ok := ch.(channels.StructChannel) 238 | if !ok { 239 | return 240 | } 241 | 242 | structChan <- struct{}{} 243 | } 244 | 245 | func generateID() string { 246 | return uuid.New().String() 247 | } 248 | -------------------------------------------------------------------------------- /meshsync/handlers.go: -------------------------------------------------------------------------------- 1 | package meshsync 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/meshery/meshkit/broker" 9 | "github.com/meshery/meshkit/utils" 10 | "github.com/meshery/meshkit/utils/kubernetes" 11 | "github.com/meshery/meshsync/internal/channels" 12 | "github.com/meshery/meshsync/internal/config" 13 | "github.com/meshery/meshsync/pkg/model" 14 | "golang.org/x/net/context" 15 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 16 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 17 | "k8s.io/apimachinery/pkg/runtime/schema" 18 | "k8s.io/apimachinery/pkg/watch" 19 | "k8s.io/client-go/dynamic" 20 | ) 21 | 22 | func debounce(d time.Duration, f func(ch chan struct{})) func(ch chan struct{}) { 23 | timer := time.NewTimer(d) 24 | return func(pipelineCh chan struct{}) { 25 | timer.Stop() 26 | timer = time.NewTimer(d) 27 | <-timer.C 28 | f(pipelineCh) 29 | timer.Reset(d) 30 | timer.Stop() 31 | } 32 | } 33 | 34 | func (h *Handler) Run() { 35 | pipelineCh := make(chan struct{}) 36 | go h.startDiscovery(pipelineCh) 37 | 38 | debouncedStartDiscovery := debounce(time.Second*5, func(pipelinechannel chan struct{}) { 39 | if !utils.IsClosed[struct{}](pipelinechannel) { 40 | h.Log.Info("closing previous instance ") 41 | close(pipelinechannel) 42 | } 43 | pipelineCh = make(chan struct{}) 44 | 45 | err := h.UpdateInformer() 46 | if err != nil { 47 | h.Log.Error(err) 48 | } 49 | h.Log.Info("starting over") 50 | h.startDiscovery(pipelineCh) 51 | 52 | }) 53 | for range h.channelPool[channels.ReSync].(channels.ReSyncChannel) { 54 | go debouncedStartDiscovery(pipelineCh) 55 | } 56 | } 57 | 58 | func (h *Handler) UpdateInformer() error { 59 | dynamicClient, err := dynamic.NewForConfig(&h.restConfig) 60 | if err != nil { 61 | return ErrNewInformer(err) 62 | } 63 | listOptionsFunc, err := GetListOptionsFunc(h.Config) 64 | if err != nil { 65 | return err 66 | } 67 | h.informer = GetDynamicInformer(h.Config, dynamicClient, listOptionsFunc) 68 | return nil 69 | } 70 | 71 | // TODO fix cyclop error 72 | // Error: meshsync/handlers.go:71:1: calculated cyclomatic complexity for function ListenToRequests is 19, max is 10 (cyclop) 73 | // 74 | //nolint:cyclop 75 | func (h *Handler) ListenToRequests() { 76 | listenerConfigs := make(map[string]config.ListenerConfig, 10) 77 | err := h.Config.GetObject(config.ListenersKey, &listenerConfigs) 78 | if err != nil { 79 | h.Log.Error(ErrGetObject(err)) 80 | } 81 | 82 | h.Log.Info("Listening for requests in: ", listenerConfigs[config.RequestStream].SubscribeTo) 83 | reqChan := make(chan *broker.Message) 84 | err = h.Broker.SubscribeWithChannel(listenerConfigs[config.RequestStream].SubscribeTo, listenerConfigs[config.RequestStream].ConnectionName, reqChan) 85 | if err != nil { 86 | h.Log.Error(ErrSubscribeRequest(err)) 87 | } 88 | 89 | for request := range reqChan { 90 | if request.Request == nil { 91 | h.Log.Error(ErrInvalidRequest) 92 | continue 93 | } 94 | 95 | switch request.Request.Entity { 96 | case broker.LogRequestEntity: 97 | h.Log.Info("Starting log session") 98 | err := h.processLogRequest(request.Request.Payload, listenerConfigs[config.LogStream]) 99 | if err != nil { 100 | h.Log.Error(err) 101 | continue 102 | } 103 | 104 | // TODO: Add this to the broker pkg 105 | case "informer-store": 106 | d, err := json.Marshal(request.Request.Payload) 107 | // TODO: Update broker pkg in Meshkit to include Reply types 108 | var payload struct{ Reply string } 109 | if err != nil { 110 | h.Log.Error(err) 111 | continue 112 | } 113 | err = json.Unmarshal(d, &payload) 114 | if err != nil { 115 | h.Log.Error(err) 116 | continue 117 | } 118 | replySubject := payload.Reply 119 | storeObjects := h.listStoreObjects() 120 | splitSlices := splitIntoMultipleSlices(storeObjects, 5) // performance of NATS is bound to degrade if huge messages are sent 121 | 122 | h.Log.Info("Publishing the data from informer stores to the subject: ", replySubject) 123 | for _, val := range splitSlices { 124 | err = h.Broker.Publish(replySubject, &broker.Message{ 125 | Object: val, 126 | }) 127 | if err != nil { 128 | h.Log.Error(err) 129 | continue 130 | } 131 | } 132 | 133 | case broker.ReSyncDiscoveryEntity: 134 | h.Log.Info("Resyncing") 135 | h.channelPool[channels.ReSync].(channels.ReSyncChannel) <- struct{}{} 136 | case broker.ExecRequestEntity: 137 | h.Log.Info("Starting interactive session") 138 | err := h.processExecRequest(request.Request.Payload, listenerConfigs[config.ExecShell]) 139 | if err != nil { 140 | h.Log.Error(err) 141 | continue 142 | } 143 | case broker.ActiveExecEntity: 144 | h.Log.Info("Connecting to channel pool") 145 | err := h.processActiveExecRequest() 146 | if err != nil { 147 | h.Log.Error(err) 148 | continue 149 | } 150 | case "meshsync-meta": 151 | h.Log.Info("Publishing MeshSync metadata to the subject") 152 | err := h.Broker.Publish("meshsync-meta", &broker.Message{ 153 | Object: config.Server["version"], 154 | }) 155 | if err != nil { 156 | h.Log.Error(err) 157 | continue 158 | } 159 | } 160 | } 161 | } 162 | 163 | func (h *Handler) listStoreObjects() []model.KubernetesResource { 164 | objects := make([]interface{}, 0) 165 | for _, v := range h.stores { 166 | objects = append(objects, v.List()...) 167 | } 168 | parsedObjects := make([]model.KubernetesResource, 0) 169 | for _, obj := range objects { 170 | parsedObjects = append(parsedObjects, model.ParseList(*obj.(*unstructured.Unstructured), broker.Add)) 171 | } 172 | return parsedObjects 173 | } 174 | 175 | func (h *Handler) WatchCRDs() { 176 | kubeclient, err := kubernetes.New(nil) 177 | if err != nil { 178 | h.Log.Error(err) 179 | return 180 | } 181 | 182 | crdWatcher, err := kubeclient.DynamicKubeClient.Resource(schema.GroupVersionResource{ 183 | Group: "apiextensions.k8s.io", 184 | Version: "v1", 185 | Resource: "customresourcedefinitions", 186 | }).Watch(context.Background(), metav1.ListOptions{}) 187 | 188 | if err != nil { 189 | h.Log.Error(err) 190 | return 191 | } 192 | 193 | for event := range crdWatcher.ResultChan() { 194 | 195 | crd := &kubernetes.CRDItem{} 196 | byt, err := json.Marshal(event.Object) 197 | if err != nil { 198 | h.Log.Error(err) 199 | continue 200 | } 201 | 202 | err = json.Unmarshal(byt, crd) 203 | if err != nil { 204 | h.Log.Error(err) 205 | continue 206 | } 207 | 208 | gvr := kubernetes.GetGVRForCustomResources(crd) 209 | 210 | existingPipelines := config.Pipelines 211 | err = h.Config.GetObject(config.ResourcesKey, existingPipelines) 212 | if err != nil { 213 | h.Log.Error(err) 214 | continue 215 | } 216 | 217 | existingPipelineConfigs := existingPipelines[config.GlobalResourceKey] 218 | 219 | configName := fmt.Sprintf("%s.%s.%s", gvr.Resource, gvr.Version, gvr.Group) 220 | updatedPipelineConfigs := existingPipelineConfigs 221 | 222 | switch event.Type { 223 | case watch.Added: 224 | // No need to verify if config is already added because If the config already exists then it indicates the informer has already synced that resource. 225 | // Any subsequent updates will have event type as "modified" 226 | updatedPipelineConfigs = existingPipelineConfigs.Add(config.PipelineConfig{ 227 | Name: configName, 228 | PublishTo: config.DefaultPublishingSubject, 229 | Events: []string{"ADDED", "MODIFIED", "DELETED"}, 230 | }) 231 | case watch.Deleted: 232 | updatedPipelineConfigs = existingPipelineConfigs.Delete(config.PipelineConfig{ 233 | Name: configName, 234 | }) 235 | } 236 | existingPipelines[config.GlobalResourceKey] = updatedPipelineConfigs 237 | err = h.Config.SetObject(config.ResourcesKey, existingPipelines) 238 | if err != nil { 239 | h.Log.Error(err) 240 | h.Log.Info("skipping informer resync") 241 | return 242 | } 243 | h.channelPool[channels.ReSync].(channels.ReSyncChannel).ReSyncInformer() 244 | } 245 | } 246 | 247 | // TODO: move this to meshkit 248 | // given [1,2,3,4,5,6,7,5,4,4] and 3 as its arguments, it would 249 | // return [[1,2,3], [4,5,6], [7,5,4], [4]] 250 | func splitIntoMultipleSlices(s []model.KubernetesResource, maxItmsPerSlice int) []([]model.KubernetesResource) { 251 | result := make([]([]model.KubernetesResource), 0) 252 | temp := make([]model.KubernetesResource, 0) 253 | 254 | for idx, val := range s { 255 | temp = append(temp, val) 256 | if ((idx + 1) % maxItmsPerSlice) == 0 { 257 | result = append(result, temp) 258 | temp = nil 259 | } 260 | if idx+1 == len(s) { 261 | if len(temp) != 0 { 262 | result = append(result, temp) 263 | } 264 | } 265 | } 266 | 267 | return result 268 | } 269 | -------------------------------------------------------------------------------- /meshsync/handlers_test.go: -------------------------------------------------------------------------------- 1 | package meshsync 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/meshery/meshsync/pkg/model" 8 | ) 9 | 10 | // TestSplitIntoMultipleSlices tests the splitIntoMultipleSlices function 11 | // by providing different input test cases and comparing the output with the expected output. 12 | func TestSplitIntoMultipleSlices(t *testing.T) { 13 | testCases := []struct { 14 | name string 15 | input []model.KubernetesResource 16 | maxItmsPerSlice int 17 | expectedOutput [][]model.KubernetesResource 18 | }{ 19 | { 20 | name: "test with 0 items", 21 | input: []model.KubernetesResource{}, 22 | maxItmsPerSlice: 10, 23 | expectedOutput: [][]model.KubernetesResource{}, 24 | }, 25 | 26 | { 27 | name: "test with 1 item", 28 | input: []model.KubernetesResource{ 29 | { 30 | Kind: "test", 31 | }, 32 | }, 33 | maxItmsPerSlice: 10, 34 | expectedOutput: [][]model.KubernetesResource{ 35 | { 36 | { 37 | Kind: "test", 38 | }, 39 | }, 40 | }, 41 | }, 42 | } 43 | 44 | for _, tc := range testCases { 45 | t.Run(tc.name, func(t *testing.T) { 46 | output := splitIntoMultipleSlices(tc.input, tc.maxItmsPerSlice) 47 | if !reflect.DeepEqual(output, tc.expectedOutput) { 48 | t.Errorf("expected %v, but got %v", tc.expectedOutput, output) 49 | } 50 | }) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /meshsync/logstream.go: -------------------------------------------------------------------------------- 1 | package meshsync 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | 9 | "github.com/meshery/meshkit/broker" 10 | "github.com/meshery/meshsync/internal/channels" 11 | "github.com/meshery/meshsync/internal/config" 12 | "github.com/meshery/meshsync/pkg/model" 13 | v1 "k8s.io/api/core/v1" 14 | ) 15 | 16 | func (h *Handler) processLogRequest(obj interface{}, cfg config.ListenerConfig) error { 17 | reqs := make(model.LogRequests) 18 | d, err := json.Marshal(obj) 19 | if err != nil { 20 | return err 21 | } 22 | 23 | err = json.Unmarshal(d, &reqs) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | for _, req := range reqs { 29 | id := fmt.Sprintf("logs.%s.%s.%s", req.Namespace, req.Name, req.Container) 30 | if _, ok := h.channelPool[id]; !ok { 31 | // Subscribing the first time 32 | if !bool(req.Stop) { 33 | h.channelPool[id] = channels.NewStructChannel() 34 | go h.streamLogs(id, req, cfg) 35 | } 36 | } else { 37 | // Already running subscription 38 | if bool(req.Stop) { 39 | h.channelPool[id].(channels.StructChannel) <- struct{}{} 40 | } 41 | } 42 | } 43 | 44 | return nil 45 | } 46 | 47 | func (h *Handler) streamLogs(id string, req model.LogRequest, cfg config.ListenerConfig) { 48 | resp, err := h.staticClient.CoreV1().Pods(req.Namespace).GetLogs(req.Name, &v1.PodLogOptions{ 49 | Container: req.Container, 50 | Follow: req.Follow, 51 | Previous: req.Previous, 52 | Timestamps: true, 53 | TailLines: &req.TailLines, 54 | //SinceSeconds: 55 | //SinceTime: 56 | //LimitBytes:, 57 | //InsecureSkipTLSVerifyBackend: true, 58 | }).Stream(context.TODO()) 59 | if err != nil { 60 | h.Log.Error(ErrLogStream(err)) 61 | delete(h.channelPool, id) 62 | return 63 | } 64 | 65 | go func() { 66 | <-h.channelPool[id].(channels.StructChannel) 67 | h.Log.Info("Closing", id) 68 | delete(h.channelPool, id) 69 | resp.Close() 70 | }() 71 | 72 | for { 73 | buf := make([]byte, 2000) 74 | numBytes, err := resp.Read(buf) 75 | if err == io.EOF { 76 | break 77 | } 78 | if numBytes == 0 { 79 | continue 80 | } 81 | if err != nil { 82 | h.Log.Error(ErrCopyBuffer(err)) 83 | delete(h.channelPool, id) 84 | } 85 | 86 | message := string(buf[:numBytes]) 87 | err = h.Broker.Publish(cfg.PublishTo, &broker.Message{ 88 | ObjectType: broker.LogStreamObject, 89 | EventType: broker.Add, 90 | Object: &model.LogObject{ 91 | ID: req.ID, 92 | Data: message, 93 | Primary: req.Name, 94 | Secondary: req.Container, 95 | }, 96 | }) 97 | if err != nil { 98 | h.Log.Error(ErrCopyBuffer(err)) 99 | } 100 | } 101 | 102 | } 103 | -------------------------------------------------------------------------------- /meshsync/meshsync.go: -------------------------------------------------------------------------------- 1 | package meshsync 2 | 3 | import ( 4 | "github.com/meshery/meshkit/broker" 5 | "github.com/meshery/meshkit/config" 6 | "github.com/meshery/meshkit/logger" 7 | mesherykube "github.com/meshery/meshkit/utils/kubernetes" 8 | "github.com/meshery/meshsync/internal/channels" 9 | "github.com/meshery/meshsync/internal/output" 10 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/client-go/dynamic" 12 | "k8s.io/client-go/dynamic/dynamicinformer" 13 | "k8s.io/client-go/kubernetes" 14 | "k8s.io/client-go/rest" 15 | "k8s.io/client-go/tools/cache" 16 | ) 17 | 18 | // Handler contains all handlers, channels, clients, and other parameters for an adapter. 19 | // Use type embedding in a specific adapter to extend it. 20 | type Handler struct { 21 | Config config.Handler 22 | Log logger.Handler 23 | Broker broker.Handler 24 | 25 | restConfig rest.Config 26 | informer dynamicinformer.DynamicSharedInformerFactory 27 | staticClient *kubernetes.Clientset 28 | channelPool map[string]channels.GenericChannel 29 | stores map[string]cache.Store 30 | outputWriter output.Writer 31 | } 32 | 33 | func GetListOptionsFunc(config config.Handler) (func(*v1.ListOptions), error) { 34 | var blacklist []string 35 | err := config.GetObject("spec.informer_config", blacklist) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | return func(lo *v1.ListOptions) { 41 | // Create a label selector to include all objects 42 | labelSelector := &v1.LabelSelector{} 43 | 44 | // Add label selector requirements to exclude blacklisted types 45 | labelSelectorReq := v1.LabelSelectorRequirement{ 46 | Key: "type", 47 | Operator: v1.LabelSelectorOpNotIn, 48 | Values: blacklist, 49 | } 50 | labelSelector.MatchExpressions = append(labelSelector.MatchExpressions, labelSelectorReq) 51 | }, nil 52 | } 53 | 54 | func New(config config.Handler, log logger.Handler, br broker.Handler, ow output.Writer, pool map[string]channels.GenericChannel) (*Handler, error) { 55 | // Initialize Kubeconfig 56 | kubeClient, err := mesherykube.New(nil) 57 | if err != nil { 58 | return nil, ErrKubeConfig(err) 59 | } 60 | listOptionsFunc, err := GetListOptionsFunc(config) 61 | if err != nil { 62 | return nil, err 63 | } 64 | 65 | informer := GetDynamicInformer(config, kubeClient.DynamicKubeClient, listOptionsFunc) 66 | 67 | return &Handler{ 68 | Config: config, 69 | Log: log, 70 | Broker: br, 71 | outputWriter: ow, 72 | informer: informer, 73 | restConfig: kubeClient.RestConfig, 74 | staticClient: kubeClient.KubeClient, 75 | channelPool: pool, 76 | }, nil 77 | } 78 | 79 | func GetDynamicInformer(config config.Handler, dynamicKubeClient dynamic.Interface, listOptionsFunc func(*v1.ListOptions)) dynamicinformer.DynamicSharedInformerFactory { 80 | return dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynamicKubeClient, 0, v1.NamespaceAll, listOptionsFunc) 81 | } 82 | -------------------------------------------------------------------------------- /pkg/model/exec.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | type ExecRequest struct { 4 | ID string `json:"id,omitempty"` 5 | Name string `json:"name,omitempty"` 6 | Namespace string `json:"namespace,omitempty"` 7 | Container string `json:"container,omitempty"` 8 | Stop bool `json:"stop,omitempty"` 9 | } 10 | 11 | type ExecObject struct { 12 | ID string `json:"id,omitempty"` 13 | Data string `json:"data,omitempty"` 14 | } 15 | 16 | type ExecRequests map[string]ExecRequest 17 | -------------------------------------------------------------------------------- /pkg/model/log.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | type LogObject struct { 4 | ID string `json:"id,omitempty"` 5 | Data string `json:"data,omitempty"` 6 | // Name of Pod, Svc, Deply getting logs for 7 | Primary string `json:"primary,omitempty"` 8 | // Specific detail about the log like Container or Pod Name 9 | Secondary string `json:"secondary,omitempty"` 10 | } 11 | 12 | type LogRequest struct { 13 | ID string `json:"id,omitempty"` 14 | Name string `json:"name,omitempty"` 15 | Namespace string `json:"namespace,omitempty"` 16 | Container string `json:"container,omitempty"` 17 | Follow bool `json:"follow,omitempty"` 18 | Previous bool `json:"previous,omitempty"` 19 | TailLines int64 `json:"taillines,omitempty"` 20 | Stop bool `json:"stop,omitempty"` 21 | } 22 | 23 | type LogRequests map[string]LogRequest 24 | -------------------------------------------------------------------------------- /pkg/model/model.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "github.com/google/uuid" 5 | "gorm.io/gorm" 6 | ) 7 | 8 | const ( 9 | KindLabel string = "label" 10 | KindAnnotation string = "annotation" 11 | ) 12 | 13 | type KubernetesResource struct { 14 | ID string `json:"id" gorm:"primarykey"` 15 | APIVersion string `json:"apiVersion" gorm:"index"` 16 | Kind string `json:"kind" gorm:"index"` 17 | Model string `json:"model" gorm:"index"` 18 | KubernetesResourceMeta *KubernetesResourceObjectMeta `json:"metadata" gorm:"foreignkey:ID;references:id;constraint:OnUpdate:CASCADE,OnDelete:SET NULL;"` 19 | Spec *KubernetesResourceSpec `json:"spec,omitempty" gorm:"foreignkey:ID;references:id;constraint:OnUpdate:CASCADE,OnDelete:SET NULL;"` 20 | Status *KubernetesResourceStatus `json:"status,omitempty" gorm:"foreignkey:ID;references:id;constraint:OnUpdate:CASCADE,OnDelete:SET NULL;"` 21 | ClusterID string `json:"cluster_id"` 22 | PatternResource *uuid.UUID `json:"pattern_resource"` 23 | ComponentMetadata map[string]interface{} `json:"component_metadata" gorm:"type:bytes;serializer:json"` 24 | // Secondary fields for configsmaps and secrets 25 | Immutable string `json:"immutable,omitempty"` 26 | Data string `json:"data,omitempty"` 27 | BinaryData string `json:"binaryData,omitempty"` 28 | StringData string `json:"stringData,omitempty"` 29 | Type string `json:"type,omitempty"` 30 | } 31 | 32 | type KubernetesKeyValue struct { 33 | ID string `json:"id" gorm:"primarykey"` 34 | UniqueID string `json:"unique_id" gorm:"index"` 35 | Kind string `json:"kind" gorm:"primarykey"` 36 | Key string `json:"key,omitempty" gorm:"primarykey"` 37 | Value string `json:"value,omitempty" gorm:"primarykey"` 38 | } 39 | 40 | type KubernetesResourceObjectMeta struct { 41 | ID string `json:"id" gorm:"primarykey"` 42 | Name string `json:"name,omitempty" gorm:"index"` 43 | GenerateName string `json:"generateName,omitempty"` 44 | Namespace string `json:"namespace,omitempty"` 45 | SelfLink string `json:"selfLink,omitempty"` 46 | UID string `json:"uid"` 47 | ResourceVersion string `json:"resourceVersion,omitempty"` 48 | Generation int64 `json:"generation,omitempty"` 49 | CreationTimestamp string `json:"creationTimestamp,omitempty"` 50 | DeletionTimestamp string `json:"deletionTimestamp,omitempty"` 51 | DeletionGracePeriodSeconds *int64 `json:"deletionGracePeriodSeconds,omitempty"` 52 | Labels []*KubernetesKeyValue `json:"labels,omitempty" gorm:"foreignkey:ID;references:id;constraint:OnUpdate:CASCADE,OnDelete:SET NULL;"` 53 | Annotations []*KubernetesKeyValue `json:"annotations,omitempty" gorm:"foreignkey:ID;references:id;constraint:OnUpdate:CASCADE,OnDelete:SET NULL;"` 54 | OwnerReferences string `json:"ownerReferences,omitempty" gorm:"-"` 55 | Finalizers string `json:"finalizers,omitempty" gorm:"-"` 56 | ClusterName string `json:"clusterName,omitempty"` 57 | ManagedFields string `json:"managedFields,omitempty" gorm:"-"` 58 | ClusterID string `json:"cluster_id"` 59 | } 60 | 61 | type KubernetesResourceSpec struct { 62 | ID string `json:"id" gorm:"primarykey"` 63 | Attribute string `json:"attribute,omitempty"` 64 | } 65 | 66 | type KubernetesResourceStatus struct { 67 | ID string `json:"id" gorm:"primarykey"` 68 | Attribute string `json:"attribute,omitempty"` 69 | } 70 | 71 | func (obj *KubernetesResource) BeforeCreate(tx *gorm.DB) (err error) { 72 | SetID(obj) 73 | return nil 74 | } 75 | 76 | func (obj *KubernetesResource) BeforeSave(tx *gorm.DB) (err error) { 77 | SetID(obj) 78 | return nil 79 | } 80 | 81 | func (obj *KubernetesResource) BeforeDelete(tx *gorm.DB) (err error) { 82 | SetID(obj) 83 | return nil 84 | } 85 | -------------------------------------------------------------------------------- /pkg/model/model_converter.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "fmt" 7 | 8 | "github.com/buger/jsonparser" 9 | "github.com/google/uuid" 10 | "github.com/meshery/meshkit/broker" 11 | "github.com/meshery/meshkit/orchestration" 12 | iutils "github.com/meshery/meshsync/pkg/utils" 13 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 14 | ) 15 | 16 | // TODO fix cyclop error 17 | // Error: pkg/model/model_converter.go:16:1: calculated cyclomatic complexity for function ParseList is 13, max is 10 (cyclop) 18 | // 19 | //nolint:cyclop 20 | func ParseList(object unstructured.Unstructured, eventType broker.EventType) KubernetesResource { 21 | data, _ := object.MarshalJSON() 22 | result := KubernetesResource{} 23 | _ = json.Unmarshal(data, &result) 24 | 25 | processorInstance := GetProcessorInstance(result.Kind) 26 | // ObjectMeta internal models 27 | labels := make([]*KubernetesKeyValue, 0) 28 | _ = jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error { 29 | labels = append(labels, &KubernetesKeyValue{ 30 | Kind: KindLabel, 31 | Key: string(key), 32 | Value: string(value), 33 | }) 34 | 35 | if string(key) == orchestration.ResourceSourceDesignIdLabelKey { 36 | id, _ := uuid.FromBytes(value) 37 | result.PatternResource = &id 38 | } 39 | 40 | return nil 41 | }, "metadata", "labels") 42 | result.KubernetesResourceMeta.Labels = labels 43 | 44 | annotations := make([]*KubernetesKeyValue, 0) 45 | _ = jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error { 46 | annotations = append(annotations, &KubernetesKeyValue{ 47 | Kind: KindAnnotation, 48 | Key: string(key), 49 | Value: string(value), 50 | }) 51 | return nil 52 | }, "metadata", "annotations") 53 | result.KubernetesResourceMeta.Annotations = annotations 54 | 55 | if finalizers, _, _, err := jsonparser.Get(data, "metadata", "finalizers"); err == nil { 56 | result.KubernetesResourceMeta.Finalizers = string(finalizers) 57 | } 58 | 59 | if managedFields, _, _, err := jsonparser.Get(data, "metadata", "managedFields"); err == nil { 60 | result.KubernetesResourceMeta.ManagedFields = string(managedFields) 61 | } 62 | 63 | if ownerReferences, _, _, err := jsonparser.Get(data, "metadata", "ownerReferences"); err == nil { 64 | result.KubernetesResourceMeta.OwnerReferences = string(ownerReferences) 65 | } 66 | 67 | if spec, _, _, err := jsonparser.Get(data, "spec"); err == nil { 68 | result.Spec.Attribute = string(spec) 69 | } 70 | 71 | if status, _, _, err := jsonparser.Get(data, "status"); err == nil { 72 | result.Status.Attribute = string(status) 73 | } 74 | 75 | if immutable, _, _, err := jsonparser.Get(data, "immutable"); err == nil { 76 | result.Immutable = string(immutable) 77 | } 78 | 79 | if objData, _, _, err := jsonparser.Get(data, "data"); err == nil { 80 | result.Data = string(objData) 81 | } 82 | 83 | if binaryData, _, _, err := jsonparser.Get(data, "binaryData"); err == nil { 84 | result.BinaryData = string(binaryData) 85 | } 86 | 87 | if stringData, _, _, err := jsonparser.Get(data, "stringData"); err == nil { 88 | result.StringData = string(stringData) 89 | } 90 | 91 | if objType, _, _, err := jsonparser.Get(data, "type"); err == nil { 92 | result.Type = string(objType) 93 | } 94 | 95 | result.ClusterID = iutils.GetClusterID() 96 | if processorInstance != nil { 97 | _ = processorInstance.Process(data, &result, eventType) 98 | } 99 | 100 | return result 101 | } 102 | 103 | func IsObject(obj KubernetesResource) bool { 104 | return obj.KubernetesResourceMeta != nil 105 | } 106 | 107 | func SetID(obj *KubernetesResource) { 108 | if obj != nil && IsObject(*obj) { 109 | id := base64.StdEncoding.EncodeToString([]byte( 110 | fmt.Sprintf("%s.%s.%s.%s.%s", obj.ClusterID, obj.Kind, obj.APIVersion, obj.KubernetesResourceMeta.Namespace, obj.KubernetesResourceMeta.Name), 111 | )) 112 | obj.ID = id 113 | obj.KubernetesResourceMeta.ID = id 114 | 115 | if len(obj.KubernetesResourceMeta.Labels) > 0 { 116 | for _, label := range obj.KubernetesResourceMeta.Labels { 117 | label.ID = id 118 | label.UniqueID = uuid.New().String() 119 | } 120 | } 121 | 122 | if len(obj.KubernetesResourceMeta.Annotations) > 0 { 123 | for _, annotation := range obj.KubernetesResourceMeta.Annotations { 124 | annotation.ID = id 125 | annotation.UniqueID = uuid.New().String() 126 | } 127 | } 128 | 129 | if obj.Spec != nil { 130 | obj.Spec.ID = id 131 | } 132 | 133 | if obj.Status != nil { 134 | obj.Status.ID = id 135 | } 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /pkg/model/preprocessor.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "github.com/meshery/meshkit/broker" 5 | ) 6 | 7 | type ProcessFunc interface { 8 | Process(obj []byte, k8sresource *KubernetesResource, evtype broker.EventType) error 9 | } 10 | 11 | func GetProcessorInstance(kind string) ProcessFunc { 12 | switch kind { 13 | case "Service": 14 | return &K8SService{} 15 | default: 16 | return nil 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /pkg/model/process.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/url" 8 | "strings" 9 | 10 | "github.com/meshery/meshkit/broker" 11 | "github.com/meshery/meshkit/utils/kubernetes" 12 | v1 "k8s.io/api/core/v1" 13 | ) 14 | 15 | type K8SService struct{} 16 | 17 | func (s *K8SService) Process(data []byte, k8sresource *KubernetesResource, evtype broker.EventType) error { 18 | if evtype == broker.Delete { 19 | return nil 20 | } 21 | 22 | k8sservice := &v1.Service{} 23 | 24 | err := json.Unmarshal(data, k8sservice) 25 | if err != nil { 26 | return err 27 | } 28 | 29 | urls := []string{} 30 | endpoint, err := kubernetes.GetEndpoint(context.Background(), &kubernetes.ServiceOptions{}, k8sservice) 31 | if err != nil { 32 | return err 33 | } 34 | 35 | if endpoint != nil { 36 | if endpoint.External != nil { 37 | url, err := s.validateURL(endpoint.External.Address, endpoint.External.Port) 38 | if err == nil { 39 | urls = append(urls, url) 40 | } 41 | } 42 | if endpoint.Internal != nil { 43 | url, err := s.validateURL(endpoint.Internal.Address, endpoint.Internal.Port) 44 | if err == nil { 45 | urls = append(urls, url) 46 | } 47 | } 48 | } 49 | 50 | if k8sresource.ComponentMetadata == nil { 51 | k8sresource.ComponentMetadata = make(map[string]interface{}) 52 | } 53 | k8sresource.ComponentMetadata = map[string]interface{}{ 54 | "capabilities": map[string]interface{}{ 55 | // indicates that this svc can be upgraded to "Meshery Connection". 56 | "connection": true, 57 | "urls": urls, 58 | }, 59 | } 60 | 61 | return nil 62 | } 63 | 64 | func (s *K8SService) validateURL(address string, port int32) (serviceurl string, err error) { 65 | protocol := "http" 66 | if port == 443 { 67 | protocol = "https" 68 | } 69 | 70 | // For some Cluster IP type svc the address is set as None 71 | // Hence to prevent adding these IPs as URLs, below check is added. 72 | if strings.Contains(strings.ToLower(address), "none") { 73 | return serviceurl, kubernetes.ErrEndpointNotFound 74 | } 75 | serviceurl = fmt.Sprintf("%s://%s:%d", protocol, address, port) 76 | _, err = url.ParseRequestURI(serviceurl) 77 | return serviceurl, err 78 | } 79 | -------------------------------------------------------------------------------- /pkg/utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/meshery/meshkit/utils/kubernetes" 7 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | ) 9 | 10 | // clusterID is a unique identifier for the cluster 11 | // in which MeshSync is running 12 | var clusterID *string = nil 13 | 14 | // GetClusterID returns a unique identifier for the cluster in which 15 | // meshsync is running 16 | // 17 | // Notes: 18 | // 1. If MeshSync is running out of cluster then the function will return 19 | // an empty string 20 | // 2. Function caches the cluster ID whenever it is invoked for the first time 21 | // assuming that the cluster ID cannot and will not change throughout MeshSync's 22 | // lifecycle 23 | func GetClusterID() string { 24 | if clusterID != nil { 25 | return *clusterID 26 | } 27 | 28 | client, err := kubernetes.New(nil) 29 | if err != nil { 30 | return "" 31 | } 32 | 33 | ksns, err := client.KubeClient.CoreV1().Namespaces().Get(context.TODO(), "kube-system", v1.GetOptions{}) 34 | if err != nil { 35 | return "" 36 | } 37 | 38 | uid := string(ksns.ObjectMeta.GetUID()) 39 | clusterID = &uid 40 | 41 | return *clusterID 42 | } 43 | -------------------------------------------------------------------------------- /scripts/go-mod-tidy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/zsh 2 | 3 | go mod tidy 4 | if ! git diff --exit-code go.mod go.sum; then 5 | echo -e "\nPlease commit the changes made by 'go mod tidy'" 6 | exit 1 7 | fi 8 | -------------------------------------------------------------------------------- /scripts/go-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/zsh 2 | 3 | go test -failfast -race -v ./... 4 | echo $? 5 | --------------------------------------------------------------------------------