├── static ├── .nojekyll ├── img │ ├── arrange.png │ ├── dispatch.png │ ├── favicon.ico │ ├── network.png │ ├── title-bg.png │ └── logo.svg └── script │ ├── script.js │ └── ad.js ├── docs ├── proposals │ ├── img │ │ ├── Leaf_Nodes.png │ │ ├── cluster-deploy.jpg │ │ ├── mutaing_api_latency.jpg │ │ ├── readonly_api_latency.jpg │ │ ├── readonly_api_latency5.jpg │ │ └── readonly_api_latency30.jpg │ ├── leaf-node-gen.md │ ├── k8s-in-k8s.md │ ├── distribution-policy.md │ └── performance-testing.md ├── tutorials │ ├── img │ │ ├── Kubenest-CR.png │ │ ├── netdr-arch.png │ │ ├── EXEC_Log_Arch.png │ │ ├── IPsec_Tunnel.jpeg │ │ ├── MC_Scheduler.png │ │ ├── MCN_Floater_Flow.png │ │ ├── Kubenest-Host-K8s.png │ │ ├── K8s_Admission_Webhook.png │ │ ├── Kubenest-Architecture.png │ │ ├── MCN_GW_Architecture.png │ │ ├── MCN_P2P_Architecture.png │ │ ├── PV_PVC_Dynamic_Storage.png │ │ ├── Kosmos_Admission_Webhook.png │ │ ├── Kubenest-Solution-Overview.png │ │ ├── Kubenest-NodeAgent-Flowchart.png │ │ ├── kosmos-scheduler-workload-policy-1.png │ │ ├── kosmos-scheduler-workload-policy-flow-cn.png │ │ ├── kosmos-scheduler-workload-policy-flow-en.png │ │ └── kosmos-scheduler-workload-policy-scheduling-framework.png │ ├── ccn-ipsec-tunnel.md │ ├── multi-cluster-svc-discovery.md │ ├── application-migration.md │ ├── kubenest.md │ └── node-not-ready.md └── getting-started │ ├── img │ ├── manager-list.png │ ├── host_vx_check.png │ ├── member_vx_check.png │ ├── host_ippools_check.png │ ├── member_ippools_check.png │ ├── member_service_check.png │ ├── host_nodeconfigs_check.png │ ├── ClusterLink_Architecture.png │ ├── ClusterTree_Architecture.png │ └── host_clusternodes_check.png │ └── introduction.md ├── babel.config.js ├── OWNERS ├── .github ├── dependabot.yml ├── ISSUE_TEMPLATE │ ├── enhancement.md │ └── request-doc.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── typos.yml │ ├── test-deploy.yml │ └── deploy.yml ├── src ├── pages │ ├── markdown-page.md │ ├── index.module.css │ └── index.js ├── plugins │ └── docusaurus-tailwindcss-loader │ │ └── index.js ├── components │ ├── HomepageDescription.module.css │ ├── HomepageDescription.js │ ├── HomepageFeatures.module.css │ └── HomepageFeatures.js └── css │ └── custom.css ├── i18n └── zh-Hans │ ├── docusaurus-plugin-content-docs │ ├── current │ │ ├── proposals │ │ │ ├── img │ │ │ │ ├── Leaf_Nodes.png │ │ │ │ ├── cluster-deploy.jpg │ │ │ │ ├── mutaing_api_latency.jpg │ │ │ │ ├── readonly_api_latency.jpg │ │ │ │ ├── readonly_api_latency5.jpg │ │ │ │ └── readonly_api_latency30.jpg │ │ │ ├── leaf-node-gen.md │ │ │ ├── k8s-in-k8s.md │ │ │ └── distribution-policy.md │ │ ├── tutorials │ │ │ ├── img │ │ │ │ ├── Kubenest-CR.png │ │ │ │ ├── netdr-arch.png │ │ │ │ ├── EXEC_Log_Arch.png │ │ │ │ ├── IPsec_Tunnel.jpeg │ │ │ │ ├── MC_Scheduler.png │ │ │ │ ├── MCN_Floater_Flow.png │ │ │ │ ├── Kubenest-Host-K8s.png │ │ │ │ ├── MCN_GW_Architecture.png │ │ │ │ ├── K8s_Admission_Webhook.png │ │ │ │ ├── Kubenest-Architecture.png │ │ │ │ ├── MCN_P2P_Architecture.png │ │ │ │ ├── Kosmos_Admission_Webhook.png │ │ │ │ ├── PV_PVC_Dynamic_Storage.png │ │ │ │ ├── Kubenest-Solution-Overview.png │ │ │ │ └── Kubenest-NodeAgent-Flowchart.png │ │ │ ├── ccn-ipsec-tunnel.md │ │ │ ├── netdoctor.md │ │ │ ├── multi-cluster-svc-discovery.md │ │ │ ├── application-migration.md │ │ │ ├── kubenest.md │ │ │ └── node-not-ready.md │ │ └── getting-started │ │ │ ├── img │ │ │ ├── host_vx_check.png │ │ │ ├── manager-list.png │ │ │ ├── member_vx_check.png │ │ │ ├── host_ippools_check.png │ │ │ ├── member_ippools_check.png │ │ │ ├── member_service_check.png │ │ │ ├── host_clusternodes_check.png │ │ │ ├── host_nodeconfigs_check.png │ │ │ ├── ClusterLink_Architecture.png │ │ │ └── ClusterTree_Architecture.png │ │ │ ├── introduction.md │ │ │ └── quickstart.md │ └── current.json │ ├── docusaurus-plugin-content-blog │ └── options.json │ └── docusaurus-theme-classic │ ├── navbar.json │ └── footer.json ├── tailwind.config.js ├── .gitignore ├── sidebars.js ├── typos.toml ├── package.json ├── README.md ├── README_ZH.md └── docusaurus.config.js /static/.nojekyll: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /static/img/arrange.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/static/img/arrange.png -------------------------------------------------------------------------------- /static/img/dispatch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/static/img/dispatch.png -------------------------------------------------------------------------------- /static/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/static/img/favicon.ico -------------------------------------------------------------------------------- /static/img/network.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/static/img/network.png -------------------------------------------------------------------------------- /static/img/title-bg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/static/img/title-bg.png -------------------------------------------------------------------------------- /docs/proposals/img/Leaf_Nodes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/proposals/img/Leaf_Nodes.png -------------------------------------------------------------------------------- /docs/tutorials/img/Kubenest-CR.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/tutorials/img/Kubenest-CR.png -------------------------------------------------------------------------------- /docs/tutorials/img/netdr-arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/tutorials/img/netdr-arch.png -------------------------------------------------------------------------------- /docs/tutorials/img/EXEC_Log_Arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/tutorials/img/EXEC_Log_Arch.png -------------------------------------------------------------------------------- /docs/tutorials/img/IPsec_Tunnel.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/tutorials/img/IPsec_Tunnel.jpeg -------------------------------------------------------------------------------- /docs/tutorials/img/MC_Scheduler.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/tutorials/img/MC_Scheduler.png -------------------------------------------------------------------------------- /babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [require.resolve('@docusaurus/core/lib/babel/preset')], 3 | plugins: [ 4 | ] 5 | }; -------------------------------------------------------------------------------- /docs/proposals/img/cluster-deploy.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/proposals/img/cluster-deploy.jpg -------------------------------------------------------------------------------- /docs/tutorials/img/MCN_Floater_Flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/tutorials/img/MCN_Floater_Flow.png -------------------------------------------------------------------------------- /docs/getting-started/img/manager-list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/getting-started/img/manager-list.png -------------------------------------------------------------------------------- /docs/tutorials/img/Kubenest-Host-K8s.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/tutorials/img/Kubenest-Host-K8s.png -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - ONE7live 3 | - xiaomihuylx 4 | - duanmengkk 5 | reviewers: 6 | - ONE7live 7 | - xiaomihuylx 8 | - duanmengkk 9 | -------------------------------------------------------------------------------- /docs/getting-started/img/host_vx_check.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/getting-started/img/host_vx_check.png -------------------------------------------------------------------------------- /docs/getting-started/img/member_vx_check.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/getting-started/img/member_vx_check.png -------------------------------------------------------------------------------- /docs/proposals/img/mutaing_api_latency.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/proposals/img/mutaing_api_latency.jpg -------------------------------------------------------------------------------- /docs/proposals/img/readonly_api_latency.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/proposals/img/readonly_api_latency.jpg -------------------------------------------------------------------------------- /docs/proposals/img/readonly_api_latency5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/proposals/img/readonly_api_latency5.jpg -------------------------------------------------------------------------------- /docs/tutorials/img/K8s_Admission_Webhook.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/tutorials/img/K8s_Admission_Webhook.png -------------------------------------------------------------------------------- /docs/tutorials/img/Kubenest-Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/tutorials/img/Kubenest-Architecture.png -------------------------------------------------------------------------------- /docs/tutorials/img/MCN_GW_Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/tutorials/img/MCN_GW_Architecture.png -------------------------------------------------------------------------------- /docs/tutorials/img/MCN_P2P_Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/tutorials/img/MCN_P2P_Architecture.png -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: 'npm' 4 | directory: '/' 5 | schedule: 6 | interval: 'weekly' 7 | -------------------------------------------------------------------------------- /docs/proposals/img/readonly_api_latency30.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/proposals/img/readonly_api_latency30.jpg -------------------------------------------------------------------------------- /docs/tutorials/img/PV_PVC_Dynamic_Storage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/tutorials/img/PV_PVC_Dynamic_Storage.png -------------------------------------------------------------------------------- /docs/getting-started/img/host_ippools_check.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/getting-started/img/host_ippools_check.png -------------------------------------------------------------------------------- /docs/getting-started/img/member_ippools_check.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/getting-started/img/member_ippools_check.png -------------------------------------------------------------------------------- /docs/getting-started/img/member_service_check.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/getting-started/img/member_service_check.png -------------------------------------------------------------------------------- /docs/tutorials/img/Kosmos_Admission_Webhook.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/tutorials/img/Kosmos_Admission_Webhook.png -------------------------------------------------------------------------------- /docs/tutorials/img/Kubenest-Solution-Overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/tutorials/img/Kubenest-Solution-Overview.png -------------------------------------------------------------------------------- /docs/getting-started/img/host_nodeconfigs_check.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/getting-started/img/host_nodeconfigs_check.png -------------------------------------------------------------------------------- /docs/tutorials/img/Kubenest-NodeAgent-Flowchart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/tutorials/img/Kubenest-NodeAgent-Flowchart.png -------------------------------------------------------------------------------- /docs/getting-started/img/ClusterLink_Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/getting-started/img/ClusterLink_Architecture.png -------------------------------------------------------------------------------- /docs/getting-started/img/ClusterTree_Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/getting-started/img/ClusterTree_Architecture.png -------------------------------------------------------------------------------- /docs/getting-started/img/host_clusternodes_check.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/getting-started/img/host_clusternodes_check.png -------------------------------------------------------------------------------- /src/pages/markdown-page.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Markdown page example 3 | --- 4 | 5 | # Markdown page example 6 | 7 | You don't need React to write simple standalone pages. 8 | -------------------------------------------------------------------------------- /docs/tutorials/img/kosmos-scheduler-workload-policy-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/tutorials/img/kosmos-scheduler-workload-policy-1.png -------------------------------------------------------------------------------- /docs/tutorials/img/kosmos-scheduler-workload-policy-flow-cn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/tutorials/img/kosmos-scheduler-workload-policy-flow-cn.png -------------------------------------------------------------------------------- /docs/tutorials/img/kosmos-scheduler-workload-policy-flow-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/tutorials/img/kosmos-scheduler-workload-policy-flow-en.png -------------------------------------------------------------------------------- /docs/tutorials/img/kosmos-scheduler-workload-policy-scheduling-framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/docs/tutorials/img/kosmos-scheduler-workload-policy-scheduling-framework.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/proposals/img/Leaf_Nodes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/proposals/img/Leaf_Nodes.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/Kubenest-CR.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/Kubenest-CR.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/netdr-arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/netdr-arch.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/proposals/img/cluster-deploy.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/proposals/img/cluster-deploy.jpg -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/EXEC_Log_Arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/EXEC_Log_Arch.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/IPsec_Tunnel.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/IPsec_Tunnel.jpeg -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/MC_Scheduler.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/MC_Scheduler.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/MCN_Floater_Flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/MCN_Floater_Flow.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/host_vx_check.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/host_vx_check.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/manager-list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/manager-list.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/proposals/img/mutaing_api_latency.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/proposals/img/mutaing_api_latency.jpg -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/Kubenest-Host-K8s.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/Kubenest-Host-K8s.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/MCN_GW_Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/MCN_GW_Architecture.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/member_vx_check.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/member_vx_check.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/proposals/img/readonly_api_latency.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/proposals/img/readonly_api_latency.jpg -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/proposals/img/readonly_api_latency5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/proposals/img/readonly_api_latency5.jpg -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/K8s_Admission_Webhook.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/K8s_Admission_Webhook.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/Kubenest-Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/Kubenest-Architecture.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/MCN_P2P_Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/MCN_P2P_Architecture.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/host_ippools_check.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/host_ippools_check.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/proposals/img/readonly_api_latency30.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/proposals/img/readonly_api_latency30.jpg -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/Kosmos_Admission_Webhook.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/Kosmos_Admission_Webhook.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/PV_PVC_Dynamic_Storage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/PV_PVC_Dynamic_Storage.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/member_ippools_check.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/member_ippools_check.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/member_service_check.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/member_service_check.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/Kubenest-Solution-Overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/Kubenest-Solution-Overview.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/host_clusternodes_check.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/host_clusternodes_check.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/host_nodeconfigs_check.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/host_nodeconfigs_check.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/Kubenest-NodeAgent-Flowchart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/img/Kubenest-NodeAgent-Flowchart.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/ClusterLink_Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/ClusterLink_Architecture.png -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/ClusterTree_Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kosmos-io/kosmos-website/HEAD/i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/img/ClusterTree_Architecture.png -------------------------------------------------------------------------------- /tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | module.exports = { 3 | content: ["./src/**/*.{js,jsx,ts,tsx,md,mdx}", './docs/**/*.{js,jsx,ts,tsx,md,mdx}'], 4 | corePlugins: { 5 | preflight: false 6 | }, 7 | theme: { 8 | extend: {}, 9 | }, 10 | plugins: [], 11 | } -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/enhancement.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Enhancement Request 3 | about: Suggest an enhancement to the project 4 | labels: kind/feature 5 | 6 | --- 7 | 8 | 9 | **What would you like to be added**: 10 | 11 | **Why is this needed**: 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | node_modules 3 | 4 | # Production 5 | build 6 | 7 | # Generated files 8 | .docusaurus 9 | .cache-loader 10 | 11 | # Misc 12 | .DS_Store 13 | .env.local 14 | .env.development.local 15 | .env.test.local 16 | .env.production.local 17 | 18 | npm-debug.log* 19 | yarn-debug.log* 20 | yarn-error.log* 21 | 22 | .idea/ 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | **What type of PR is this?** 2 | 3 | 15 | 16 | **What this PR does / why we need it**: 17 | 18 | **Which issue(s) this PR fixes**: 19 | Fixes # 20 | 21 | **Special notes for your reviewer**: 22 | 23 | 24 | -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-blog/options.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": { 3 | "message": "Blog", 4 | "description": "The title for the blog used in SEO" 5 | }, 6 | "description": { 7 | "message": "Blog", 8 | "description": "The description for the blog used in SEO" 9 | }, 10 | "sidebar.title": { 11 | "message": "Recent posts", 12 | "description": "The label for the left sidebar" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /.github/workflows/typos.yml: -------------------------------------------------------------------------------- 1 | name: Typos Check 2 | on: 3 | push: 4 | pull_request: 5 | jobs: 6 | typos-check: 7 | name: Spell Check with Typos 8 | runs-on: ubuntu-22.04 9 | steps: 10 | - name: Checkout Actions Repository 11 | uses: actions/checkout@v4 12 | - name: Check spelling with custom config file 13 | uses: crate-ci/typos@v1.17.2 14 | with: 15 | config: ./typos.toml 16 | -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-theme-classic/navbar.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": { 3 | "message": "Kosmos", 4 | "description": "The title in the navbar" 5 | }, 6 | "logo.alt": { 7 | "message": "Kosmos Logo", 8 | "description": "The alt text of navbar logo" 9 | }, 10 | "item.label.Documentation": { 11 | "message": "Documentation", 12 | "description": "Navbar item with label Documentation" 13 | }, 14 | "item.label.Examples": { 15 | "message": "Examples", 16 | "description": "Navbar item with label Examples" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /docs/proposals/leaf-node-gen.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: leaf-node-generate-rules 3 | title: 'Leaf Node Generate Rules' 4 | --- 5 | 6 | # Leaf Node Generate Rules 7 | 8 | ## Summary 9 | Provide a member cluster to pretend one or more Leaf Nodes in clusterTree by some rules like Node labelSelector 10 | 11 | ## Motivation & User Stories 12 | 1、Some products can provide idle Nodes to join Kosmos 13 | 2、Some products want an easy way to make a second level pod schedule in their member clusters. 14 | 15 | ## Design Details 16 | 17 | ### Architecture 18 | ![leaf_node_rules](img/Leaf_Nodes.png) -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/proposals/leaf-node-gen.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: leaf-node-generate-rules 3 | title: 'Leaf Node Generate Rules' 4 | --- 5 | 6 | # Leaf Node Generate Rules 7 | 8 | ## Summary 9 | Provide a member cluster to pretend one or more Leaf Nodes in clusterTree by some rules like Node labelSelector 10 | 11 | ## Motivation & User Stories 12 | 1、Some products can provide idle Nodes to join Kosmos 13 | 2、Some products want an easy way to make a second level pod schedule in their member clusters. 14 | 15 | ## Design Details 16 | 17 | ### Architecture 18 | ![leaf_node_rules](img/Leaf_Nodes.png) -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/request-doc.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Request Documentation 3 | about: Request for new documents or supplementation or updating of existing documents. 4 | labels: help wanted 5 | --- 6 | 7 | **What documentation would you like to be added**: 8 | 9 | **Why do you think this document is needed**: 10 | 11 | **Where do you think the document should be placed**: 12 | 18 | -------------------------------------------------------------------------------- /sidebars.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creating a sidebar enables you to: 3 | - create an ordered group of docs 4 | - render a sidebar for each doc of that group 5 | - provide next/previous navigation 6 | 7 | The sidebars can be generated from the filesystem, or explicitly defined here. 8 | 9 | Create as many sidebars as you want. 10 | */ 11 | 12 | // @ts-check 13 | 14 | /** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */ 15 | const sidebars = { 16 | // By default, Docusaurus generates a sidebar from the docs folder structure 17 | tutorialSidebar: [{type: 'autogenerated', dirName: '.'}], 18 | }; 19 | 20 | export default sidebars; 21 | 22 | -------------------------------------------------------------------------------- /typos.toml: -------------------------------------------------------------------------------- 1 | # This is the custom config file used by `typos` command and `crate-ci/typos@master` action to do source code spelling correction. 2 | # Sometimes, what looks like a typo is intentional, like with people's names, acronyms, or localized content. 3 | # To mark a word or an identifier (grouping of words) as valid, add it to this file by declaring itself as the valid spelling. 4 | # See https://github.com/crate-ci/typos/blob/master/docs/reference.md for more info to configure typos. 5 | [files] 6 | extend-exclude = [ 7 | "*.svg", 8 | "*.drawio", 9 | ] 10 | [default.extend-words] 11 | BA = "BA" 12 | Ded = "Ded" 13 | EDE = "EDE" 14 | intoto = "intoto" 15 | -------------------------------------------------------------------------------- /.github/workflows/test-deploy.yml: -------------------------------------------------------------------------------- 1 | name: 测试部署 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | # 如果你想要进一步定义触发、路径以及其他内容，请检阅 Github Actions 文档 8 | # https://docs.github.com/zh/actions/using-workflows/workflow-syntax-for-github-actions#on 9 | 10 | jobs: 11 | test-deploy: 12 | name: 测试部署 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v4 16 | with: 17 | fetch-depth: 0 18 | - uses: actions/setup-node@v4 19 | with: 20 | node-version: 18 21 | cache: yarn 22 | 23 | - name: 安装依赖 24 | run: yarn install --frozen-lockfile 25 | - name: 测试构建网站 26 | run: yarn build -------------------------------------------------------------------------------- /src/plugins/docusaurus-tailwindcss-loader/index.js: -------------------------------------------------------------------------------- 1 | module.exports = function (context, options) { 2 | return { 3 | name: 'postcss-tailwindcss-loader', 4 | configurePostCss(postcssOptions) { 5 | // Appends new PostCSS plugin. 6 | postcssOptions.plugins.push( 7 | require('postcss-import'), 8 | require('tailwindcss'), 9 | require('postcss-preset-env')({ 10 | autoprefixer: { 11 | flexbox: 'no-2009', 12 | }, 13 | stage: 4, 14 | }), 15 | ) 16 | return postcssOptions 17 | }, 18 | } 19 | } -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current.json: -------------------------------------------------------------------------------- 1 | { 2 | "version.label": { 3 | "message": "Next", 4 | "description": "The label for version current" 5 | }, 6 | "sidebar.tutorialSidebar.category.getting-started": { 7 | "message": "getting-started", 8 | "description": "The label for category getting-started in sidebar tutorialSidebar" 9 | }, 10 | "sidebar.tutorialSidebar.category.proposals": { 11 | "message": "proposals", 12 | "description": "The label for category proposals in sidebar tutorialSidebar" 13 | }, 14 | "sidebar.tutorialSidebar.category.tutorials": { 15 | "message": "tutorials", 16 | "description": "The label for category tutorials in sidebar tutorialSidebar" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/components/HomepageDescription.module.css: -------------------------------------------------------------------------------- 1 | .featuresText { 2 | display: flex; 3 | justify-content: space-between; 4 | height: 535px; 5 | margin: 0px 80px; 6 | max-width: 1800px; 7 | width: 100%; 8 | } 9 | 10 | .rightInfo { 11 | width: 50%; 12 | display: flex; 13 | flex-direction: column; 14 | justify-content: center; 15 | margin-right: 30px; 16 | } 17 | 18 | .leftInfo { 19 | width: 50%; 20 | margin-right: 150px; 21 | } 22 | 23 | .containerView { 24 | justify-content: center; 25 | display: flex; 26 | } 27 | 28 | .featureTitle { 29 | font-family: PingFangSC-Semibold; 30 | font-size: 40px; 31 | color: #333; 32 | font-weight: 600; 33 | margin-bottom: 20px; 34 | } 35 | 36 | .featuresDes { 37 | max-width: 415px; 38 | } 39 | -------------------------------------------------------------------------------- /src/components/HomepageDescription.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import styles from './HomepageDescription.module.css'; 3 | 4 | 5 | function Feature() { 6 | return ( 7 |
8 |
9 | 10 |
11 |
12 |
What is Kosmos？
13 |
14 |

Kosmos is an open-source, integrated distributed cloud-native solution. The name "kosmos" is formed by combining the "k" representing Kubernetes with "cosmos" from Greek, symbolizing the infinite scalability of Kubernetes.

15 |

Kosmos mainly consists of three major modules: ClusterLink, ClusterTree, and Scheduler. In addition, Kosmos is also equipped with a tool named kosmosctl, which can quickly deploy Kosmos components, add clusters, and test network connection.

16 |
17 |
18 |
19 | ) 20 | } 21 | 22 | export default function HomepageFeatures() { 23 | return ( 24 |
25 |
26 |
27 | 28 |
29 |
30 |
31 | ); 32 | } 33 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: 部署到 GitHub Pages 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | # 如果你想要进一步定义触发、路径以及其他内容，请检阅 Github Actions 文档 8 | # https://docs.github.com/zh/actions/using-workflows/workflow-syntax-for-github-actions#on 9 | 10 | jobs: 11 | build: 12 | name: 构建 Docusaurus 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v4 16 | with: 17 | fetch-depth: 0 18 | - uses: actions/setup-node@v4 19 | with: 20 | node-version: 18 21 | cache: yarn 22 | 23 | - name: 安装依赖 24 | run: yarn install --frozen-lockfile 25 | - name: 构建网站 26 | run: yarn build 27 | 28 | - name: 上传构建制品 29 | uses: actions/upload-pages-artifact@v3 30 | with: 31 | path: build 32 | 33 | deploy: 34 | name: 部署到 GitHub Pages 35 | needs: build 36 | 37 | # 给予 GITHUB_TOKEN 进行 Pages 部署所必须的权限 38 | permissions: 39 | pages: write # 以部署到 Pages 40 | id-token: write # 以验证部署来自恰当的源 41 | 42 | # 部署到 Github Pages 环境 43 | environment: 44 | name: github-pages 45 | url: ${{ steps.deployment.outputs.page_url }} 46 | 47 | runs-on: ubuntu-latest 48 | steps: 49 | - name: 部署到 GitHub Pages 50 | id: deployment 51 | uses: actions/deploy-pages@v4 -------------------------------------------------------------------------------- /static/script/script.js: -------------------------------------------------------------------------------- 1 | // 兼容旧版文档相关链接进行跳转 2 | let hash = window.location.hash; 3 | 4 | 5 | let hashMap = { 6 | '#/question?id=reset-pwd': '/config/config-debug', 7 | '#/advanced?id=onedrive-cf': 'advanced/cf-worker', 8 | '#/example': '/category/存储源示例配置', 9 | '#/example/': '/category/存储源示例配置', 10 | '#/pro': '/install/pro-linux', 11 | } 12 | 13 | if (hashMap[hash]) { 14 | window.location.href = hashMap[hash]; 15 | } 16 | 17 | let path = window.location.href.replace(window.location.origin, ""); 18 | 19 | let pathMap = { 20 | '/zfile': '/', 21 | '/advanced#only-office': '/advanced/only-office', 22 | '/advanced/#only-office': '/advanced/only-office', 23 | '/advanced#google-drive-api': '/advanced/google-drive-api', 24 | '/advanced/#google-drive-api': '/advanced/google-drive-api', 25 | '/question#reset-pwd': '/config/config-debug', 26 | '/question/#reset-pwd': '/config/config-debug', 27 | '/example': '/category/存储源示例配置', 28 | '/example/': '/category/存储源示例配置' 29 | } 30 | 31 | if (pathMap[path]) { 32 | window.location.href = pathMap[path]; 33 | } 34 | 35 | 36 | // 百度统计 37 | var _hmt = _hmt || []; 38 | (function() { 39 | var hm = document.createElement("script"); 40 | hm.src = "https://hm.baidu.com/hm.js?e73e6c1d4f6b625ef90d78262c28cbb1"; 41 | var s = document.getElementsByTagName("script")[0]; 42 | s.parentNode.insertBefore(hm, s); 43 | })(); 44 | -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-theme-classic/footer.json: -------------------------------------------------------------------------------- 1 | { 2 | "link.title.Documentation": { 3 | "message": "Documentation", 4 | "description": "The title of the footer links column with title=Documentation in the footer" 5 | }, 6 | "link.title.Community": { 7 | "message": "Community", 8 | "description": "The title of the footer links column with title=Community in the footer" 9 | }, 10 | "link.title.More": { 11 | "message": "More", 12 | "description": "The title of the footer links column with title=More in the footer" 13 | }, 14 | "link.item.label.Getting Started": { 15 | "message": "Getting Started", 16 | "description": "The label of footer link with label=Getting Started linking to getting-started/introduction" 17 | }, 18 | "link.item.label.Tutorials": { 19 | "message": "Tutorials", 20 | "description": "The label of footer link with label=Tutorials linking to tutorials/mcs-discovery" 21 | }, 22 | "link.item.label.Proposals": { 23 | "message": "Proposals", 24 | "description": "The label of footer link with label=Proposals linking to proposals/k8s-in-k8s" 25 | }, 26 | "link.item.label.Community Address Name": { 27 | "message": "Community Address Name", 28 | "description": "The label of footer link with label=Community Address Name linking to https://github.com/kosmos-io/kosmos" 29 | }, 30 | "link.item.label.GitHub": { 31 | "message": "GitHub", 32 | "description": "The label of footer link with label=GitHub linking to https://github.com/kosmos-io/kosmos" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "zfile-docs", 3 | "version": "0.0.0", 4 | "private": true, 5 | "scripts": { 6 | "docusaurus": "docusaurus", 7 | "start": "docusaurus start --port=3111", 8 | "build": "docusaurus build", 9 | "swizzle": "docusaurus swizzle", 10 | "deploy": "docusaurus deploy", 11 | "clear": "docusaurus clear", 12 | "serve": "docusaurus serve", 13 | "write-translations": "docusaurus write-translations", 14 | "write-heading-ids": "docusaurus write-heading-ids" 15 | }, 16 | "dependencies": { 17 | "@docusaurus/core": "^3.1.1", 18 | "@docusaurus/preset-classic": "^3.5.2", 19 | "@docusaurus/theme-live-codeblock": "^3.5.2", 20 | "@easyops-cn/docusaurus-search-local": "^0.40.1", 21 | "@mdx-js/react": "^3.0.1", 22 | "autoprefixer": "^10.4.17", 23 | "clsx": "^2.1.1", 24 | "docusaurus-plugin-image-zoom": "^2.0.0", 25 | "docusaurus-tailwindcss-loader": "file:src/plugins/docusaurus-tailwindcss-loader", 26 | "postcss": "^8.4.34", 27 | "postcss-import": "^16.1.0", 28 | "postcss-preset-env": "^9.3.0", 29 | "prism-react-renderer": "^2.3.1", 30 | "react": "^18.2.0", 31 | "react-dom": "^18.3.1", 32 | "react-github-btn": "^1.4.0", 33 | "tailwindcss": "^3.4.10", 34 | "twin.macro": "^3.4.1" 35 | }, 36 | "devDependencies": { 37 | "@docusaurus/module-type-aliases": "^3.5.2" 38 | }, 39 | "browserslist": { 40 | "production": [ 41 | ">0.5%", 42 | "not dead", 43 | "not op_mini all" 44 | ], 45 | "development": [ 46 | "last 1 chrome version", 47 | "last 1 firefox version", 48 | "last 1 safari version" 49 | ] 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/pages/index.module.css: -------------------------------------------------------------------------------- 1 | .container { 2 | display: flex; 3 | flex: 1; 4 | align-items: center; 5 | height: 690px !important; 6 | padding: 0px !important; 7 | width: 100%; 8 | margin: 0 80px 0 80px; 9 | max-width: 1800px; 10 | background-image: url("../../static/img/top-bg.svg"); 11 | background-size: cover; 12 | background-position: center; 13 | } 14 | .heroBanner { 15 | background-color: #f8f9fc; 16 | text-align: center; 17 | position: relative; 18 | overflow: hidden; 19 | margin-bottom: 100px; 20 | } 21 | .button1 { 22 | background-color: #3b54e0; 23 | color: #fff !important; 24 | border-radius: 25px; 25 | margin-bottom: 65px; 26 | border: none; 27 | } 28 | .iconStar { 29 | border-radius: 0px !important; 30 | border-right: 1px solid #d1d5d7; 31 | } 32 | .starNum { 33 | display: inline-block; 34 | width: 55px; 35 | font-size: 14px; 36 | text-align: center; 37 | color: black; 38 | } 39 | .heroTitle { 40 | font-size: 50px; 41 | color: black; 42 | margin-bottom: 20px; 43 | } 44 | .heroText { 45 | color: #626262; 46 | font-size: 16px; 47 | margin-bottom: 40px; 48 | font-weight: bold; 49 | width: 475px; 50 | } 51 | .headerLeft { 52 | height: 613px; 53 | width: 50%; 54 | text-align: left; 55 | padding: 130px 70px 30px 0px; 56 | } 57 | .topImg { 58 | max-width: 100%; 59 | max-height: 530px; 60 | display: block; 61 | } 62 | .headerDes { 63 | color: #414141; 64 | font-size: 16px; 65 | } 66 | .headerDesInfo { 67 | margin-bottom: 10px; 68 | } 69 | .mainContainer { 70 | display: flex; 71 | justify-content: space-between; 72 | flex-direction: column; 73 | } 74 | .star { 75 | margin-bottom: 5px; 76 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | > English | [中文](README_ZH.md) 2 | > 3 | # Kosmos Docs & Website 4 | 5 | This repo contains the source code of [Kosmos website](https://kosmos-io.github.io/website/) and all of the docs for Kosmos. 6 | It's built by [Docusaurus](https://docusaurus.io/), a modern static website generator. 7 | 8 | - [Kosmos website](https://kosmos-io.github.io/kosmos-website/) 9 | - [Kosmos docs](https://kosmos-io.github.io/kosmos-website/docs/getting-started/introduction) 10 | 11 | Welcome to join us and you are more than appreciated to contribute! 12 | 13 | ## Add or Update Docs 14 | 15 | When you add or modify the docs, these two files(`docs/` and `i18n/zh-Hans/docusaurus-plugin-content-docs/current`) should be taken into consideration. 16 | 17 | ## Run with Node.js 18 | 19 | If you have the Node.js environment, you can run the website locally. 20 | - It is recommended to use version node v18.0.0+ and npm v8.6+ 21 | - You can download [Node.js](https://nodejs.org/download/release/v18.0.0) 22 | 23 | ```shell script 24 | # Clone the repo, or your own fork 25 | git clone https://github.com//website.git 26 | 27 | # build 28 | yarn 29 | 30 | # Start the site 31 | yarn start 32 | ``` 33 | 34 | Once the site is running locally, you can preview the site by visiting . 35 | 36 | ## How to Submit Documentation to this repo 37 | 38 | The default documentation language for the project is English. 39 | - English documentation submission path: `./docs` 40 | - Chinese documentation submission path: `./i18n/zh-Hans/docusaurus-plugin-content-docs/current` 41 | 42 | Note: The directory structures for both Chinese and English documentation must be completely identical. 43 | 44 | ## Send your pull request 45 | 46 | After all changes checked well, please [creating a pull request](https://help.github.com/en/articles/creating-a-pull-request) with [DCO](https://github.com/apps/dco). -------------------------------------------------------------------------------- /src/pages/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import clsx from 'clsx'; 3 | import Layout from '@theme/Layout'; 4 | import Link from '@docusaurus/Link'; 5 | import useDocusaurusContext from '@docusaurus/useDocusaurusContext'; 6 | import styles from './index.module.css'; 7 | import HomepageDescription from '../components/HomepageDescription'; 8 | import HomepageFeatures from '../components/HomepageFeatures'; 9 | import GitHubButton from 'react-github-btn'; 10 | 11 | function HomepageHeader() { 12 | const { siteConfig } = useDocusaurusContext(); 13 | return ( 14 |
15 |
16 |
17 |

{siteConfig.title}

18 |

{siteConfig.tagline}

19 |
20 | 23 | Get Started 24 | 25 |
26 |
27 | 33 | Star 34 | 35 |
36 |
37 |

If you like Kosmos, give it a star on GitHub!

38 |

Kosmos v0.5.0 is now available.

39 |
40 |
41 |
42 | 43 |
44 |
45 |
46 | ); 47 | } 48 | 49 | export default function Home() { 50 | return ( 51 | 52 | 53 |
54 | 55 | 56 |
57 | 58 | ) 59 | } 60 | -------------------------------------------------------------------------------- /README_ZH.md: -------------------------------------------------------------------------------- 1 | > [English](README.md) | 中文 2 | 3 | # Kosmos 文档与网站 4 | 5 | 这个仓库包含了[Kosmos 网站](https://kosmos-io.github.io/website/)的源代码以及所有的 Kosmos 文档。 6 | 它是由[Docusaurus](https://docusaurus.io/)构建的，一个现代的静态网站生成器。 7 | 8 | - [Kosmos 网站](https://kosmos-io.github.io/kosmos-website/) 9 | - [Kosmos 文档](https://kosmos-io.github.io/kosmos-website/docs/getting-started/introduction) 10 | 11 | 欢迎加入我们，非常感谢您的贡献！ 12 | 13 | ## 添加或更新文档 14 | 15 | 当您添加或修改文档时，应该考虑这两个文件夹([`docs/`](command:_github.copilot.openRelativePath?%5B%7B%22scheme%22%3A%22file%22%2C%22authority%22%3A%22%22%2C%22path%22%3A%22%2FUsers%2Fduanmeng%2FCode%2Fgithub2%2Fwebsite%2Fdocs%2F%22%2C%22query%22%3A%22%22%2C%22fragment%22%3A%22%22%7D%5D "/Users/duanmeng/Code/github2/website/docs/") 和 [`i18n/zh-Hans/docusaurus-plugin-content-docs/current`](command:_github.copilot.openRelativePath?%5B%7B%22scheme%22%3A%22file%22%2C%22authority%22%3A%22%22%2C%22path%22%3A%22%2FUsers%2Fduanmeng%2FCode%2Fgithub2%2Fwebsite%2Fi18n%2Fzh-Hans%2Fdocusaurus-plugin-content-docs%2Fcurrent%22%2C%22query%22%3A%22%22%2C%22fragment%22%3A%22%22%7D%5D "/Users/duanmeng/Code/github2/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current"))。 16 | 17 | ## 使用 Node.js 运行 18 | 19 | 如果您有 Node.js 环境，可以在本地运行该网站。 20 | - 推荐使用 Node.js v18.0.0+ 和 npm v8.6+ 21 | - 您可以下载 [Node.js](https://nodejs.org/download/release/v18.0.0) 22 | 23 | ```shell script 24 | # 克隆仓库，或克隆您自己的分叉 25 | git clone https://github.com//website.git 26 | 27 | # 构建 28 | yarn 29 | 30 | # 启动网站 31 | yarn start 32 | ``` 33 | 34 | 一旦网站在本地运行，您可以通过访问 预览网站。 35 | 36 | ## 如何提交文档到这个仓库 37 | 38 | 该项目的默认文档语言是英语。 39 | - 英文文档提交路径：[`./docs`](command:_github.copilot.openRelativePath?%5B%7B%22scheme%22%3A%22file%22%2C%22authority%22%3A%22%22%2C%22path%22%3A%22%2FUsers%2Fduanmeng%2FCode%2Fgithub2%2Fwebsite%2Fdocs%22%2C%22query%22%3A%22%22%2C%22fragment%22%3A%22%22%7D%5D "/Users/duanmeng/Code/github2/website/docs") 40 | - 中文文档提交路径：[`./i18n/zh-Hans/docusaurus-plugin-content-docs/current`](command:_github.copilot.openRelativePath?%5B%7B%22scheme%22%3A%22file%22%2C%22authority%22%3A%22%22%2C%22path%22%3A%22%2FUsers%2Fduanmeng%2FCode%2Fgithub2%2Fwebsite%2Fi18n%2Fzh-Hans%2Fdocusaurus-plugin-content-docs%2Fcurrent%22%2C%22query%22%3A%22%22%2C%22fragment%22%3A%22%22%7D%5D "/Users/duanmeng/Code/github2/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current") 41 | 42 | 注意：中文和英文文档的目录结构必须完全相同。 43 | 44 | ## 发送您的拉取请求 45 | 46 | 在检查所有更改后，请[创建一个拉取请求](https://help.github.com/en/articles/creating-a-pull-request)并附上[DCO](https://github.com/apps/dco)。 -------------------------------------------------------------------------------- /src/components/HomepageFeatures.module.css: -------------------------------------------------------------------------------- 1 | .features { 2 | display: flex; 3 | justify-content: center; 4 | } 5 | .featuresContainer { 6 | max-width: 1800px; 7 | margin: 80px 80px 40px 80px; 8 | width: 100%; 9 | } 10 | .featureTitle { 11 | position: relative; 12 | font-family: PingFangSC-Semibold; 13 | font-size: 40px; 14 | color: #333333; 15 | font-weight: 600; 16 | margin-bottom: 40px; 17 | } 18 | .titleBg { 19 | position: absolute; 20 | height: 100px; 21 | top: 0px; 22 | left: 170px; 23 | z-index: -1; 24 | } 25 | .featureText { 26 | display: flex; 27 | flex-direction: column; 28 | justify-content: flex-end; 29 | margin-bottom: 10px; 30 | height: 100px; 31 | width: 270px; 32 | } 33 | .line { 34 | width: 55px; 35 | border-bottom: 5px solid #3355e9; 36 | margin: 0 0 20px 5px; 37 | } 38 | .featureItemLeft { 39 | display: flex; 40 | align-items: flex-start; 41 | justify-content: space-between; 42 | width: 100%; 43 | background-color: #F9FBFF; 44 | position: relative; 45 | margin-bottom: 85px; 46 | border-radius: 30px; 47 | } 48 | 49 | .featureItemRight { 50 | flex-direction: row-reverse; 51 | } 52 | 53 | .featureDes { 54 | flex: 1; 55 | display: flex; 56 | padding-top: 30px; 57 | flex-direction: column; 58 | align-items: left; 59 | justify-content: center; 60 | margin-left: 45px; 61 | margin-bottom: 30px; 62 | z-index: 1; 63 | } 64 | 65 | .featureItemRight .featureDes { 66 | align-items: flex-end; 67 | margin-right: 45px; 68 | } 69 | 70 | .featureItemRight .featureDesTitle { 71 | width: 55%; 72 | } 73 | 74 | .featureArrangeImg { 75 | position: absolute; 76 | height: 400px; 77 | top: 50%; 78 | transform: translateY(-50%); 79 | right: 0; 80 | } 81 | 82 | .featureDispatchImg { 83 | position: absolute; 84 | height: 440px; 85 | top: 50%; 86 | transform: translateY(-50%); 87 | left: 0; 88 | } 89 | 90 | .featureNetworkImg { 91 | position: absolute; 92 | height: 440px; 93 | top: 50%; 94 | transform: translateY(-50%); 95 | right: 0; 96 | } 97 | 98 | .featureImg { 99 | width: 100%; 100 | height: 100%; 101 | } 102 | 103 | .featureDesTitle { 104 | font-family: PingFangSC-Semibold; 105 | font-size: 32px; 106 | color: #333333; 107 | font-weight: 600; 108 | margin-bottom: 5px; 109 | } 110 | 111 | .featureDesInfo { 112 | font-family: PingFangSC-Semibold; 113 | font-size: 16px; 114 | color: #626262; 115 | line-height: 22px; 116 | font-weight: 600; 117 | max-width: 55%; 118 | } 119 | 120 | .featureDesDetails { 121 | font-size: 16px; 122 | max-width: 55%; 123 | } 124 | -------------------------------------------------------------------------------- /static/script/ad.js: -------------------------------------------------------------------------------- 1 | // function called if wwads is blocked 2 | function ABDetected() { 3 | document.getElementsByClassName("wwads-cn")[0].insertAdjacentHTML("beforeend", "
为了本站的长期运营，请将我们的网站加入广告拦截器的白名单，感谢您的支持！万维广告
"); 4 | }; 5 | 6 | //check document ready 7 | function docReady(t) { 8 | "complete" === document.readyState || 9 | "interactive" === document.readyState 10 | ? setTimeout(t, 1) 11 | : document.addEventListener("DOMContentLoaded", t); 12 | } 13 | 14 | //check if wwads' fire function was blocked after document is ready with 3s timeout (waiting the ad loading) 15 | docReady(function () { 16 | setTimeout(function () { 17 | if( window._AdBlockInit === undefined ){ 18 | ABDetected(); 19 | } 20 | }, 3000); 21 | }); 22 | -------------------------------------------------------------------------------- /docusaurus.config.js: -------------------------------------------------------------------------------- 1 | //具体配置可以查看 https://docusaurus.io/docs/api/docusaurus-config 2 | import { url } from "inspector"; 3 | import { type } from "os"; 4 | import { themes as prismThemes } from "prism-react-renderer"; 5 | 6 | // /** @type {import('@docusaurus/types').Config} */ 7 | const config = { 8 | title: "Kosmos", //网站标题 9 | tagline: "Flexible and scalable Kubernetes multi-cluster management solution. The limitless expansion of Kubernetes. Make Kubernetes without boundaries", //网站标语 10 | favicon: "img/favicon.ico", //你的网站图标的路径；必须是可以用于链接 href 的 URL 11 | 12 | // Set the production url of your site here 13 | url: "https://kosmos-io.github.io", 14 | 15 | // Set the // pathname under which your site is served 16 | // For GitHub pages deployment, it is often '//' 17 | baseUrl: "/kosmos-website/", 18 | 19 | // GitHub pages deployment config. 20 | // If you aren't using GitHub pages, you don't need these. 21 | organizationName: "kosmos", // Usually your GitHub org/user name. 22 | projectName: "kosmos-website", // Usually your repo name. 23 | deploymentBranch: "gh-pages", 24 | trailingSlash: false, 25 | 26 | onBrokenLinks: "ignore", //Docusaurus 在检测到无效链接时的行为 27 | onBrokenMarkdownLinks: "warn", 28 | 29 | //i18n配置相关 30 | i18n: { 31 | defaultLocale: "en", 32 | locales: ["en", "zh-Hans"], 33 | localeConfigs: { 34 | en: { 35 | label: "English", 36 | }, 37 | "zh-Hans": { 38 | label: "简体中文", 39 | }, 40 | }, 41 | }, 42 | 43 | scripts: [], //一组要加载的脚本 44 | presets: [ 45 | [ 46 | "classic", 47 | // /** @type {import('@docusaurus/preset-classic').Options} */ 48 | { 49 | docs: { 50 | sidebarPath: require.resolve("./sidebars.js"), 51 | showLastUpdateTime: true, 52 | showLastUpdateAuthor: true, 53 | editUrl: 54 | "https://github.com/facebook/docusaurus/tree/main/packages/create-docusaurus/templates/shared/", 55 | }, 56 | blog: { 57 | showReadingTime: true, 58 | editUrl: 59 | "https://github.com/facebook/docusaurus/tree/main/packages/create-docusaurus/templates/shared/", 60 | }, 61 | theme: { 62 | customCss: require.resolve("./src/css/custom.css"), 63 | }, 64 | sitemap: { 65 | changefreq: "weekly", 66 | priority: 0.5, 67 | }, 68 | }, 69 | ], 70 | ], 71 | plugins: ["docusaurus-tailwindcss-loader", "docusaurus-plugin-image-zoom"], 72 | themeConfig: { 73 | image: "img/docusaurus-social-card.jpg", 74 | //切换主题按钮关闭 75 | colorMode: { 76 | defaultMode: "light", 77 | disableSwitch: true, 78 | respectPrefersColorScheme: false, 79 | }, 80 | navbar: { 81 | title: "Kosmos", 82 | logo: { 83 | alt: "Kosmos Logo", 84 | src: "img/logo.svg", 85 | }, 86 | items: [ 87 | { 88 | type: "docSidebar", 89 | sidebarId: "tutorialSidebar", 90 | label: "Documentation", 91 | position: "left", 92 | }, 93 | { 94 | type: "localeDropdown", 95 | position: "right", 96 | }, 97 | { 98 | href: "https://kosmos-io.github.io/website/quick-start", 99 | label: "Examples", 100 | position: "right" 101 | }, 102 | { 103 | href: "https://github.com/kosmos-io/kosmos", 104 | className: "header-github-link", 105 | "aria-label": "GitHub repository", 106 | position: "right", 107 | }, 108 | ], 109 | }, 110 | footer: { 111 | style: "dark", 112 | links: [ 113 | { 114 | title: "Documentation", 115 | items: [ 116 | { 117 | label: "Getting Started", 118 | to: "getting-started/introduction", 119 | }, 120 | { 121 | label: "Tutorials", 122 | to: "tutorials/mcs-discovery", 123 | }, 124 | { 125 | label: "Proposals", 126 | to: "proposals/k8s-in-k8s", 127 | }, 128 | ], 129 | }, 130 | { 131 | title: "Community", 132 | items: [ 133 | { 134 | label: "Community Address Name", 135 | href: "https://github.com/kosmos-io/kosmos", 136 | }, 137 | ], 138 | }, 139 | { 140 | title: "More", 141 | items: [ 142 | { 143 | label: "GitHub", 144 | href: "https://github.com/kosmos-io/kosmos", 145 | }, 146 | ], 147 | }, 148 | ], 149 | }, 150 | prism: { 151 | theme: prismThemes.github, 152 | darkTheme: prismThemes.dracula, 153 | additionalLanguages: ["cue", "powershell"], 154 | }, 155 | zoom: { 156 | selector: ".markdown :not(em) > img", 157 | config: { 158 | background: { 159 | light: "rgb(255, 255, 255)", 160 | dark: "rgb(50, 50, 50)", 161 | }, 162 | }, 163 | }, 164 | }, 165 | }; 166 | 167 | export default config; 168 | -------------------------------------------------------------------------------- /docs/proposals/k8s-in-k8s.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: k8s-in-k8s 3 | title: 'Kubernetes in Kubernetes' 4 | --- 5 | 6 | # Kubernetes in Kubernetes 7 | 8 | ## Summary 9 | 10 | Kosmos now supports the management of different Kubernetes (k8s) clusters, offering a unified control plane view for resource management and scheduling. This proposal introduces a method to achieve Kubernetes within Kubernetes (k8s in k8s) using Kosmos. 11 | 12 | The approach involves creating different Kubernetes control plane services within a single k8s cluster and deploying Kosmos' ClusterTree service. The ClusterTree listens to the kube-apiserver address of the created k8s control plane and incorporates the current cluster as a node into the dynamically created control plane. Each control plane has its exclusive etcd, ensuring the isolation of permissions and data for different tenants. 13 | 14 | ## Motivation 15 | 16 | In some cases, cloud providers offer a large k8s cluster as infrastructure to different tenants, providing computational resources like CPU and GPU. Therefore, a k8s in k8s solution is required to isolate permissions and data for different tenants. 17 | 18 | ### Goals 19 | 20 | Divide the computational power of a single k8s cluster into multiple distinct k8s clusters, achieving permission and data isolation for different tenants. 21 | 22 | ### Non-Goals 23 | 24 | ## Proposal 25 | 26 | Implement a k8s in k8s solution with Kosmos, allocating a single cluster's computational resources to different tenants. 27 | 28 | ### Function Overview 29 | 30 | 1. Enable splitting a single cluster into different smaller clusters through Kosmos. 31 | 2. Provide tenants with exclusive etcd and k8s control planes to ensure complete isolation of permissions and resources. 32 | 3. Ensure tenants' control planes offer the same functionalities as a single k8s control plane, without changing the user experience. 33 | 34 | ### User Stories (Optional) 35 | 36 | #### Story 1 37 | 38 | Imagine a supercluster with tens of thousands of nodes, hosting many different tenants. These tenants do not require all the computational resources of the entire k8s cluster, and for security reasons, it's crucial to ensure complete permission isolation between tenants. The native RBAC permission isolation solution in k8s might be insufficient, and k8s's reliance on namespaces for resource isolation could alter the user experience, restricting operations to the namespace level, which could be user-unfriendly. 39 | 40 | ## Design Details 41 | 42 | ### Overall Architecture 43 | 44 | ![Kubernetes_in_Kubernetes.svg](img/Kubernetes_in_Kubernetes.svg) 45 | 46 | As shown in the figure, each tenant has a separate k8s control plane and etcd. Each tenant's k8s control plane uses Kosmos to manage the nodes of the current cluster as virtual nodes for the control plane. The CRD configuration determines the number of nodes from the current cluster each control plane should manage. 47 | 48 | ### CRD Design 49 | 50 | ````shell script 51 | apiVersion: kosmos.io/v1alpha1 52 | kind: VirtualCluster 53 | metadata: 54 | name: tenant1-cluster 55 | spec: 56 | kubeconfig: XXXXXX 57 | promoteResources: 58 | nodes: 59 | - node1 60 | - node2 61 | resources: 62 | cpu: 10 63 | memory: 100Gi 64 | status: 65 | phase: Initialized 66 | ```` 67 | 68 | ### Overall Process 69 | 70 | 1. A cluster administrator creates a VirtualCluster CR resource. 71 | 2. The Kosmos-operator listens for the creation of the VirtualCluster, starts the creation of the Kubernetes control plane services including the kube-apiserver, kube-controller-manager, kube-scheduler, and etcd. Once the Kubernetes control plane services are setup, the VirtualCluster's status.phase will be set to Initialized, and the spec.kubeconfig field will be populated. 72 | 3. After the control plane is set up, the Kosmos-operator creates the Kosmos clusterTree service and configures the clusterTree's apiserver address to the current control plane using the kubeconfig value from the VirtualCluster. 73 | 4. Once the clusterTree is ready, the Kosmos-operator manages the tenant's required node resources or computational resources like CPU and memory into the tenant's control plane based on the VirtualCluster's promoteResource configuration. Tenants can then manage resources in the control plane using the spec.kubeconfig. 74 | 75 | ## Test Plan 76 | 77 | ### Unit Test 78 | 79 | ### E2E Test 80 | 81 | | Test Case | Case Description | 82 | | ----------------------------- | ------------------------------------------------------------- | 83 | | Virtual Control Plane Creation | Test the creation of a virtual control plane through the VirtualCluster CRD | 84 | | Service Creation in Virtual Control Plane | Test the creation of services in the virtual control plane and their external functionality | 85 | -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/proposals/k8s-in-k8s.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: k8s-in-k8s 3 | title: 'Kubernetes in Kubernetes' 4 | --- 5 | 6 | # Kubernetes in Kubernetes 7 | 8 | ## Summary 9 | 10 | Kosmos now supports the management of different Kubernetes (k8s) clusters, offering a unified control plane view for resource management and scheduling. This proposal introduces a method to achieve Kubernetes within Kubernetes (k8s in k8s) using Kosmos. 11 | 12 | The approach involves creating different Kubernetes control plane services within a single k8s cluster and deploying Kosmos' ClusterTree service. The ClusterTree listens to the kube-apiserver address of the created k8s control plane and incorporates the current cluster as a node into the dynamically created control plane. Each control plane has its exclusive etcd, ensuring the isolation of permissions and data for different tenants. 13 | 14 | ## Motivation 15 | 16 | In some cases, cloud providers offer a large k8s cluster as infrastructure to different tenants, providing computational resources like CPU and GPU. Therefore, a k8s in k8s solution is required to isolate permissions and data for different tenants. 17 | 18 | ### Goals 19 | 20 | Divide the computational power of a single k8s cluster into multiple distinct k8s clusters, achieving permission and data isolation for different tenants. 21 | 22 | ### Non-Goals 23 | 24 | ## Proposal 25 | 26 | Implement a k8s in k8s solution with Kosmos, allocating a single cluster's computational resources to different tenants. 27 | 28 | ### Function Overview 29 | 30 | 1. Enable splitting a single cluster into different smaller clusters through Kosmos. 31 | 2. Provide tenants with exclusive etcd and k8s control planes to ensure complete isolation of permissions and resources. 32 | 3. Ensure tenants' control planes offer the same functionalities as a single k8s control plane, without changing the user experience. 33 | 34 | ### User Stories (Optional) 35 | 36 | #### Story 1 37 | 38 | Imagine a supercluster with tens of thousands of nodes, hosting many different tenants. These tenants do not require all the computational resources of the entire k8s cluster, and for security reasons, it's crucial to ensure complete permission isolation between tenants. The native RBAC permission isolation solution in k8s might be insufficient, and k8s's reliance on namespaces for resource isolation could alter the user experience, restricting operations to the namespace level, which could be user-unfriendly. 39 | 40 | ## Design Details 41 | 42 | ### Overall Architecture 43 | 44 | ![Kubernetes_in_Kubernetes.svg](img/Kubernetes_in_Kubernetes.svg) 45 | 46 | As shown in the figure, each tenant has a separate k8s control plane and etcd. Each tenant's k8s control plane uses Kosmos to manage the nodes of the current cluster as virtual nodes for the control plane. The CRD configuration determines the number of nodes from the current cluster each control plane should manage. 47 | 48 | ### CRD Design 49 | 50 | ````shell script 51 | apiVersion: kosmos.io/v1alpha1 52 | kind: VirtualCluster 53 | metadata: 54 | name: tenant1-cluster 55 | spec: 56 | kubeconfig: XXXXXX 57 | promoteResources: 58 | nodes: 59 | - node1 60 | - node2 61 | resources: 62 | cpu: 10 63 | memory: 100Gi 64 | status: 65 | phase: Initialized 66 | ```` 67 | 68 | ### Overall Process 69 | 70 | 1. A cluster administrator creates a VirtualCluster CR resource. 71 | 2. The Kosmos-operator listens for the creation of the VirtualCluster, starts the creation of the Kubernetes control plane services including the kube-apiserver, kube-controller-manager, kube-scheduler, and etcd. Once the Kubernetes control plane services are setup, the VirtualCluster's status.phase will be set to Initialized, and the spec.kubeconfig field will be populated. 72 | 3. After the control plane is set up, the Kosmos-operator creates the Kosmos clusterTree service and configures the clusterTree's apiserver address to the current control plane using the kubeconfig value from the VirtualCluster. 73 | 4. Once the clusterTree is ready, the Kosmos-operator manages the tenant's required node resources or computational resources like CPU and memory into the tenant's control plane based on the VirtualCluster's promoteResource configuration. Tenants can then manage resources in the control plane using the spec.kubeconfig. 74 | 75 | ## Test Plan 76 | 77 | ### Unit Test 78 | 79 | ### E2E Test 80 | 81 | | Test Case | Case Description | 82 | | ----------------------------- | ------------------------------------------------------------- | 83 | | Virtual Control Plane Creation | Test the creation of a virtual control plane through the VirtualCluster CRD | 84 | | Service Creation in Virtual Control Plane | Test the creation of services in the virtual control plane and their external functionality | 85 | -------------------------------------------------------------------------------- /docs/getting-started/introduction.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: introduction 3 | title: 'Introduction' 4 | --- 5 | 6 | # Introduction 7 | 8 | ## What is Kosmos? 9 | 10 | Kosmos is an open-source, all-in-one distributed cloud-native solution. The name "kosmos" combines 'k' representing Kubernetes and 'cosmos' which means universe in Greek, symbolizing the limitless expansion of Kubernetes. 11 | 12 | Currently, Kosmos primarily consists of three major modules: `ClusterLink`, `ClusterTree` and `Scheduler`. 13 | 14 | ## ClusterLink 15 | 16 | The target of ClusterLink is to establish connectivity between multiple Kubernetes clusters. This module can be deployed and used independently. ClusterLink enables `Pods` to access `Pods` and `Services` across clusters, as if they were in the single cluster. Currently, this module primarily offers the following ability: 17 | 1. **Cross-cluster PodIP and ServiceIP communication**: L3 network connections across multiple Kubernetes clusters, based on tunneling technologies such as VxLAN and IPsec. This allows users to conduct `Pod-to-Pod` and `Pod-to-Service` communication within the global clusters scope. 18 | 2. **Multi-Mode Support**: When join clusters, you can choose `P2P` or `Gateway` mode. The P2P model provides a second-layer network interconnection at the overlay level, offering shorter network paths and superior performance. When selecting the `Gateway` mode, it demonstrates superior compatibility, which is well-suited for hybrid and multi-cloud scenarios. 19 | 3. **Support for Global IP Allocation**: ClusterLink allows for the presence of two or more clusters within the global clusters to use the same `Pod/Service` network segments, making it convenient for users to manage subnet. ClusterLink supports configuring the mapping relationship between `PodCIDR/ServiceCIDR` and `GlobalCIDR`. `GlobalIP` is globally unique, enabling cross-cluster communication for services with conflicting network segments through `GlobalIP`. 20 | 4. **IPv6/IPv4 Dual-Stack Support** 21 | 22 | ### Network Architecture 23 | 24 | The Kosmos ClusterLink module currently includes the following key components: 25 | ![ClusterLink_Architecture.png](img/ClusterLink_Architecture.png) 26 | 27 | - `Controller-Manager`：Collect network information of the current cluster and monitors changes in network settings. 28 | - `Network-manager`：Calculates the network configurations required for each node. 29 | - `Agent`：A DaemonSet used for configuring the host network, including tasks such as tunnel creation, routing, NAT, and so on. 30 | - `Multi-Cluster-Coredns`：Implements multi-cluster service discovery. 31 | - `Elector`：Elects the gateway node. 32 | 33 | ## ClusterTree 34 | 35 | The Kosmos clustertree module realizes the tree-like scaling of Kubernetes and achieves cross-cluster orchestration of applications.This is the technological foundation for Kosmos to achieve boundless of Kubernetes. 36 | ![ClusterTree_Architecture.png](img/ClusterTree_Architecture.png) 37 | 38 | Currently, it primarily supports the following ability: 39 | 1. **Full Compatibility with k8s API**: Users can interact with the host cluster's `kube-apiserver` using tools like `kubectl`, `client-go`, and others just like they normally would. However, the `Pods` are actually distributed across the entire multi-cloud, multi-cluster environment. 40 | 2. **Support for Stateful and k8s-native Applications**: In addition to stateless applications, Kosmos also facilitates the orchestration of stateful applications and k8s-native applications (interacting with `kube-apiserver`). Kosmos will automatically detect the storage and permission resources that `Pods` depend on, such as pv/pvc, sa, etc., and perform bidirectional synchronization between control plane clusters and data plane clusters. 41 | 3. **Diverse Pod Topology Constraints**: Users can easily control the distribution of Pods within the global clusters, such as by region, availability zone, cluster, or node. This helps achieve high availability and improve resource utilization. 42 | 43 | ## Scheduler 44 | 45 | The Kosmos scheduling module is an extension developed on top of the Kubernetes scheduling framework, aiming to meet the container management needs in mixed-node and sub-cluster environments. It provides the following core features to enhance the flexibility and efficiency of container management: 46 | 47 | 1. **Flexible Node and Cluster Hybrid Scheduling**: The Kosmos scheduling module allows users to intelligently schedule workloads between real nodes and sub-clusters based on custom configurations. This enables users to make optimal use of resources across different nodes, ensuring the best performance and availability of workloads. Based on this capability, Kosmos enables workloads to achieve flexible cross-cloud and cross-cluster deployments. 48 | 2. **Fine-grained Container Distribution Strategy**: By introducing Custom Resource Definitions (CRD), users can exert precise control over the distribution of workloads. The configuration of CRD allows users to explicitly specify the number of pods for the workload in different clusters and adjust the distribution ratio as needed. 49 | 3. **Fine-grained Fragmented Resource Handling**: The Kosmos scheduling module intelligently detects fragmented resources within sub-clusters, effectively avoiding situations where pod deployment encounters insufficient resources in the sub-cluster. This helps ensure a more balanced allocation of resources across different nodes, enhancing system stability and performance. 50 | 51 | Whether building a hybrid cloud environment or requiring flexible deployment of workloads across different clusters, the Kosmos scheduling module serves as a reliable solution, assisting users in managing containerized applications more efficiently. 52 | 53 | ## What's Next 54 | - Start to [install Kosmos](https://kosmos-io.github.io/website/v0.2.0/quick-start). 55 | - Learn Kosmos's [tutorials](https://kosmos-io.github.io/website/v0.2.0/tutorials/mcs-discovery). 56 | - Learn Kosmos's [proposals](https://kosmos-io.github.io/website/v0.2.0/proposals/k8s-in-k8s). 57 | -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/introduction.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: introduction 3 | title: 'Introduction' 4 | --- 5 | 6 | # 介绍 7 | 8 | ## What is Kosmos? 9 | 10 | Kosmos is an open-source, all-in-one distributed cloud-native solution. The name "kosmos" combines 'k' representing Kubernetes and 'cosmos' which means universe in Greek, symbolizing the limitless expansion of Kubernetes. 11 | 12 | Currently, Kosmos primarily consists of three major modules: `ClusterLink`, `ClusterTree` and `Scheduler`. 13 | 14 | ## ClusterLink 15 | 16 | The target of ClusterLink is to establish connectivity between multiple Kubernetes clusters. This module can be deployed and used independently. ClusterLink enables `Pods` to access `Pods` and `Services` across clusters, as if they were in the single cluster. Currently, this module primarily offers the following ability: 17 | 1. **Cross-cluster PodIP and ServiceIP communication**: L3 network connections across multiple Kubernetes clusters, based on tunneling technologies such as VxLAN and IPsec. This allows users to conduct `Pod-to-Pod` and `Pod-to-Service` communication within the global clusters scope. 18 | 2. **Multi-Mode Support**: When join clusters, you can choose `P2P` or `Gateway` mode. The P2P model provides a second-layer network interconnection at the overlay level, offering shorter network paths and superior performance. When selecting the `Gateway` mode, it demonstrates superior compatibility, which is well-suited for hybrid and multi-cloud scenarios. 19 | 3. **Support for Global IP Allocation**: ClusterLink allows for the presence of two or more clusters within the global clusters to use the same `Pod/Service` network segments, making it convenient for users to manage subnet. ClusterLink supports configuring the mapping relationship between `PodCIDR/ServiceCIDR` and `GlobalCIDR`. `GlobalIP` is globally unique, enabling cross-cluster communication for services with conflicting network segments through `GlobalIP`. 20 | 4. **IPv6/IPv4 Dual-Stack Support** 21 | 22 | ### Network Architecture 23 | 24 | The Kosmos ClusterLink module currently includes the following key components: 25 | ![ClusterLink_Architecture.png](img/ClusterLink_Architecture.png) 26 | 27 | - `Controller-Manager`：Collect network information of the current cluster and monitors changes in network settings. 28 | - `Network-manager`：Calculates the network configurations required for each node. 29 | - `Agent`：A DaemonSet used for configuring the host network, including tasks such as tunnel creation, routing, NAT, and so on. 30 | - `Multi-Cluster-Coredns`：Implements multi-cluster service discovery. 31 | - `Elector`：Elects the gateway node. 32 | 33 | ## ClusterTree 34 | 35 | The Kosmos clustertree module realizes the tree-like scaling of Kubernetes and achieves cross-cluster orchestration of applications.This is the technological foundation for Kosmos to achieve boundless of Kubernetes. 36 | ![ClusterTree_Architecture.png](img/ClusterTree_Architecture.png) 37 | 38 | Currently, it primarily supports the following ability: 39 | 1. **Full Compatibility with k8s API**: Users can interact with the host cluster's `kube-apiserver` using tools like `kubectl`, `client-go`, and others just like they normally would. However, the `Pods` are actually distributed across the entire multi-cloud, multi-cluster environment. 40 | 2. **Support for Stateful and k8s-native Applications**: In addition to stateless applications, Kosmos also facilitates the orchestration of stateful applications and k8s-native applications (interacting with `kube-apiserver`). Kosmos will automatically detect the storage and permission resources that `Pods` depend on, such as pv/pvc, sa, etc., and perform bidirectional synchronization between control plane clusters and data plane clusters. 41 | 3. **Diverse Pod Topology Constraints**: Users can easily control the distribution of Pods within the global clusters, such as by region, availability zone, cluster, or node. This helps achieve high availability and improve resource utilization. 42 | 43 | ## Scheduler 44 | 45 | The Kosmos scheduling module is an extension developed on top of the Kubernetes scheduling framework, aiming to meet the container management needs in mixed-node and sub-cluster environments. It provides the following core features to enhance the flexibility and efficiency of container management: 46 | 47 | 1. **Flexible Node and Cluster Hybrid Scheduling**: The Kosmos scheduling module allows users to intelligently schedule workloads between real nodes and sub-clusters based on custom configurations. This enables users to make optimal use of resources across different nodes, ensuring the best performance and availability of workloads. Based on this capability, Kosmos enables workloads to achieve flexible cross-cloud and cross-cluster deployments. 48 | 2. **Fine-grained Container Distribution Strategy**: By introducing Custom Resource Definitions (CRD), users can exert precise control over the distribution of workloads. The configuration of CRD allows users to explicitly specify the number of pods for the workload in different clusters and adjust the distribution ratio as needed. 49 | 3. **Fine-grained Fragmented Resource Handling**: The Kosmos scheduling module intelligently detects fragmented resources within sub-clusters, effectively avoiding situations where pod deployment encounters insufficient resources in the sub-cluster. This helps ensure a more balanced allocation of resources across different nodes, enhancing system stability and performance. 50 | 51 | Whether building a hybrid cloud environment or requiring flexible deployment of workloads across different clusters, the Kosmos scheduling module serves as a reliable solution, assisting users in managing containerized applications more efficiently. 52 | 53 | ## What's Next 54 | - Start to [install Kosmos](https://kosmos-io.github.io/website/quick-start). 55 | - Learn Kosmos's [tutorials](https://kosmos-io.github.io/website/tutorials/mcs-discovery). 56 | - Learn Kosmos's [proposals](https://kosmos-io.github.io/website/proposals/k8s-in-k8s). 57 | -------------------------------------------------------------------------------- /src/css/custom.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | /** 6 | * Any CSS included here will be global. The classic template 7 | * bundles Infima by default. Infima is a CSS framework designed to 8 | * work well for content-centric websites. 9 | */ 10 | 11 | /* You can override the default Infima variables here. */ 12 | :root { 13 | --ifm-color-primary: #3b54e0; 14 | --ifm-color-primary-dark: #0598ea; 15 | --ifm-color-primary-darker: #0590dd; 16 | --ifm-color-primary-darkest: #0476b6; 17 | --ifm-color-primary-light: #2ab0fb; 18 | --ifm-color-primary-lighter: #37b5fb; 19 | --ifm-color-primary-lightest: #5ec3fc; 20 | --ifm-code-font-size: 95%; 21 | --ifm-font-size-base: 16px; 22 | --hero-border-color: rgba(0, 0, 0, 0.1); 23 | --docusaurus-announcement-bar-height: "auto" !important; 24 | } 25 | 26 | .code-block-primary-line { 27 | display: block; 28 | margin: 0 calc(-1 * var(--ifm-pre-padding)); 29 | padding: 0 var(--ifm-pre-padding); 30 | background-color: #e0f2fe; 31 | } 32 | .code-block-primary-line .codeLineNumber_node_modules-\@docusaurus-theme-classic-lib-theme-CodeBlock-Line-styles-module{ 33 | background-color: #e0f2fe; 34 | border-left: 3px solid #60a5fa; 35 | } 36 | 37 | .code-block-warning-line { 38 | display: block; 39 | margin: 0 calc(-1 * var(--ifm-pre-padding)); 40 | padding: 0 var(--ifm-pre-padding); 41 | background-color: #fef9c3; 42 | } 43 | .code-block-warning-line .codeLineNumber_node_modules-\@docusaurus-theme-classic-lib-theme-CodeBlock-Line-styles-module{ 44 | background-color: #fef9c3; 45 | border-left: 3px solid #facc15; 46 | } 47 | 48 | [data-theme='dark'] .code-block-primary-line, 49 | [data-theme='dark'] .code-block-warning-line { 50 | background-color: rgba(0, 0, 0, 0.6); 51 | } 52 | [data-theme='dark'] .code-block-primary-line .codeLineNumber_node_modules-\@docusaurus-theme-classic-lib-theme-CodeBlock-Line-styles-module, 53 | [data-theme='dark'] .code-block-warning-line .codeLineNumber_node_modules-\@docusaurus-theme-classic-lib-theme-CodeBlock-Line-styles-module{ 54 | background-color: rgba(0, 0, 0, 0.6); 55 | } 56 | 57 | 58 | 59 | .tabs-container { 60 | box-shadow: 0 0 12px rgba(0, 0, 0, .18); 61 | @apply p-2 rounded-lg; 62 | } 63 | 64 | 65 | .theme-code-block:hover .buttonGroup_node_modules-\@docusaurus-theme-classic-lib-theme-CodeBlock-Content-styles-module button { 66 | opacity: 0.75 !important; 67 | } 68 | 69 | 70 | .divider { 71 | border: 1px dashed transparent; 72 | background: linear-gradient(-45deg, #94a3b8 30%, #f1f4ff 0, #f1f4ff 50%, #f1f4ff 0, #f1f4ff 80%, #f1f4ff 0); 73 | background-size: 1.5rem 1.5rem; 74 | } 75 | 76 | 77 | /* github icon */ 78 | .header-github-link:hover { 79 | opacity: 0.6; 80 | } 81 | .header-github-link::before { 82 | content: ''; 83 | width: 24px; 84 | height: 24px; 85 | display: flex; 86 | background: url("data:image/svg+xml,%3Csvg viewBox='0 0 24 24' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath d='M12 .297c-6.63 0-12 5.373-12 12 0 5.303 3.438 9.8 8.205 11.385.6.113.82-.258.82-.577 0-.285-.01-1.04-.015-2.04-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729 1.205.084 1.838 1.236 1.838 1.236 1.07 1.835 2.809 1.305 3.495.998.108-.776.417-1.305.76-1.605-2.665-.3-5.466-1.332-5.466-5.93 0-1.31.465-2.38 1.235-3.22-.135-.303-.54-1.523.105-3.176 0 0 1.005-.322 3.3 1.23.96-.267 1.98-.399 3-.405 1.02.006 2.04.138 3 .405 2.28-1.552 3.285-1.23 3.285-1.23.645 1.653.24 2.873.12 3.176.765.84 1.23 1.91 1.23 3.22 0 4.61-2.805 5.625-5.475 5.92.42.36.81 1.096.81 2.22 0 1.606-.015 2.896-.015 3.286 0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12'/%3E%3C/svg%3E") 87 | no-repeat; 88 | } 89 | 90 | [data-theme='dark'] .header-github-link::before { 91 | background: url("data:image/svg+xml,%3Csvg viewBox='0 0 24 24' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath fill='white' d='M12 .297c-6.63 0-12 5.373-12 12 0 5.303 3.438 9.8 8.205 11.385.6.113.82-.258.82-.577 0-.285-.01-1.04-.015-2.04-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729 1.205.084 1.838 1.236 1.838 1.236 1.07 1.835 2.809 1.305 3.495.998.108-.776.417-1.305.76-1.605-2.665-.3-5.466-1.332-5.466-5.93 0-1.31.465-2.38 1.235-3.22-.135-.303-.54-1.523.105-3.176 0 0 1.005-.322 3.3 1.23.96-.267 1.98-.399 3-.405 1.02.006 2.04.138 3 .405 2.28-1.552 3.285-1.23 3.285-1.23.645 1.653.24 2.873.12 3.176.765.84 1.23 1.91 1.23 3.22 0 4.61-2.805 5.625-5.475 5.92.42.36.81 1.096.81 2.22 0 1.606-.015 2.896-.015 3.286 0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12'/%3E%3C/svg%3E") 92 | no-repeat; 93 | } 94 | 95 | .sm-hidden { 96 | display: flex; 97 | } 98 | @media (min-width: 640px) { 99 | .sm-hidden { 100 | display: none !important; 101 | } 102 | } 103 | 104 | .z-input { 105 | @apply block w-full border-0 py-1.5 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6; 106 | } 107 | 108 | .z-button { 109 | @apply bg-blue-500 hover:bg-blue-600 text-white font-bold py-2 px-4 w-36 110 | } 111 | 112 | .z-textarea { 113 | @apply block w-full border-0 py-1.5 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6 114 | } 115 | 116 | /* `代码块` 的文字颜色 */ 117 | code { 118 | color: #c8254e; 119 | } 120 | 121 | 122 | .navbar--fixed-top { 123 | height:70px 124 | } 125 | 126 | .navbar--fixed-top .navbar__logo { 127 | height: 44px; 128 | margin-right: 11px; 129 | padding-left: 5px; 130 | } 131 | .navbar--fixed-top .navbar__title { 132 | font-size:28px 133 | } 134 | .navbar__link { 135 | font-size:16px 136 | } 137 | 138 | .header-github-link::before { 139 | height:35px; 140 | width:35px; 141 | } 142 | 143 | .navbar__title { 144 | font-size: 30px; /* 调整标题文字大小 */ 145 | font-weight: bold; /* 调整标题文字粗细 */ 146 | } 147 | 148 | .hero--primary { 149 | justify-content: center; 150 | } 151 | -------------------------------------------------------------------------------- /src/components/HomepageFeatures.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import clsx from 'clsx'; 3 | import styles from './HomepageFeatures.module.css'; 4 | 5 | 6 | 7 | function FeatureArrange() { 8 | return ( 9 |
10 |
11 |
Orchestration(ClusterTree)
12 |
The Kosmos ClusterTree module extends Kubernetes clusters in a tree-like manner, achieving cross-cluster orchestration of applications.
13 |
    14 |
  • 15 | Fully compatible with k8s API: Users can interact with the kube-apiserver of the host cluster as usual using tools like kubectl, client-go, etc. However, Pods are actually distributed across the entire multi-cloud, multi-cluster environment. 16 |
    • 17 |
  • 18 | Automated resource discovery: Kosmos supports the orchestration of all Kubernetes native applications. For stateful applications, Kosmos will automatically detect storage and permission resources that Pods depend on, such as pv/pvc, sa, etc., and perform automated bi-directional synchronization. 19 |
    • 20 |
  • 21 | Diverse Pod topology constraints: Users can easily control the distribution of Pods within the global cluster, for example, by region, availability zone, cluster, or node. This helps achieve high availability of applications and improve cluster resource utilization. 22 |
    • 23 |
24 |
25 |
26 | 27 |
28 |
29 | ) 30 | } 31 | 32 | function FeatureDispatch() { 33 | return ( 34 |
35 |
36 |
Scheduler
37 |
The Kosmos scheduler module is developed as an extension of the Kubernetes scheduling framework and serves as a reliable solution for managing containerized applications more efficiently and flexibly in hybrid, multi-cloud, and multi-cluster environments.
38 |
    39 |
  • 40 | Flexible scheduling of nodes and clusters: The Kosmos scheduler module allows users to intelligently schedule workloads between physical nodes and sub-clusters based on custom configurations. This enables users to fully utilize the resources of different nodes to ensure the best performance and availability of workloads. With this functionality, Kosmos enables flexible cross-cloud and cross-cluster deployment of workloads. 41 |
    • 42 |
  • 43 | Refined container distribution strategy: By introducing Custom Resource Definitions (CRD), users can precisely control the topology distribution of workloads. The configuration of CRDs allows users to specify the number of pods of workloads in different clusters and adjust the distribution ratio according to needs. 44 |
    • 45 |
  • 46 | Granular fragment resource consolidation: The Kosmos scheduler module can intelligently sense fragmentary resources in sub-clusters, effectively preventing situations where sub-cluster resources are insufficient after pods have been scheduled for deployment. This helps to ensure a more even distribution of workload resources across different nodes, improving the system's stability and performance. 47 |
    • 48 |
49 |
50 |
51 | 52 |
53 |
54 | ) 55 | } 56 | 57 | function FeatureNetwork() { 58 | return ( 59 |
60 |
61 |
Multi-Cluster Network(ClusterLink)
62 |
The goal of ClusterLink is to establish connections between multiple Kubernetes clusters. This module can be deployed and used independently. It supports cross-cluster PodIP and ServiceIP communication, multi-mode support (P2P or Gateway), global IP allocation, and dual-stack IPv6/IPv4, among other features.
63 |
    64 |
  • 65 | Cross-cluster PodIP and ServiceIP communication: Based on Linux vxlan tunnel technology, it makes L3 network connections across multiple Kubernetes clusters possible. This allows users to conduct Pod-to-Pod and Pod-to-Service communication within a global cluster scope. 66 |
    • 67 |
  • 68 | Multi-mode support:When joining clusters, you can choose between P2P or Gateway mode. Choosing P2P mode is suitable for underlying network interconnection, offering shorter network paths and superior performance. When selecting Gateway mode, it provides excellent compatibility, making it very suitable for hybrid multi-cloud scenarios. 69 |
    • 70 |
  • 71 | Support for global IP allocation:The Kosmos network allows two or more clusters within a global cluster to use the same Pod/Service subnet, facilitating subnet management for users. It supports configuring the mapping relationship between PodCIDR/ServiceCIDR and GlobalCIDR. GlobalIP is globally unique, and through GlobalIP, cross-cluster communication between services with conflicting network segments can be achieved. 72 |
    • 73 |
74 |
75 |
76 | 77 |
78 |
79 | ) 80 | } 81 | 82 | 83 | export default function HomepageFeatures() { 84 | return ( 85 |
86 |
87 |
88 |
Core Features
89 |
90 | 91 |
92 | 93 | 94 | 95 |
96 |
97 | ); 98 | } 99 | -------------------------------------------------------------------------------- /docs/tutorials/ccn-ipsec-tunnel.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: ipsec-network 3 | title: 'IPsec Cross-cluster Network' 4 | --- 5 | 6 | # IPsec Cross-cluster container network solution 7 | 8 | ## Using IPsec Tunnels for Cross-Cluster Container Network Communication over Public IP 9 | 10 | ### Introduction 11 | Kosmos is a multi-cluster solution, and networking is an essential part of it. 12 | Sometimes, there is a need for communication between Kubernetes clusters in different networks. 13 | In some cases, two or more clusters can only communicate with each other over the public internet. 14 | To address this, Kosmos has implemented a cross-cluster container network communication solution based on IPsec tunnels. 15 | 16 | ### Motivation 17 | For the sake of disaster recovery, application deployments may require communication across different clouds or across regions within a single cloud (across VPCs). 18 | In such scenarios, container communication becomes challenging as the internal IP addresses of the machines are usually not directly accessible without a dedicated network connection. 19 | Common CNI tunnel technologies like `VxLAN` or `IPIP` may not work effectively in public internet environments. 20 | To solve this problem, Kosmos has implemented a container network communication solution based on IPsec tunnels for cross-cloud communication over the public internet. 21 | This solution addresses the need for communication across public networks while also considering data transmission security. 22 | 23 | ### Goals 24 | The goal is to enable communication between pods in two clusters using elastic public IP addresses. The flow of traffic is illustrated in the diagram below: 25 | 26 | ![IPsec_Tunnel](img/IPsec_Tunnel.jpeg) 27 | 28 | :::info NOTE 29 | This solution does not address container network communication in host network mode within a cluster. 30 | Only focuses on IPv4 container network communication and does not cover IPv6 container networks. 31 | ::: 32 | 33 | ## Design Details 34 | 35 | ### API Changes 36 | 37 | #### Cluster API Changes 38 | This solution adds three fields to the `.spec.ClusterLinkOptions`: `NodeElasticIPMap`, `ClusterPodCIDRs`, and `UseExternalApiserver`. 39 | ````shell script 40 | type ClusterLinkOptions struct { 41 | ... 42 | // NodeElasticIPMap presents mapping between nodename in kubernetes and elasticIP 43 | // +optional 44 | NodeElasticIPMap map[string]string `json:"nodeElasticIPMap,omitempty"` 45 | // +optional 46 | ClusterPodCIDRs []string `json:"clusterpodCIDRs,omitempty"` 47 | // +optional 48 | UseExternalApiserver bool `json:"useexternalapiserver,omitempty"` 49 | } 50 | ```` 51 | - `NodeElasticIPMap` field represents the mapping relationship between the NodeName in Kubernetes and the elastic public IP mounted on the node. 52 | - `ClusterPodCIDRs` field is added to input the Pod CIDR, as it is not always easy to obtain the Pod CIDR for some CNI plugins. 53 | 54 | Typically, Kosmos retrieves the service CIDR through the kube-apiserver parameters. However, in some cases, the kube-apiserver is not a pod within the cluster. Therefore, the `UseExternalApiserver` field is added to handle this scenario. 55 | 56 | #### Clusternode API Changes 57 | This solution adds a new field, `ElasticIP`, to the `.spec`, and a new field, `NodeStatus`, to the `.status`. 58 | ````shell script 59 | type ClusterNodeSpec struct { 60 | ... 61 | // +optional 62 | ElasticIP string `json:"elasticip,omitempty"` 63 | } 64 | 65 | type ClusterNodeStatus struct { 66 | // +optional 67 | NodeStatus string `json:"nodeStatus,omitempty"` 68 | } 69 | ```` 70 | - `ElasticIP` field describes the elastic public IP mounted on the node. 71 | - `NodeStatus` field describes the status of the node, either "Ready" or "NotReady". 72 | 73 | #### Nodeconfig API Changes 74 | This solution adds two new fields, `XfrmPoliciesXfrmStates` and `IPsetsAvoidMasqs`, to the `.spec`. 75 | ````shell script 76 | type NodeConfigSpec struct { 77 | XfrmPolicies []XfrmPolicy `json:"xfrmpolicies,omitempty"` 78 | XfrmStates []XfrmState `json:"xfrmstates,omitempty"` 79 | IPsetsAvoidMasqs []IPset `json:"ipsetsavoidmasq,omitempty"` 80 | } 81 | 82 | type XfrmPolicy struct { 83 | LeftIP string `json:"leftip"` 84 | LeftNet string `json:"leftnet"` 85 | RightIP string `json:"rightip"` 86 | RightNet string `json:"rightnet"` 87 | ReqID int `json:"reqid"` 88 | Dir int `json:"dir"` 89 | } 90 | 91 | type XfrmState struct { 92 | LeftIP string `json:"leftip"` 93 | RightIP string `json:"rightip"` 94 | ReqID int `json:"reqid"` 95 | SPI uint32 `json:"spi"` 96 | PSK string `json:"PSK"` 97 | } 98 | 99 | type IPset struct { 100 | CIDR string `json:"cidr"` 101 | Name string `json:"name"` 102 | } 103 | ```` 104 | 105 | The new `XfrmPolicies` and `XfrmStates` fields define the IPsec-related rules created by Kosmos. 106 | 107 | `IPsetsAvoidMasqs` field describes the network segments that need to avoid masquerading, allowing outbound traffic from containers to retain their container IP address. 108 | 109 | ### Component Modifications 110 | 111 | #### Clusterlink-controller-manager 112 | Handling scenarios where kube-apiserver pods are not within the cluster: 113 | - The cluster-controller optimizes the retrieval of service CIDR through the `GetSvcByCreateInvalidSvc` function. 114 | 115 | Node status synchronization: 116 | - The node-controller synchronizes the `ElasticIP` field of the `clusternode` object based on the values in the `NodeElasticIPMap` field of the `cluster` object. 117 | - The node-controller now updates the `.Status.NodeStatus` field of the `clusternode` object based on the node's status, "Ready" or "NotReady". 118 | 119 | #### Clusterlink-elector 120 | The elector module is used for `Gateway` selection in Gateway mode. It can now select a `Gateway` from the nodes in "Ready" state. 121 | 122 | #### Clusterlink-network-manager 123 | 1. Adding support for some CNI plugins 124 | For some CNI plugins, iptables rules are added to avoid masquerading, allowing outbound traffic from containers to retain their container IP address. 125 | 126 | 2. Building IPsec rules 127 | Typically, Kosmos creates routes to achieve container communication. In IPsec tunnel mode, Kosmos creates `ip xfrm state` and `ip xfrm policy` rules if the `ElasticIP` field of the `clusternode` is not empty. 128 | 129 | #### clusterlink-agent 130 | Functions have been added to execute specific operations, equivalent to executing `ip xfrm state add/del` and `ip xfrm policy add/del` commands on the operating system. 131 | 132 | To avoid masquerading, functions have been added to execute `ipset` commands and create iptables rules. 133 | 134 | #### kosmosctl 135 | `NodeElasticIP`, `UseExternalApiserver`, and `ClusterPodCIDRs` input parameters have been added to populate the new fields `NodeElasticIPMap`, `UseExternalApiserver`, and `ClusterPodCIDRs` in the `Cluster` CRD. -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/ccn-ipsec-tunnel.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: ipsec-network 3 | title: 'IPsec Cross-cluster Network' 4 | --- 5 | 6 | # IPsec Cross-cluster container network solution 7 | 8 | ## Using IPsec Tunnels for Cross-Cluster Container Network Communication over Public IP 9 | 10 | ### Introduction 11 | Kosmos is a multi-cluster solution, and networking is an essential part of it. 12 | Sometimes, there is a need for communication between Kubernetes clusters in different networks. 13 | In some cases, two or more clusters can only communicate with each other over the public internet. 14 | To address this, Kosmos has implemented a cross-cluster container network communication solution based on IPsec tunnels. 15 | 16 | ### Motivation 17 | For the sake of disaster recovery, application deployments may require communication across different clouds or across regions within a single cloud (across VPCs). 18 | In such scenarios, container communication becomes challenging as the internal IP addresses of the machines are usually not directly accessible without a dedicated network connection. 19 | Common CNI tunnel technologies like `VxLAN` or `IPIP` may not work effectively in public internet environments. 20 | To solve this problem, Kosmos has implemented a container network communication solution based on IPsec tunnels for cross-cloud communication over the public internet. 21 | This solution addresses the need for communication across public networks while also considering data transmission security. 22 | 23 | ### Goals 24 | The goal is to enable communication between pods in two clusters using elastic public IP addresses. The flow of traffic is illustrated in the diagram below: 25 | 26 | ![IPsec_Tunnel](img/IPsec_Tunnel.jpeg) 27 | 28 | :::info NOTE 29 | This solution does not address container network communication in host network mode within a cluster. 30 | Only focuses on IPv4 container network communication and does not cover IPv6 container networks. 31 | ::: 32 | 33 | ## Design Details 34 | 35 | ### API Changes 36 | 37 | #### Cluster API Changes 38 | This solution adds three fields to the `.spec.ClusterLinkOptions`: `NodeElasticIPMap`, `ClusterPodCIDRs`, and `UseExternalApiserver`. 39 | ````shell script 40 | type ClusterLinkOptions struct { 41 | ... 42 | // NodeElasticIPMap presents mapping between nodename in kubernetes and elasticIP 43 | // +optional 44 | NodeElasticIPMap map[string]string `json:"nodeElasticIPMap,omitempty"` 45 | // +optional 46 | ClusterPodCIDRs []string `json:"clusterpodCIDRs,omitempty"` 47 | // +optional 48 | UseExternalApiserver bool `json:"useexternalapiserver,omitempty"` 49 | } 50 | ```` 51 | - `NodeElasticIPMap` field represents the mapping relationship between the NodeName in Kubernetes and the elastic public IP mounted on the node. 52 | - `ClusterPodCIDRs` field is added to input the Pod CIDR, as it is not always easy to obtain the Pod CIDR for some CNI plugins. 53 | 54 | Typically, Kosmos retrieves the service CIDR through the kube-apiserver parameters. However, in some cases, the kube-apiserver is not a pod within the cluster. Therefore, the `UseExternalApiserver` field is added to handle this scenario. 55 | 56 | #### Clusternode API Changes 57 | This solution adds a new field, `ElasticIP`, to the `.spec`, and a new field, `NodeStatus`, to the `.status`. 58 | ````shell script 59 | type ClusterNodeSpec struct { 60 | ... 61 | // +optional 62 | ElasticIP string `json:"elasticip,omitempty"` 63 | } 64 | 65 | type ClusterNodeStatus struct { 66 | // +optional 67 | NodeStatus string `json:"nodeStatus,omitempty"` 68 | } 69 | ```` 70 | - `ElasticIP` field describes the elastic public IP mounted on the node. 71 | - `NodeStatus` field describes the status of the node, either "Ready" or "NotReady". 72 | 73 | #### Nodeconfig API Changes 74 | This solution adds two new fields, `XfrmPoliciesXfrmStates` and `IPsetsAvoidMasqs`, to the `.spec`. 75 | ````shell script 76 | type NodeConfigSpec struct { 77 | XfrmPolicies []XfrmPolicy `json:"xfrmpolicies,omitempty"` 78 | XfrmStates []XfrmState `json:"xfrmstates,omitempty"` 79 | IPsetsAvoidMasqs []IPset `json:"ipsetsavoidmasq,omitempty"` 80 | } 81 | 82 | type XfrmPolicy struct { 83 | LeftIP string `json:"leftip"` 84 | LeftNet string `json:"leftnet"` 85 | RightIP string `json:"rightip"` 86 | RightNet string `json:"rightnet"` 87 | ReqID int `json:"reqid"` 88 | Dir int `json:"dir"` 89 | } 90 | 91 | type XfrmState struct { 92 | LeftIP string `json:"leftip"` 93 | RightIP string `json:"rightip"` 94 | ReqID int `json:"reqid"` 95 | SPI uint32 `json:"spi"` 96 | PSK string `json:"PSK"` 97 | } 98 | 99 | type IPset struct { 100 | CIDR string `json:"cidr"` 101 | Name string `json:"name"` 102 | } 103 | ```` 104 | 105 | The new `XfrmPolicies` and `XfrmStates` fields define the IPsec-related rules created by Kosmos. 106 | 107 | `IPsetsAvoidMasqs` field describes the network segments that need to avoid masquerading, allowing outbound traffic from containers to retain their container IP address. 108 | 109 | ### Component Modifications 110 | 111 | #### Clusterlink-controller-manager 112 | Handling scenarios where kube-apiserver pods are not within the cluster: 113 | - The cluster-controller optimizes the retrieval of service CIDR through the `GetSvcByCreateInvalidSvc` function. 114 | 115 | Node status synchronization: 116 | - The node-controller synchronizes the `ElasticIP` field of the `clusternode` object based on the values in the `NodeElasticIPMap` field of the `cluster` object. 117 | - The node-controller now updates the `.Status.NodeStatus` field of the `clusternode` object based on the node's status, "Ready" or "NotReady". 118 | 119 | #### Clusterlink-elector 120 | The elector module is used for `Gateway` selection in Gateway mode. It can now select a `Gateway` from the nodes in "Ready" state. 121 | 122 | #### Clusterlink-network-manager 123 | 1. Adding support for some CNI plugins 124 | For some CNI plugins, iptables rules are added to avoid masquerading, allowing outbound traffic from containers to retain their container IP address. 125 | 126 | 2. Building IPsec rules 127 | Typically, Kosmos creates routes to achieve container communication. In IPsec tunnel mode, Kosmos creates `ip xfrm state` and `ip xfrm policy` rules if the `ElasticIP` field of the `clusternode` is not empty. 128 | 129 | #### clusterlink-agent 130 | Functions have been added to execute specific operations, equivalent to executing `ip xfrm state add/del` and `ip xfrm policy add/del` commands on the operating system. 131 | 132 | To avoid masquerading, functions have been added to execute `ipset` commands and create iptables rules. 133 | 134 | #### kosmosctl 135 | `NodeElasticIP`, `UseExternalApiserver`, and `ClusterPodCIDRs` input parameters have been added to populate the new fields `NodeElasticIPMap`, `UseExternalApiserver`, and `ClusterPodCIDRs` in the `Cluster` CRD. -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/netdoctor.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: netdoctor 3 | title: 'Netdoctor' 4 | --- 5 | 6 | # Netdoctor 7 | 8 | ## 介绍 9 | 10 | * Kubernetes集群投入使用后集群网络可能会存在种种的连通性问题，因此我们希望可以有一个验收工具，在完成部署后检查集群的网络连通性是否正常。 11 | 12 | * 另一方面，Kosmos是一个跨集群的解决方案，在Kosmos管理多个集群之前，需要先检查各个集群的容器网络自身是否存在问题，部署完成后也需要验证跨集群的网络是否已经被Kosmos连通。 13 | 14 | * 出于以上两个方面，我们设计了 [NetDoctor](https://github.com/kosmos-io/netdoctor) 工具用于解决Kubernetes集群遇到的网络问题。 15 | 16 | ## 架构 17 | 18 | ![NetDoctor](img/netdr-arch.png) 19 | 20 | ## 先决条件 21 | 22 | * `go` 版本 v1.15+ 23 | * `kubernetes` 版本 v1.16+ 24 | 25 | ## 快速开始 26 | 27 | ### netctl工具 28 | * NetDoctor提供配套工具`netctl`，您可以方便的通过命令行去进行Kubernetes集群的网络连通性检查。 29 | #### 从制品库获取 30 | ````bash 31 | wget https://github.com/kosmos-io/netdoctor/releases/download/v0.0.1/netctl-linux-amd64 32 | mv netctl-linux-amd64 netctl 33 | ```` 34 | #### 从源码编译 35 | ````bash 36 | # 下载项目源码 37 | $ git clone https://github.com/kosmos-io/netdoctor.git 38 | # 执行后netctl会输出至./netdoctor/_output/bin/linux/amd64目录 39 | $ make netctl 40 | ```` 41 | ### netctl命令 42 | #### 初始化配置 43 | `netctl init`命令用于在当前目录生成网络检查需要的配置文件`config.json`，示例如下： 44 | + `namespace`: pod所在的命名空间 45 | + `version`: 镜像的版本 46 | + `mode`: 根据src到dst访问的ip设置，支持`pod`, `node`,`node-to-pod`,`pod-to-node`。例如，如果希望检验src的node到dst的pod的网络联通性，那么mode设置为`node-to-pod`。 47 | + `protocol`: 支持`tcp`, `udp`，`dns`。 48 | + `podWaitTime`: 等待pod running的时间 49 | + `port`: pod开放的端口 50 | + `maxNum`: 设置允许的最大并发数量 51 | + `cmdTimeout`: 命令执行的超时时间 52 | + `srcKubeConfig`: 源集群的kubeconfig 53 | + `srcImageRepository`: 源集群的镜像仓库 54 | 55 | 其他参数： 56 | + `dstKubeConfig`: 目标集群的kubeconfig 57 | + `dstImageRepository`: 目标集群的镜像仓库 58 | + `autoClean`: 检查结束后，是否自动清理测试pod 59 | + `targetDNSServer`: 目标DNS服务器 60 | + `targetHostToLookup`: 目标域名 61 | ````bash 62 | $ netctl init 63 | I1017 11:10:36.809902 7794 init.go:74] write opts success 64 | $ cat config.json 65 | { 66 | "namespace": "kosmos-system", 67 | "version": "0.3.0", 68 | "mode": "pod", 69 | "protocol": "tcp", 70 | "podWaitTime": 30, 71 | "port": "8889", 72 | "maxNum": 3, 73 | "cmdTimeout": 10, 74 | "srcKubeConfig": "~/.kube/config", 75 | "srcImageRepository": "ghcr.io/kosmos-io" 76 | } 77 | ```` 78 | #### 检查网络 79 | * `netctl check`命令会读取`config.json`，然后创建一个名为`Floater`的`DaemonSet`以及相关联的一些资源，之后会获取所有的`Floater`的`IP`信息，然后依次进入到`Pod`中执行`Ping`或者`Curl`命令。 80 | ````bash 81 | $ netctl check 82 | I0205 16:34:06.147671 2769373 check.go:61] use config from file!!!!!! 83 | I0205 16:34:06.148619 2769373 floater.go:73] create Clusterlink floater, namespace: kosmos-system 84 | I0205 16:34:06.157582 2769373 floater.go:83] create Clusterlink floater, apply RBAC 85 | I0205 16:34:06.167799 2769373 floater.go:94] create Clusterlink floater, version: v0.2.0 86 | I0205 16:34:09.178566 2769373 verify.go:79] pod: clusterlink-floater-9dzsg is ready. status: Running 87 | I0205 16:34:09.179593 2769373 verify.go:79] pod: clusterlink-floater-cscdh is ready. status: Running 88 | Do check... 100% [================================================================================] [0s] 89 | +-----+----------------+----------------+-----------+-----------+ 90 | | S/N | SRC NODE NAME | DST NODE NAME | TARGET IP | RESULT | 91 | +-----+----------------+----------------+-----------+-----------+ 92 | | 1 | ecs-net-dr-001 | ecs-net-dr-001 | 10.0.1.86 | SUCCEEDED | 93 | | 2 | ecs-net-dr-002 | ecs-net-dr-002 | 10.0.2.29 | SUCCEEDED | 94 | +-----+----------------+----------------+-----------+-----------+ 95 | 96 | +-----+----------------+----------------+-----------+-----------+-------------------------------+ 97 | | S/N | SRC NODE NAME | DST NODE NAME | TARGET IP | RESULT | LOG | 98 | +-----+----------------+----------------+-----------+-----------+-------------------------------+ 99 | | 1 | ecs-net-dr-002 | ecs-net-dr-001 | 10.0.1.86 | EXCEPTION |exec error: unable to upgrade | 100 | | 2 | ecs-net-dr-001 | ecs-net-dr-002 | 10.0.2.29 | EXCEPTION |connection: container not......| 101 | +-----+----------------+----------------+-----------+-----------+-------------------------------+ 102 | I0205 16:34:09.280220 2769373 do.go:93] write opts success 103 | ```` 104 | 105 | 需要注意的是： 106 | + 源ip地址：如果`mode`为`node`或`pod-to-node`时，则源ip地址为node.status.addresses里面的InternalIP地址。如果为`pod`或`node-to-pod`，则目标ip地址为pod.status.podIPs的地址 107 | + 目标ip地址：如果`mode`为`node`或`pod-to-node`时，则目标ip地址为node.status.addresses里面的InternalIP地址。如果为`pod`或`node-to-pod`，则目标ip地址为pod.status.podIPs的地址 108 | + 如果`pod`或者`node`的ip地址列表ipv4和ipv6地址都有，则两个地址都会被检验。 109 | + protocol不同模式执行的命令 110 | + `tcp`：curl -k http:\//srcIP:dstIP/ 111 | + `udp`：ping -c 1 %s; 112 | + `dns`：nslookup targetHost dnsServer; targetHost默认值为kubernetes.default.svc.cluster.local，dnsServer默认值为coredns。 113 | 114 | #### 检查结果查看 115 | * 在`check`命令执行的过程中，会有进度条显示校验进度。命令执行完成后，会打印失败结果，并将结果保存在文件`resume.json`中。 116 | ````bash 117 | [ 118 | { 119 | "Status": 0, 120 | "ResultStr": "exec error: unable to upgrade connection: container not found (\"floater\"), stderr: ", 121 | "srcNodeName": "ecs-sealos-001", 122 | "dstNodeName": "ecs-sealos-002", 123 | "targetIP": "10.0.2.29" 124 | }, 125 | { 126 | "Status": 0, 127 | "ResultStr": "exec error: command terminated with exit code 7, stderr % Total % Received % Xferd Average Speed Time Time Time Current\n Dload Upload Total Spent Left Speed\n\r 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0\r 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0\ncurl: (7) Failed to connect to 10.0.0.36 port 8889 after 0 ms: Couldn't connect to server\n", 128 | "srcNodeName": "ecs-sealos-002", 129 | "dstNodeName": "ecs-sealos-001", 130 | "targetIP": "10.0.0.36" 131 | } 132 | ] 133 | ```` 134 | #### 集群间检查 135 | * 如果需要检查Kosmos集群联邦中任意两个集群之间的网络连通性，则可以在配置文件`config.json`增加参数`dstKubeConfig`和`dstImageRepository`，这样就可以检查两个集群之间网络连通性了。 136 | ````bash 137 | $ vim config.json 138 | { 139 | "namespace": "kosmos-system", 140 | "version": "0.3.0", 141 | "mode": "pod", 142 | "protocol": "tcp", 143 | "podWaitTime": 30, 144 | "port": "8889", 145 | "maxNum": 3, 146 | "cmdTimeout": 10, 147 | "srcKubeConfig": "~/.kube/config", 148 | "srcImageRepository": "ghcr.io/kosmos-io" 149 | "dstKubeConfig": "~/.kube/dst-config", 150 | "dstImageRepository": "ghcr.io/kosmos-io" 151 | } 152 | ```` 153 | 154 | #### 重新检查 155 | * `netctl resume`命令用于复测时只检验第一次检查时有问题的集群节点。因为线上环境节点数量很多，单次检查可能会需要比较长的时间才能生成结果，所以我们希望仅对前一次检查异常的节点进行复测。`resume`命令因此被开发，该命令会读取`resume.json`文件，并对前一次异常的节点进行再次检查，我们可以重复执行此命令至没有异常的结果后再执行全量检查。 156 | ````bash 157 | $ netctl resume 158 | I0205 16:34:06.147671 2769373 check.go:61] use config from file!!!!!! 159 | I0205 16:34:06.148619 2769373 floater.go:73] create Clusterlink floater, namespace: kosmos-system 160 | I0205 16:34:06.157582 2769373 floater.go:83] create Clusterlink floater, apply RBAC 161 | I0205 16:34:06.167799 2769373 floater.go:94] create Clusterlink floater, version: v0.2.0 162 | I0205 16:34:09.178566 2769373 verify.go:79] pod: clusterlink-floater-9dzsg is ready. status: Running 163 | I0205 16:34:09.179593 2769373 verify.go:79] pod: clusterlink-floater-cscdh is ready. status: Running 164 | Do check... 100% [================================================================================] [0s] 165 | +-----+----------------+----------------+-----------+-----------+ 166 | | S/N | SRC NODE NAME | DST NODE NAME | TARGET IP | RESULT | 167 | +-----+----------------+----------------+-----------+-----------+ 168 | | 1 | ecs-net-dr-002 | ecs-net-dr-001 | 10.0.1.86 | SUCCEEDED | 169 | | 2 | ecs-net-dr-001 | ecs-net-dr-002 | 10.0.2.29 | SUCCEEDED | 170 | +-----+----------------+----------------+-----------+-----------+ 171 | ```` 172 | 173 | #### 清理 174 | * `netctl clean`命令用于清理`NetDoctor`创建的所有资源。 175 | -------------------------------------------------------------------------------- /docs/tutorials/multi-cluster-svc-discovery.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: mcs-discovery 3 | title: 'Multi-cluster Service Discovery' 4 | --- 5 | 6 | # Multi-cluster Service Discovery 7 | Kosmos provides two capabilities for multi-cluster service discovery: 8 | - Distributed multi-cluster service solution 9 | - Global centralized core-dns solution. Among them, in production environments, we recommend the decentralized distributed multi-cluster service solution. 10 | 11 | ## Distributed Multi-Cluster Service Solution 12 | 13 | ### Introduction 14 | Kosmos implements multi-cluster service discovery based on the Multi-Cluster Service API. Users can export services generated by the control plane cluster to member clusters, thereby providing services externally in member clusters. 15 | 16 | :::info NOTE 17 | To use this feature, ensure that the control plane cluster and member clusters have a version of at least v1.21, and the version of core-dns in the cluster is not less than v1.84. 18 | ::: 19 | 20 | The architecture of Kosmos' distributed multi-cluster service solution is as follows. 21 | 22 | ![MCS Architecture.svg](img/MCS_Architecture.svg) 23 | 24 | Kosmos' controller monitors the ServiceExport and ServiceImport resources of the control plane and data plane, and synchronizes the Service and EndpointSlice from the control plane to the data plane cluster based on the configuration of these two MCS resources. In any data plane cluster, services can be exposed externally through Service, and each data plane cluster's EndpointSlice will contain all Pod IPs of the workloads in the control plane cluster, achieving cross-cluster multi-active services across different locations. 25 | 26 | ### Prerequisites 27 | 28 | #### Install Kosmos 29 | Refer to the Kosmos Quick Start documentation https://github.com/kosmos-io/kosmos and enable the ClusterLink module for multi-cluster networking. Using the kosmosctl tool: 30 | ````shell script 31 | kosmosctl install --cni calico --default-nic eth0 (We build a network tunnel based on the network interface value passed by the arg default-nic) 32 | ```` 33 | 34 | #### Join the Leaf Cluster 35 | ````shell script 36 | kosmosctl join cluster --name cluster1 --kubeconfig ~/kubeconfig/cluster1-kubeconfig --cni calico --default-nic eth0 --enable-link 37 | ```` 38 | 39 | ### Using Distributed Multi-Cluster Service 40 | 41 | #### Manual Exporting and Importing of Service 42 | Users can manually create ServiceExport and ServiceImport to distribute services to data plane clusters. 43 | 44 | 1. Create a nginx workload in the data plane cluster, the nginx yaml is as follows: 45 | ````shell script 46 | apiVersion: apps/v1 47 | kind: Deployment 48 | metadata: 49 | name: nginx 50 | spec: 51 | replicas: 2 52 | selector: 53 | matchLabels: 54 | app: nginx 55 | template: 56 | metadata: 57 | labels: 58 | app: nginx 59 | spec: 60 | tolerations: 61 | - key: "kosmos.io/node" 62 | operator: "Equal" 63 | value: "true" 64 | effect: "NoSchedule" 65 | podAntiAffinity: 66 | requiredDuringSchedulingIgnoredDuringExecution: 67 | - labelSelector: 68 | matchLabels: 69 | app: nginx 70 | topologyKey: kubernetes.io/hostname 71 | containers: 72 | - name: nginx 73 | image: nginx:latest 74 | ports: 75 | - containerPort: 80 76 | --- 77 | apiVersion: v1 78 | kind: Service 79 | metadata: 80 | name: nginx 81 | spec: 82 | selector: 83 | app: nginx 84 | ports: 85 | - protocol: TCP 86 | port: 80 87 | targetPort: 80 88 | nodePort: 31444 89 | type: NodePort 90 | ```` 91 | :::info NOTE 92 | The workload's pods can be scheduled to Kosmos' leaf nodes by specifying node affinity in the yaml. 93 | ::: 94 | 95 | 2. Create a ServiceExport in the control plane to export the nginx Service: 96 | ````shell script 97 | apiVersion: multicluster.x-k8s.io/v1alpha1 98 | kind: ServiceExport 99 | metadata: 100 | name: nginx 101 | ```` 102 | 103 | 3. Create a ServiceImport in data plane cluster 1 to import the exported Service: 104 | ````shell script 105 | apiVersion: multicluster.x-k8s.io/v1alpha1 106 | kind: ServiceImport 107 | metadata: 108 | name: nginx 109 | spec: 110 | type: ClusterSetIP 111 | ports: 112 | - protocol: TCP 113 | port: 80 114 | ```` 115 | At this point, the service can be exposed externally in data plane cluster 1 using the same Service. 116 | 117 | ### Automatic Exporting and Importing of Service to all Data Plane Clusters 118 | In some scenarios, users need to automatically synchronize the control plane's Service to all data plane clusters without manually creating ServiceExport and ServiceImport. Kosmos provides two methods for automatic export and import of Service. 119 | 120 | #### Annotation-based Recognition via Control Plane Service 121 | For services that need to be automatically exported and distributed to all data plane clusters, users can manually add an annotation to the Service: `kosmos.io/auto-create-mcs: "true"`. An example service yaml is as follows: 122 | ````shell script 123 | apiVersion: v1 124 | kind: Service 125 | metadata: 126 | name: nginx 127 | annotations: 128 | kosmos.io/auto-create-mcs: "true" 129 | spec: 130 | selector: 131 | app: nginx 132 | ports: 133 | - protocol: TCP 134 | port: 80 135 | targetPort: 80 136 | nodePort: 31444 137 | type: NodePort 138 | ```` 139 | 140 | #### Global Startup Parameters 141 | For cases where code modification is not desired, Kosmos also supports automatic distribution of services by configuring startup parameters. Add the parameter `--auto-mcs-prefix` to the ClusterTree startup service, for example, configuring `--auto-mcs-prefix=test`, kosmos will automatically export services under namespaces with the test and kosmos prefixes to all data plane clusters. 142 | 143 | ### Service Cross-Cluster Network Connectivity Verification 144 | `eps-probe-plugin` is a plugin for cross-cluster service network verification. As mentioned above, Kosmos' controller synchronizes the Service and EndpointSlice from the control cluster to the data plane cluster. When there is a network connectivity failure across clusters, the endpoints in the EndpointSlice become unreachable, leading to malfunction. 145 | 146 | `eps-probe-plugin` checks whether the pod-ips in the endpoint are reachable on a regular basis. It updates the addresses of unreachable endpoints to `"kosmos.io/disconnected-address"` in serviceImport. The Kosmos controller will remove the unreachable endpoints from the exported Service's corresponding EndpointSlice. 147 | 148 | `eps-probe-plugin` can be installed using the following command: 149 | ````shell script 150 | kubectl apply -f https://raw.githubusercontent.com/kosmos-io/eps-probe-plugin/main/deploy/eps-probe-plugin.yaml 151 | ```` 152 | 153 | ## Global Centralized Core-DNS Solution 154 | 155 | ### Introduction 156 | The global centralized core-dns solution is shown in the diagram below. All pods distributed through Kosmos are resolved through the core-dns of the control plane cluster. This approach is only suitable for testing environments and may cause excessive request load on the core-dns of the control plane cluster in production environments. 157 | 158 | To enable the global centralized core-dns solution, simply modify the startup parameter of the clusterTree service to `--multi-cluster-service=false`. 159 | 160 | ![CoreDNS_Centralized_Architecture.svg](img/CoreDNS_Centralized_Architecture.svg) -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/multi-cluster-svc-discovery.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: mcs-discovery 3 | title: 'Multi-cluster Service Discovery' 4 | --- 5 | 6 | # Multi-cluster Service Discovery 7 | Kosmos provides two capabilities for multi-cluster service discovery: 8 | - Distributed multi-cluster service solution 9 | - Global centralized core-dns solution. Among them, in production environments, we recommend the decentralized distributed multi-cluster service solution. 10 | 11 | ## Distributed Multi-Cluster Service Solution 12 | 13 | ### Introduction 14 | Kosmos implements multi-cluster service discovery based on the Multi-Cluster Service API. Users can export services generated by the control plane cluster to member clusters, thereby providing services externally in member clusters. 15 | 16 | :::info NOTE 17 | To use this feature, ensure that the control plane cluster and member clusters have a version of at least v1.21, and the version of core-dns in the cluster is not less than v1.84. 18 | ::: 19 | 20 | The architecture of Kosmos' distributed multi-cluster service solution is as follows. 21 | 22 | ![MCS Architecture.svg](img/MCS_Architecture.svg) 23 | 24 | Kosmos' controller monitors the ServiceExport and ServiceImport resources of the control plane and data plane, and synchronizes the Service and EndpointSlice from the control plane to the data plane cluster based on the configuration of these two MCS resources. In any data plane cluster, services can be exposed externally through Service, and each data plane cluster's EndpointSlice will contain all Pod IPs of the workloads in the control plane cluster, achieving cross-cluster multi-active services across different locations. 25 | 26 | ### Prerequisites 27 | 28 | #### Install Kosmos 29 | Refer to the Kosmos Quick Start documentation https://github.com/kosmos-io/kosmos and enable the ClusterLink module for multi-cluster networking. Using the kosmosctl tool: 30 | ````shell script 31 | kosmosctl install --cni calico --default-nic eth0 (We build a network tunnel based on the network interface value passed by the arg default-nic) 32 | ```` 33 | 34 | #### Join the Leaf Cluster 35 | ````shell script 36 | kosmosctl join cluster --name cluster1 --kubeconfig ~/kubeconfig/cluster1-kubeconfig --cni calico --default-nic eth0 --enable-link 37 | ```` 38 | 39 | ### Using Distributed Multi-Cluster Service 40 | 41 | #### Manual Exporting and Importing of Service 42 | Users can manually create ServiceExport and ServiceImport to distribute services to data plane clusters. 43 | 44 | 1. Create a nginx workload in the data plane cluster, the nginx yaml is as follows: 45 | ````shell script 46 | apiVersion: apps/v1 47 | kind: Deployment 48 | metadata: 49 | name: nginx 50 | spec: 51 | replicas: 2 52 | selector: 53 | matchLabels: 54 | app: nginx 55 | template: 56 | metadata: 57 | labels: 58 | app: nginx 59 | spec: 60 | tolerations: 61 | - key: "kosmos.io/node" 62 | operator: "Equal" 63 | value: "true" 64 | effect: "NoSchedule" 65 | podAntiAffinity: 66 | requiredDuringSchedulingIgnoredDuringExecution: 67 | - labelSelector: 68 | matchLabels: 69 | app: nginx 70 | topologyKey: kubernetes.io/hostname 71 | containers: 72 | - name: nginx 73 | image: nginx:latest 74 | ports: 75 | - containerPort: 80 76 | --- 77 | apiVersion: v1 78 | kind: Service 79 | metadata: 80 | name: nginx 81 | spec: 82 | selector: 83 | app: nginx 84 | ports: 85 | - protocol: TCP 86 | port: 80 87 | targetPort: 80 88 | nodePort: 31444 89 | type: NodePort 90 | ```` 91 | :::info NOTE 92 | The workload's pods can be scheduled to Kosmos' leaf nodes by specifying node affinity in the yaml. 93 | ::: 94 | 95 | 2. Create a ServiceExport in the control plane to export the nginx Service: 96 | ````shell script 97 | apiVersion: multicluster.x-k8s.io/v1alpha1 98 | kind: ServiceExport 99 | metadata: 100 | name: nginx 101 | ```` 102 | 103 | 3. Create a ServiceImport in data plane cluster 1 to import the exported Service: 104 | ````shell script 105 | apiVersion: multicluster.x-k8s.io/v1alpha1 106 | kind: ServiceImport 107 | metadata: 108 | name: nginx 109 | spec: 110 | type: ClusterSetIP 111 | ports: 112 | - protocol: TCP 113 | port: 80 114 | ```` 115 | At this point, the service can be exposed externally in data plane cluster 1 using the same Service. 116 | 117 | ### Automatic Exporting and Importing of Service to all Data Plane Clusters 118 | In some scenarios, users need to automatically synchronize the control plane's Service to all data plane clusters without manually creating ServiceExport and ServiceImport. Kosmos provides two methods for automatic export and import of Service. 119 | 120 | #### Annotation-based Recognition via Control Plane Service 121 | For services that need to be automatically exported and distributed to all data plane clusters, users can manually add an annotation to the Service: `kosmos.io/auto-create-mcs: "true"`. An example service yaml is as follows: 122 | ````shell script 123 | apiVersion: v1 124 | kind: Service 125 | metadata: 126 | name: nginx 127 | annotations: 128 | kosmos.io/auto-create-mcs: "true" 129 | spec: 130 | selector: 131 | app: nginx 132 | ports: 133 | - protocol: TCP 134 | port: 80 135 | targetPort: 80 136 | nodePort: 31444 137 | type: NodePort 138 | ```` 139 | 140 | #### Global Startup Parameters 141 | For cases where code modification is not desired, Kosmos also supports automatic distribution of services by configuring startup parameters. Add the parameter `--auto-mcs-prefix` to the ClusterTree startup service, for example, configuring `--auto-mcs-prefix=test`, kosmos will automatically export services under namespaces with the test and kosmos prefixes to all data plane clusters. 142 | 143 | ### Service Cross-Cluster Network Connectivity Verification 144 | `eps-probe-plugin` is a plugin for cross-cluster service network verification. As mentioned above, Kosmos' controller synchronizes the Service and EndpointSlice from the control cluster to the data plane cluster. When there is a network connectivity failure across clusters, the endpoints in the EndpointSlice become unreachable, leading to malfunction. 145 | 146 | `eps-probe-plugin` checks whether the pod-ips in the endpoint are reachable on a regular basis. It updates the addresses of unreachable endpoints to `"kosmos.io/disconnected-address"` in serviceImport. The Kosmos controller will remove the unreachable endpoints from the exported Service's corresponding EndpointSlice. 147 | 148 | `eps-probe-plugin` can be installed using the following command: 149 | ````shell script 150 | kubectl apply -f https://raw.githubusercontent.com/kosmos-io/eps-probe-plugin/main/deploy/eps-probe-plugin.yaml 151 | ```` 152 | 153 | ## Global Centralized Core-DNS Solution 154 | 155 | ### Introduction 156 | The global centralized core-dns solution is shown in the diagram below. All pods distributed through Kosmos are resolved through the core-dns of the control plane cluster. This approach is only suitable for testing environments and may cause excessive request load on the core-dns of the control plane cluster in production environments. 157 | 158 | To enable the global centralized core-dns solution, simply modify the startup parameter of the clusterTree service to `--multi-cluster-service=false`. 159 | 160 | ![CoreDNS_Centralized_Architecture.svg](img/CoreDNS_Centralized_Architecture.svg) -------------------------------------------------------------------------------- /docs/proposals/distribution-policy.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: distribution-policy 3 | title: 'Distribution Policy' 4 | --- 5 | 6 | # Distribution Policy 7 | 8 | ## Summary 9 | Kosmos provides a multi-cluster scheduler (kosmos-scheduler) and some scheduling plugins. 10 | 11 | ## Motivation & User Stories 12 | 1. `distributionpolicy`: During the product migration process, we hope that the pods created by the user operator can be scheduled to the member cluster without modifying the user's code. Therefore, we provide the distributionpolicy plugin to schedule to the desired nodes (host node, leaf node, mix node) according to matching rules (namespace, labelselector, name and other fields) 13 | 14 | ## Design Details 15 | 16 | ### DistributionPolicy & ClusterDistributionPolicy 17 | 18 | #### CRD API 19 | [Code](https://github.com/kosmos-io/kosmos/pull/321) responsible for working with DistributionPolicy and ClusterDistributionPolicy CRD API will be imported in the kosmos-scheduler plugins repo. DistributionPolicy is namespace scope and ClusterDistributionPolicy is cluster scope. 20 | 21 | #### Filter extension points implementation details 22 | Since target resources distribution policies are stored in the CRD (DistributionPolicy & ClusterDistributionPolicy), kosmos-scheduler should be subscribed for updates of appropriate CRD type. kosmos-scheduler will use informers which will be generated with the name dpInformer(cdpInformer). CRD will contain in ResourceSelectors and PolicyTerms. ResourceSelectors used to select resources and is required. PolicyTerms represents the rule for select nodes to distribute resources. 23 | 24 | **Description of the ResourceSelectors rules** 25 | 1. ResourceSelectors is required 26 | 2. Prioritize matching of high-precision rule 27 | In case of two ResourceSelector, the one with a more precise 28 | matching rules in ResourceSelectors wins: 29 | - Namespace-scope resources have higher priority than cluster-scope resources 30 | - For namespace-scope resources, matching by name (resourceSelector.name) has higher priority than by namePrefix (resourceSelector.namePrefix) 31 | - For namespace-scope resources, matching by namePrefix (resourceSelector.namePrefix) has higher priority than by selector(resourceSelector.labelSelector) 32 | - For cluster-scope resources, matching by name (resourceSelector.name) has higher priority than by namePrefix (resourceSelector.namePrefix) 33 | - For cluster-scope resources, matching by NamePrefix (resourceSelector.namePrefix) has higher priority than by selector(resourceSelector.labelSelector) 34 | The more the precise, the higher the priority. Defaults to zero which means schedule to the mix node. 35 | 36 | **PolicyTerms** 37 | 1. PolicyTerms is required 38 | 2. The current node scheduling policy is divided into four nodeTypes （host, leaf, mix, adv）. 39 | 3. Advanced options will be supported in the future. Sure as NodeSelector, Affinity and so on. 40 | 41 | ## Use cases 42 | 43 | ### DistributionPolicy CRD 44 | ````shell script 45 | apiVersion: kosmos.io/v1alpha1 46 | kind: DistributionPolicy 47 | metadata: 48 | name: kosmos-node-distribution-policy 49 | namespace: test-name-scope 50 | spec: 51 | resourceSelectors: 52 | - name: busybox 53 | policyName: leaf 54 | - namePrefix: busybox-prefix 55 | policyName: host 56 | - labelSelector: 57 | matchLabels: 58 | example-distribution-policy: busybox 59 | policyName: mix 60 | policyTerms: 61 | - name: host 62 | nodeType: host 63 | - name: leaf 64 | nodeType: leaf 65 | - name: mix 66 | nodeType: mix 67 | - name: adv 68 | nodeType: adv 69 | advancedTerm: 70 | nodeSelector: 71 | advNode: "true" 72 | ```` 73 | 74 | ### ClusterDistributionPolicy CRD 75 | ````shell script 76 | apiVersion: kosmos.io/v1alpha1 77 | kind: ClusterDistributionPolicy 78 | metadata: 79 | name: kosmos-node-cluster-distribution-policy 80 | spec: 81 | resourceSelectors: 82 | - name: cluster-busybox 83 | policyName: leaf 84 | - namePrefix: cluster-busybox-prefix 85 | policyName: host 86 | - labelSelector: 87 | matchLabels: 88 | example-distribution-policy: cluster-busybox 89 | policyName: mix 90 | policyTerms: 91 | - name: host 92 | nodeType: host 93 | - name: leaf 94 | nodeType: leaf 95 | - name: mix 96 | nodeType: mix 97 | - name: adv 98 | nodeType: adv 99 | advancedTerm: 100 | nodeName: kosmos-member2-cluster-1 101 | ```` 102 | 103 | ## Test plans 104 | 105 | ### Preparatory Work 106 | First, Kosmos needs to be [deployed successfully](https://kosmos-io.github.io/website/v0.2.0/quick-start) (at least the clustertree module is deployed) and join the member cluster correctly. 107 | 108 | ### Deploy the Kosmos-scheduler 109 | 1. Configure scheduler and scheduling policy 110 | ````shell script 111 | kubectl apply -f kosmos/deploy/scheduler/. 112 | ```` 113 | 2. Verify the kosmos-scheduler service 114 | ````shell script 115 | kubectl -n kosmos-system get pod 116 | NAME READY STATUS RESTARTS AGE 117 | kosmos-scheduler-8f96d87d7-ssxrx 1/1 Running 0 24s 118 | ```` 119 | 120 | ### Try a Sample 121 | 1. Use case yaml（busybox.yaml） 122 | ````shell script 123 | apiVersion: apps/v1 124 | kind: Deployment 125 | metadata: 126 | name: busybox 127 | namespace: test-name-scope 128 | spec: 129 | replicas: 3 130 | selector: 131 | matchLabels: 132 | app: busybox 133 | template: 134 | metadata: 135 | labels: 136 | app: busybox 137 | example-distribution-policy: busybox 138 | spec: 139 | affinity: 140 | podAntiAffinity: 141 | requiredDuringSchedulingIgnoredDuringExecution: 142 | - labelSelector: 143 | matchLabels: 144 | app: busybox 145 | topologyKey: kubernetes.io/hostname 146 | schedulerName: kosmos-scheduler 147 | containers: 148 | - name: busybox 149 | image: busybox:latest 150 | imagePullPolicy: IfNotPresent 151 | command: 152 | - /bin/sh 153 | - -c 154 | - sleep 3000 155 | ```` 156 | 2. Other Operation instructions 157 | ````shell script 158 | # List all nodes in the host cluster 159 | kubectl get node 160 | NAME STATUS ROLES AGE VERSION 161 | kosmoscluster1-1 Ready control-plane,master,node 21d v1.21.5-eki.0 162 | kosmoscluster1-2 Ready node 21d v1.21.5-eki.0 163 | kosmos-member2-cluster-1 Ready agent 24m v1.21.5-eki.0 164 | kosmos-member2-cluster-2 Ready agent 24m v1.21.5-eki.0 165 | 166 | # Show the taint information on the virtual node 167 | kubectl describe node kosmos-member2-cluster-1 |grep Tai 168 | Taints: node.kubernetes.io/unreachable:NoExecute 169 | 170 | kubectl describe node kosmos-member2-cluster-2 |grep Tai 171 | Taints: node.kubernetes.io/unreachable:NoExecute 172 | 173 | # Scheduling by the kosmos-scheduler (hybrid scheduling) 174 | kubectl apply -f busybox.yaml 175 | 176 | # Show instances (hybrid) scheduling result in host cluster 177 | kubectl get pod -owide -n test-name-scope 178 | NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES 179 | busybox-69855845c9-2pl7f 1/1 Running 0 14s 10.xx.xx.12 kosmoscluster1-1 180 | busybox-69855845c9-54cm9 1/1 Running 0 14s 10.xx.xx.92 kosmoscluster1-2 181 | busybox-69855845c9-9gjs9 1/1 Running 0 14s 10.xx.xx.80 kosmos-member2-cluster-1 182 | 183 | # Show instances (hybrid) scheduling result in member cluster 184 | kubectl get pod -owide -n test-name-scope 185 | NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES 186 | busybox-69855845c9-9gjs9 1/1 Running 0 14s 10.xx.xx.80 kosmos-member2-cluster-1 187 | ```` 188 | -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/proposals/distribution-policy.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: distribution-policy 3 | title: 'Distribution Policy' 4 | --- 5 | 6 | # Distribution Policy 7 | 8 | ## Summary 9 | Kosmos provides a multi-cluster scheduler (kosmos-scheduler) and some scheduling plugins. 10 | 11 | ## Motivation & User Stories 12 | 1. `distributionpolicy`: During the product migration process, we hope that the pods created by the user operator can be scheduled to the member cluster without modifying the user's code. Therefore, we provide the distributionpolicy plugin to schedule to the desired nodes (host node, leaf node, mix node) according to matching rules (namespace, labelselector, name and other fields) 13 | 14 | ## Design Details 15 | 16 | ### DistributionPolicy & ClusterDistributionPolicy 17 | 18 | #### CRD API 19 | [Code](https://github.com/kosmos-io/kosmos/pull/321) responsible for working with DistributionPolicy and ClusterDistributionPolicy CRD API will be imported in the kosmos-scheduler plugins repo. DistributionPolicy is namespace scope and ClusterDistributionPolicy is cluster scope. 20 | 21 | #### Filter extension points implementation details 22 | Since target resources distribution policies are stored in the CRD (DistributionPolicy & ClusterDistributionPolicy), kosmos-scheduler should be subscribed for updates of appropriate CRD type. kosmos-scheduler will use informers which will be generated with the name dpInformer(cdpInformer). CRD will contain in ResourceSelectors and PolicyTerms. ResourceSelectors used to select resources and is required. PolicyTerms represents the rule for select nodes to distribute resources. 23 | 24 | **Description of the ResourceSelectors rules** 25 | 1. ResourceSelectors is required 26 | 2. Prioritize matching of high-precision rule 27 | In case of two ResourceSelector, the one with a more precise 28 | matching rules in ResourceSelectors wins: 29 | - Namespace-scope resources have higher priority than cluster-scope resources 30 | - For namespace-scope resources, matching by name (resourceSelector.name) has higher priority than by namePrefix (resourceSelector.namePrefix) 31 | - For namespace-scope resources, matching by namePrefix (resourceSelector.namePrefix) has higher priority than by selector(resourceSelector.labelSelector) 32 | - For cluster-scope resources, matching by name (resourceSelector.name) has higher priority than by namePrefix (resourceSelector.namePrefix) 33 | - For cluster-scope resources, matching by NamePrefix (resourceSelector.namePrefix) has higher priority than by selector(resourceSelector.labelSelector) 34 | The more the precise, the higher the priority. Defaults to zero which means schedule to the mix node. 35 | 36 | **PolicyTerms** 37 | 1. PolicyTerms is required 38 | 2. The current node scheduling policy is divided into four nodeTypes （host, leaf, mix, adv）. 39 | 3. Advanced options will be supported in the future. Sure as NodeSelector, Affinity and so on. 40 | 41 | ## Use cases 42 | 43 | ### DistributionPolicy CRD 44 | ````shell script 45 | apiVersion: kosmos.io/v1alpha1 46 | kind: DistributionPolicy 47 | metadata: 48 | name: kosmos-node-distribution-policy 49 | namespace: test-name-scope 50 | spec: 51 | resourceSelectors: 52 | - name: busybox 53 | policyName: leaf 54 | - namePrefix: busybox-prefix 55 | policyName: host 56 | - labelSelector: 57 | matchLabels: 58 | example-distribution-policy: busybox 59 | policyName: mix 60 | policyTerms: 61 | - name: host 62 | nodeType: host 63 | - name: leaf 64 | nodeType: leaf 65 | - name: mix 66 | nodeType: mix 67 | - name: adv 68 | nodeType: adv 69 | advancedTerm: 70 | nodeSelector: 71 | advNode: "true" 72 | ```` 73 | 74 | ### ClusterDistributionPolicy CRD 75 | ````shell script 76 | apiVersion: kosmos.io/v1alpha1 77 | kind: ClusterDistributionPolicy 78 | metadata: 79 | name: kosmos-node-cluster-distribution-policy 80 | spec: 81 | resourceSelectors: 82 | - name: cluster-busybox 83 | policyName: leaf 84 | - namePrefix: cluster-busybox-prefix 85 | policyName: host 86 | - labelSelector: 87 | matchLabels: 88 | example-distribution-policy: cluster-busybox 89 | policyName: mix 90 | policyTerms: 91 | - name: host 92 | nodeType: host 93 | - name: leaf 94 | nodeType: leaf 95 | - name: mix 96 | nodeType: mix 97 | - name: adv 98 | nodeType: adv 99 | advancedTerm: 100 | nodeName: kosmos-member2-cluster-1 101 | ```` 102 | 103 | ## Test plans 104 | 105 | ### Preparatory Work 106 | First, Kosmos needs to be [deployed successfully](https://kosmos-io.github.io/website/v0.2.0/quick-start) (at least the clustertree module is deployed) and join the member cluster correctly. 107 | 108 | ### Deploy the Kosmos-scheduler 109 | 1. Configure scheduler and scheduling policy 110 | ````shell script 111 | kubectl apply -f kosmos/deploy/scheduler/. 112 | ```` 113 | 2. Verify the kosmos-scheduler service 114 | ````shell script 115 | kubectl -n kosmos-system get pod 116 | NAME READY STATUS RESTARTS AGE 117 | kosmos-scheduler-8f96d87d7-ssxrx 1/1 Running 0 24s 118 | ```` 119 | 120 | ### Try a Sample 121 | 1. Use case yaml（busybox.yaml） 122 | ````shell script 123 | apiVersion: apps/v1 124 | kind: Deployment 125 | metadata: 126 | name: busybox 127 | namespace: test-name-scope 128 | spec: 129 | replicas: 3 130 | selector: 131 | matchLabels: 132 | app: busybox 133 | template: 134 | metadata: 135 | labels: 136 | app: busybox 137 | example-distribution-policy: busybox 138 | spec: 139 | affinity: 140 | podAntiAffinity: 141 | requiredDuringSchedulingIgnoredDuringExecution: 142 | - labelSelector: 143 | matchLabels: 144 | app: busybox 145 | topologyKey: kubernetes.io/hostname 146 | schedulerName: kosmos-scheduler 147 | containers: 148 | - name: busybox 149 | image: busybox:latest 150 | imagePullPolicy: IfNotPresent 151 | command: 152 | - /bin/sh 153 | - -c 154 | - sleep 3000 155 | ```` 156 | 2. Other Operation instructions 157 | ````shell script 158 | # List all nodes in the host cluster 159 | kubectl get node 160 | NAME STATUS ROLES AGE VERSION 161 | kosmoscluster1-1 Ready control-plane,master,node 21d v1.21.5-eki.0 162 | kosmoscluster1-2 Ready node 21d v1.21.5-eki.0 163 | kosmos-member2-cluster-1 Ready agent 24m v1.21.5-eki.0 164 | kosmos-member2-cluster-2 Ready agent 24m v1.21.5-eki.0 165 | 166 | # Show the taint information on the virtual node 167 | kubectl describe node kosmos-member2-cluster-1 |grep Tai 168 | Taints: node.kubernetes.io/unreachable:NoExecute 169 | 170 | kubectl describe node kosmos-member2-cluster-2 |grep Tai 171 | Taints: node.kubernetes.io/unreachable:NoExecute 172 | 173 | # Scheduling by the kosmos-scheduler (hybrid scheduling) 174 | kubectl apply -f busybox.yaml 175 | 176 | # Show instances (hybrid) scheduling result in host cluster 177 | kubectl get pod -owide -n test-name-scope 178 | NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES 179 | busybox-69855845c9-2pl7f 1/1 Running 0 14s 10.xx.xx.12 kosmoscluster1-1 180 | busybox-69855845c9-54cm9 1/1 Running 0 14s 10.xx.xx.92 kosmoscluster1-2 181 | busybox-69855845c9-9gjs9 1/1 Running 0 14s 10.xx.xx.80 kosmos-member2-cluster-1 182 | 183 | # Show instances (hybrid) scheduling result in member cluster 184 | kubectl get pod -owide -n test-name-scope 185 | NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES 186 | busybox-69855845c9-9gjs9 1/1 Running 0 14s 10.xx.xx.80 kosmos-member2-cluster-1 187 | ```` 188 | -------------------------------------------------------------------------------- /docs/tutorials/application-migration.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: application-migration 3 | title: 'Application Migration' 4 | --- 5 | 6 | # Application Migration 7 | Kosmos provides application migration functionality to help users migrate existing applications from subclusters to the Kosmos control plane cluster. 8 | 9 | ## Application Migration Solution 10 | 11 | ### Introduction 12 | In the Kosmos multi-cluster design architecture, users can interact with the kube-apiserver of the control plane cluster using tools such as kubectl and client-go to create deployments or statefulset applications, etc. 13 | The actual Pod instances are running in the subclusters. 14 | 15 | However, for existing applications in the subclusters that were not created through the Kosmos control plane, these applications cannot be viewed and managed in the control plane cluster. 16 | 17 | Kosmos provides application migration functionality to support the migration of applications in a namespace to the control plane cluster. 18 | The entire process does not require restarting the application Pod instances, ensuring minimal impact on business operations. 19 | 20 | ### Design Details 21 | Application migration mainly consists of three processes: application backup -> deletion of owner objects -> rebuilding applications in the control plane. 22 | 23 | #### Application Backup 24 | Kosmos first backs up all namespaced-level resources in the target namespace, as well as the dependent cluster-level resources, such as cluster roles, cluster role bindings, persistent volumes, etc. 25 | The backup files are stored in PVCs in Kosmos. 26 | 27 | #### Deletion of Owner Objects 28 | The subclusters in Kosmos only run Pods, and their owner StatefulSets or ReplicaSets need to be deleted and rebuilt in the Kosmos control plane. 29 | Similarly, the owner Deployment of a ReplicaSet, as well as the owner StatefulSet and Deployment, need to be deleted and rebuilt in the Kosmos control plane. 30 | 31 | By using a top-down cascading deletion of owner objects (e.g., deleting the Deployment first and then the ReplicaSet), the Pods are not affected and remain in a running state. 32 | 33 | #### Rebuilding Applications in the Control Plane 34 | Based on the backup resources, the control plane cluster creates all the migrated resources, including namespaces, pods, deployments, config maps, service accounts, etc. 35 | In order to maintain consistency with the subcluster Pods and keep them in a running state, the applications are rebuilt using a bottom-up approach (e.g., creating the Pod first and then the ReplicaSet). 36 | 37 | #### CRD API 38 | The PromotePolicy CRD API is provided for configuring the migration policy. 39 | PromotePolicy is a cluster-wide CRD API. Here is an example of how to use it: 40 | ````shell script 41 | apiVersion: kosmos.io/v1alpha1 42 | kind: PromotePolicy 43 | metadata: 44 | name: promote-policy-sample 45 | spec: 46 | includedNamespaces: 47 | - namespace1 48 | - namespace2 49 | excludedNamespaceScopedResources: 50 | - events 51 | - events.events.k8s.io 52 | - endpoints 53 | - endpointslices.discovery.k8s.io 54 | clusterName: member-cluster1 55 | ```` 56 | where: 57 | - includedNamespaces: The namespaces to be migrated. 58 | - excludedNamespaceScopedResources: The namespace-level resources that should not be migrated. 59 | It is recommended to keep the example configuration and add additional configurations as needed based on actual requirements. 60 | - clusterName: The name of the Kosmos subcluster. 61 | 62 | #### Rollback 63 | Kosmos supports rollback functionality for migrations. 64 | After a successful migration, the existing applications in the subcluster can be restored to their initial state. 65 | Simply edit the PromotePolicy YAML file and add the configuration 'rollback'=true. 66 | ````shell script 67 | apiVersion: kosmos.io/v1alpha1 68 | kind: PromotePolicy 69 | metadata: 70 | name: promote-policy-sample 71 | spec: 72 | rollback: "true" 73 | includedNamespaces: 74 | - namespace1 75 | - namespace2 76 | excludedNamespaceScopedResources: 77 | - events 78 | - events.events.k8s.io 79 | - endpoints 80 | - endpointslices.discovery.k8s.io 81 | clusterName: member-cluster1 82 | ```` 83 | 84 | ### Test Plans 85 | 86 | #### Preparation 87 | First, you need to deploy [Kosmos](https://github.com/kosmos-io/kosmos) (the clustertree module must be installed) and add a subcluster. 88 | 89 | #### Create an Existing Application in the Subcluster 90 | Deploy an nginx application in the subcluster as an example. 91 | ````shell script 92 | apiVersion: apps/v1 93 | kind: Deployment 94 | metadata: 95 | name: nginx 96 | namespace: nginx-test 97 | spec: 98 | selector: 99 | matchLabels: 100 | app: nginx 101 | replicas: 1 102 | template: 103 | metadata: 104 | labels: 105 | app: nginx 106 | deletionGracePeriodSeconds: 30 107 | spec: 108 | containers: 109 | - name: nginx 110 | image: nginx:1.14-alpine 111 | ports: 112 | - containerPort: 80 113 | 114 | --- 115 | apiVersion: v1 116 | kind: Service 117 | metadata: 118 | name: nginx-service 119 | namespace: nginx-test 120 | spec: 121 | selector: 122 | app: nginx 123 | ports: 124 | - protocol: TCP 125 | port: 80 126 | targetPort: 80 127 | type: ClusterIP 128 | ```` 129 | 130 | #### Create a Migration Policy 131 | ````shell script 132 | # kubectl apply -f promote-nginx.yaml 133 | apiVersion: kosmos.io/v1alpha1 134 | kind: PromotePolicy 135 | metadata: 136 | name: promote-policy-example 137 | spec: 138 | includedNamespaces: 139 | - nginx-test 140 | excludedNamespaceScopedResources: 141 | - events 142 | - events.events.k8s.io 143 | - endpoints 144 | - endpointslices.discovery.k8s.io 145 | clusterName: cluster-36-28 146 | ```` 147 | 148 | #### Check the Migration Result 149 | Check the progress of the migration: 150 | ````shell script 151 | # kubectl describe promotepolicy promote-policy-example 152 | Name: promote-policy-example 153 | Namespace: default 154 | Labels: 155 | Annotations: 156 | API Version: kosmos.io/v1alpha1 157 | Kind: PromotePolicy 158 | Metadata: 159 | Creation Timestamp: 2024-03-11T10:57:47Z 160 | Generation: 3 161 | Resource Version: 405947183 162 | UID: 0e32dd93-c370-4874-b9a7-37a6894cd373 163 | Spec: 164 | Cluster Name: cluster-36-28 165 | Excluded Namespace Scoped Resources: 166 | events 167 | events.events.k8s.io 168 | endpoints 169 | endpointslices.discovery.k8s.io 170 | controllerrevisions.apps 171 | Included Namespaces: 172 | nginx-test 173 | Status: 174 | Backedup File: /data/backup/promote-policy-sample20240311-104907 175 | Phase: Completed 176 | Events: 177 | ```` 178 | When Status.Phase is 'Completed', it means the migration is successful. At this point, you can view and manage all applications in the nginx-test namespace in the control plane cluster. 179 | 180 | #### Rollback 181 | Edit the promote-nginx.yml file and add the configuration ‘rollback’=true: 182 | ````shell script 183 | # kubectl apply -f promote-nginx.yaml 184 | apiVersion: kosmos.io/v1alpha1 185 | kind: PromotePolicy 186 | metadata: 187 | name: promote-policy-example 188 | spec: 189 | rollback: "true" 190 | includedNamespaces: 191 | - nginx-test 192 | excludedNamespaceScopedResources: 193 | - events 194 | - events.events.k8s.io 195 | - endpoints 196 | - endpointslices.discovery.k8s.io 197 | clusterName: cluster-36-28 198 | ```` 199 | 200 | Check the rollback result: 201 | ````shell script 202 | # kubectl describe promotepolicy promote-policy-example 203 | Name: promote-policy-example 204 | Namespace: default 205 | Labels: 206 | Annotations: 207 | API Version: kosmos.io/v1alpha1 208 | Kind: PromotePolicy 209 | Metadata: 210 | Creation Timestamp: 2024-03-11T10:57:47Z 211 | Generation: 5 212 | Resource Version: 405953692 213 | UID: 0e32dd93-c370-4874-b9a7-37a6894cd373 214 | Spec: 215 | Cluster Name: cluster-36-28 216 | Excluded Namespace Scoped Resources: 217 | events 218 | events.events.k8s.io 219 | endpoints 220 | endpointslices.discovery.k8s.io 221 | controllerrevisions.apps 222 | Included Namespaces: 223 | nginx-test 224 | Status: 225 | Backedup File: /data/backup/promote-policy-sample20240311-104907 226 | Phase: RolledBack 227 | Events: 228 | ```` 229 | When Status.Phase is 'RolledBack', it means the rollback is successful. 230 | At this point, the applications in the nginx-test namespace cannot be queried in the control plane cluster. 231 | -------------------------------------------------------------------------------- /static/img/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | logo@1.5x 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/application-migration.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: application-migration 3 | title: 'Application Migration' 4 | --- 5 | 6 | # Application Migration 7 | Kosmos provides application migration functionality to help users migrate existing applications from subclusters to the Kosmos control plane cluster. 8 | 9 | ## Application Migration Solution 10 | 11 | ### Introduction 12 | In the Kosmos multi-cluster design architecture, users can interact with the kube-apiserver of the control plane cluster using tools such as kubectl and client-go to create deployments or statefulset applications, etc. 13 | The actual Pod instances are running in the subclusters. 14 | 15 | However, for existing applications in the subclusters that were not created through the Kosmos control plane, these applications cannot be viewed and managed in the control plane cluster. 16 | 17 | Kosmos provides application migration functionality to support the migration of applications in a namespace to the control plane cluster. 18 | The entire process does not require restarting the application Pod instances, ensuring minimal impact on business operations. 19 | 20 | ### Design Details 21 | Application migration mainly consists of three processes: application backup -> deletion of owner objects -> rebuilding applications in the control plane. 22 | 23 | #### Application Backup 24 | Kosmos first backs up all namespaced-level resources in the target namespace, as well as the dependent cluster-level resources, such as cluster roles, cluster role bindings, persistent volumes, etc. 25 | The backup files are stored in PVCs in Kosmos. 26 | 27 | #### Deletion of Owner Objects 28 | The subclusters in Kosmos only run Pods, and their owner StatefulSets or ReplicaSets need to be deleted and rebuilt in the Kosmos control plane. 29 | Similarly, the owner Deployment of a ReplicaSet, as well as the owner StatefulSet and Deployment, need to be deleted and rebuilt in the Kosmos control plane. 30 | 31 | By using a top-down cascading deletion of owner objects (e.g., deleting the Deployment first and then the ReplicaSet), the Pods are not affected and remain in a running state. 32 | 33 | #### Rebuilding Applications in the Control Plane 34 | Based on the backup resources, the control plane cluster creates all the migrated resources, including namespaces, pods, deployments, config maps, service accounts, etc. 35 | In order to maintain consistency with the subcluster Pods and keep them in a running state, the applications are rebuilt using a bottom-up approach (e.g., creating the Pod first and then the ReplicaSet). 36 | 37 | #### CRD API 38 | The PromotePolicy CRD API is provided for configuring the migration policy. 39 | PromotePolicy is a cluster-wide CRD API. Here is an example of how to use it: 40 | ````shell script 41 | apiVersion: kosmos.io/v1alpha1 42 | kind: PromotePolicy 43 | metadata: 44 | name: promote-policy-sample 45 | spec: 46 | includedNamespaces: 47 | - namespace1 48 | - namespace2 49 | excludedNamespaceScopedResources: 50 | - events 51 | - events.events.k8s.io 52 | - endpoints 53 | - endpointslices.discovery.k8s.io 54 | clusterName: member-cluster1 55 | ```` 56 | where: 57 | - includedNamespaces: The namespaces to be migrated. 58 | - excludedNamespaceScopedResources: The namespace-level resources that should not be migrated. 59 | It is recommended to keep the example configuration and add additional configurations as needed based on actual requirements. 60 | - clusterName: The name of the Kosmos subcluster. 61 | 62 | #### Rollback 63 | Kosmos supports rollback functionality for migrations. 64 | After a successful migration, the existing applications in the subcluster can be restored to their initial state. 65 | Simply edit the PromotePolicy YAML file and add the configuration 'rollback'=true. 66 | ````shell script 67 | apiVersion: kosmos.io/v1alpha1 68 | kind: PromotePolicy 69 | metadata: 70 | name: promote-policy-sample 71 | spec: 72 | rollback: "true" 73 | includedNamespaces: 74 | - namespace1 75 | - namespace2 76 | excludedNamespaceScopedResources: 77 | - events 78 | - events.events.k8s.io 79 | - endpoints 80 | - endpointslices.discovery.k8s.io 81 | clusterName: member-cluster1 82 | ```` 83 | 84 | ### Test Plans 85 | 86 | #### Preparation 87 | First, you need to deploy [Kosmos](https://github.com/kosmos-io/kosmos) (the clustertree module must be installed) and add a subcluster. 88 | 89 | #### Create an Existing Application in the Subcluster 90 | Deploy an nginx application in the subcluster as an example. 91 | ````shell script 92 | apiVersion: apps/v1 93 | kind: Deployment 94 | metadata: 95 | name: nginx 96 | namespace: nginx-test 97 | spec: 98 | selector: 99 | matchLabels: 100 | app: nginx 101 | replicas: 1 102 | template: 103 | metadata: 104 | labels: 105 | app: nginx 106 | deletionGracePeriodSeconds: 30 107 | spec: 108 | containers: 109 | - name: nginx 110 | image: nginx:1.14-alpine 111 | ports: 112 | - containerPort: 80 113 | 114 | --- 115 | apiVersion: v1 116 | kind: Service 117 | metadata: 118 | name: nginx-service 119 | namespace: nginx-test 120 | spec: 121 | selector: 122 | app: nginx 123 | ports: 124 | - protocol: TCP 125 | port: 80 126 | targetPort: 80 127 | type: ClusterIP 128 | ```` 129 | 130 | #### Create a Migration Policy 131 | ````shell script 132 | # kubectl apply -f promote-nginx.yaml 133 | apiVersion: kosmos.io/v1alpha1 134 | kind: PromotePolicy 135 | metadata: 136 | name: promote-policy-example 137 | spec: 138 | includedNamespaces: 139 | - nginx-test 140 | excludedNamespaceScopedResources: 141 | - events 142 | - events.events.k8s.io 143 | - endpoints 144 | - endpointslices.discovery.k8s.io 145 | clusterName: cluster-36-28 146 | ```` 147 | 148 | #### Check the Migration Result 149 | Check the progress of the migration: 150 | ````shell script 151 | # kubectl describe promotepolicy promote-policy-example 152 | Name: promote-policy-example 153 | Namespace: default 154 | Labels: 155 | Annotations: 156 | API Version: kosmos.io/v1alpha1 157 | Kind: PromotePolicy 158 | Metadata: 159 | Creation Timestamp: 2024-03-11T10:57:47Z 160 | Generation: 3 161 | Resource Version: 405947183 162 | UID: 0e32dd93-c370-4874-b9a7-37a6894cd373 163 | Spec: 164 | Cluster Name: cluster-36-28 165 | Excluded Namespace Scoped Resources: 166 | events 167 | events.events.k8s.io 168 | endpoints 169 | endpointslices.discovery.k8s.io 170 | controllerrevisions.apps 171 | Included Namespaces: 172 | nginx-test 173 | Status: 174 | Backedup File: /data/backup/promote-policy-sample20240311-104907 175 | Phase: Completed 176 | Events: 177 | ```` 178 | When Status.Phase is 'Completed', it means the migration is successful. At this point, you can view and manage all applications in the nginx-test namespace in the control plane cluster. 179 | 180 | #### Rollback 181 | Edit the promote-nginx.yml file and add the configuration ‘rollback’=true: 182 | ````shell script 183 | # kubectl apply -f promote-nginx.yaml 184 | apiVersion: kosmos.io/v1alpha1 185 | kind: PromotePolicy 186 | metadata: 187 | name: promote-policy-example 188 | spec: 189 | rollback: "true" 190 | includedNamespaces: 191 | - nginx-test 192 | excludedNamespaceScopedResources: 193 | - events 194 | - events.events.k8s.io 195 | - endpoints 196 | - endpointslices.discovery.k8s.io 197 | clusterName: cluster-36-28 198 | ```` 199 | 200 | Check the rollback result: 201 | ````shell script 202 | # kubectl describe promotepolicy promote-policy-example 203 | Name: promote-policy-example 204 | Namespace: default 205 | Labels: 206 | Annotations: 207 | API Version: kosmos.io/v1alpha1 208 | Kind: PromotePolicy 209 | Metadata: 210 | Creation Timestamp: 2024-03-11T10:57:47Z 211 | Generation: 5 212 | Resource Version: 405953692 213 | UID: 0e32dd93-c370-4874-b9a7-37a6894cd373 214 | Spec: 215 | Cluster Name: cluster-36-28 216 | Excluded Namespace Scoped Resources: 217 | events 218 | events.events.k8s.io 219 | endpoints 220 | endpointslices.discovery.k8s.io 221 | controllerrevisions.apps 222 | Included Namespaces: 223 | nginx-test 224 | Status: 225 | Backedup File: /data/backup/promote-policy-sample20240311-104907 226 | Phase: RolledBack 227 | Events: 228 | ```` 229 | When Status.Phase is 'RolledBack', it means the rollback is successful. 230 | At this point, the applications in the nginx-test namespace cannot be queried in the control plane cluster. 231 | -------------------------------------------------------------------------------- /docs/tutorials/kubenest.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: kubenest 3 | title: 'Kubenest' 4 | --- 5 | 6 | # Kubenest 7 | 8 | ## Introduction 9 | 10 | ### What is kubenest? 11 | Kubenest is a solution that virtualizes multiple k8s clusters in a k8s cluster. The cluster that is virtualized with multiple k8s clusters is called host-k8s, 12 | and the virtualized cluster is called virtual-k8s, as shown in the figure below. 13 | 14 | ![kubenest](img/Kubenest-Host-K8s.png) 15 | 16 | ### Why do you need kubenest? 17 | In some cases, cloud vendors will provide a large k8s cluster as infrastructure at the bottom layer to provide computing resources such as CPU and GPU to different tenants. 18 | In order to achieve the isolation of permissions and data of different tenants, multiple small k8s clusters need to be virtualized in the large k8s cluster for tenants to use. 19 | Tenants do not perceive the existence of the large k8s cluster or other tenants, and use the virtualized k8s cluster like a normal k8s cluster. 20 | Based on this background, we designed kubenest. 21 | 22 | ## Solution 23 | 24 | ### Overview 25 | 26 | ![kubenest](img/Kubenest-Solution-Overview.png) 27 | 28 | As shown in the above figure, when a tenant accesses a virtual cluster, they first access the api-server in the virtual control plane on the hosted large cluster (Host k8s), 29 | expose the network externally through nodeport, and then interact with the virtual-k8s cluster created by the api-server in the virtual control plane. 30 | Users can deploy their own services in Virtual k8s. The nodes in Virtual k8s are separated from the hosted large cluster and then joined to virtual-k8s. 31 | Because all control plane services of k8s are installed in the control plane, metadata and network are completely isolated for tenants, 32 | ensuring the security of their data. At the same time, due to the automated disassembly and assembly of nodes, 33 | tenants are provided with the ability to scale elastically in seconds on demand. 34 | In order to virtualize a virtual-k8s cluster in a host-k8s cluster, first, virtual-k8s needs a control plane, in which api-server, 35 | controller-manager, etcd, scheduler, and core-dns components need to be installed. The control plane of this virtual-k8s needs to be installed on the master node of host-k8s. 36 | Then, in virtual-k8s, components such as calico and kube-proxy need to be installed. Among them, api-server, controller-manager, etcd, scheduler, 37 | and core-dns are installed under a namespace in the host-k8s cluster, which is specified by the tenant in CustomResource when creating a virtualcluster. 38 | CustomResource is an entry point for the virtual control plane controller. After the control plane of virtual-k8s is installed, 39 | it is necessary to install the worker node of virtual-k8s. Here we consider removing the worker node in the host-k8s cluster and installing it in the virtual-k8s cluster. 40 | This also explains why the control plane of virtual-k8s should be installed on the master node of host-k8s, because if it is installed in the worker node, 41 | the subsequent disassembly and installation of the node may use this worker node. In this way, the node in virtual-k8s is actually a node in host-k8s. 42 | Once it is removed and installed in virtual-k8s, the get node in host-k8s cannot get the removed node. Therefore, we need a place to "store" the node information in host-k8s. 43 | We designed the Custom Resource Definition globalnode to synchronize label changes on host-k8s, and also store node status (such as whether it belongs to a shared cluster node and whether it is already occupied by virtual-k8s). 44 | When creating virtual-k8s, the controller will select nodes based on CR and globalnode. 45 | 46 | ### Architecture Diagram 47 | 48 | ![Architecture Diagram](img/Kubenest-Architecture.png) 49 | 50 | #### Virtual Cluster Custom Resource 51 | Virtual cluster CR is the API of the virtual control plane operator, which is the entrance to create a virtual control plane. 52 | After the upper-level service creates this CR, the virtual cluster controller performs a series of operations such as creating 53 | a virtual control plane, disassembling nodes, and installing components. The following information needs to be paid attention to in the CR: 54 | 55 | - Metadata.namespace: the ns of the virtual-k8s created. This ns is the ns in host-k8s, used to create the virtual-k8s control plane in this ns 56 | - metadata.name: Name of virtual-k8s created 57 | - Spec.externalIP: This IP will be added to the certificate of virtual-k8s kubeconfig, and the api-server of virtual-k8s 58 | can be accessed externally through this IP 59 | - promotePolicies' labelSelector: Select the label of the worker node in host-k8s, and select the node based on the node label 60 | - promotePolicies.nodeCount: The number of selected nodes 61 | - Spec.kubeconfig: virtual-k8s' kubeconfig, encrypted with base64. No need to fill in on the client side, backfill to CR 62 | after the virtual-k8s virtual control plane is created 63 | - Status.phase: The state created by the virtual control plane that does not need to be filled in on the client side. 64 | It is also the state monitored by the two controllers of the virtual control plane and the disassembly node, and the corresponding 65 | logic is processed according to this state 66 | 67 | An example of CR is as follows: 68 | ```yaml 69 | apiVersion: kosmos.io/v1alpha1 70 | kind: VirtualCluster 71 | metadata: 72 | namespace: test-521-8 73 | name: test-521-8 74 | spec: 75 | externalIP: "192.168.0.1" #dummy ip 76 | promotePolicies: 77 | - labelSelector: 78 | matchLabels: 79 | kubernetes.io/hostname: kc-aaaaa-s-x6fn2 80 | nodeCount: 1 81 | ``` 82 | 83 | #### Virtual Cluster Controller 84 | The virtual cluster controller is used to listen to the virtual cluster CR, create a virtual control plane for virtual-k8s, 85 | disassemble worker nodes, install components in virtual-k8s, and perform a series of operations. 86 | This controller is divided into three specific controllers based on its functions. 87 | - GlobalNodeController: Used to listen to the node node and update the label value on the node. At the same time, 88 | maintain the disassembled state of the node. This information is needed in the CR created by the upper layer service 89 | - VirtualClusterInitController: Create a virtual control plane. After creating the virtual control plane, 90 | wait for the node to be added to the virtual cluster, and then start installing the components 91 | - NodeController: listens to the virtual cluster CR, and performs the operation of disassembling and assembling nodes 92 | when the virtual control plane is ready. 93 | 94 | #### Node-agent 95 | In order to automate node disassembly and provide on-demand second-level elastic scalability, 96 | we have developed a node-agent that can operate nodes through websocket without logging in to the node through ansible or ssh. 97 | We will install an agent on the node, which is a remote operator executor. 98 | The Virtual Cluster Controller implements some operations on the worker-node by calling the agent's interface, 99 | deploys the systemd service through daemonset, and needs to escape to the host user to execute the installation command. 100 | The flowchart of the node joining virtual-k8s is shown in the figure below. The dotted line part is the capability provided 101 | by the node-agent. 102 | 103 | ![flowchart](img/Kubenest-NodeAgent-Flowchart.png) 104 | 105 | ## How to use 106 | We defined the Virtual Cluster through the k8s CRD. Therefore, from a usage perspective, we only need to issue a CR, 107 | and the controller will create virtual-k8s based on the issued CR. At the same time, after the control plane is successfully installed, 108 | the kubeconfig file will be rewritten back to the CR. After the user obtains the kubeconfig, 109 | they can access the virtual-k8s cluster through base64 decryption. A complete CR is shown in the following figure. 110 | 111 | ![CR](img/Kubenest-CR.png) 112 | 113 | ## Technical advantages 114 | - Security isolation: tenant metadata and network are completely isolated to ensure tenant security 115 | - Second-level elastic expansion: quickly pull up the tenant control surface, providing node on-demand second-level elastic expansion 116 | - Efficient operation and maintenance: Create control surfaces through Cloud Native CRE to ensure that management does not interfere with existing resources 117 | 118 | ## Differences from standard k8s 119 | - Because the virtual-k8s control plane is hosted in host-k8s, the tenant cannot change the configuration of the k8s system service 120 | - The tenant's k8s control plane service is deployed in a k8s cluster mixed with other tenants, and is isolated by an exclusive etcd at the logical level 121 | - Control plane services are isolated from container networks on tenant nodes 122 | - The control plane service is exposed through the nodeport of host-k8s, and the tenant service is exposed through the nodeport of the node on virtual-k8s -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/kubenest.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: kubenest 3 | title: 'Kubenest' 4 | --- 5 | 6 | # Kubenest 7 | 8 | ## Introduction 9 | 10 | ### What is kubenest? 11 | Kubenest is a solution that virtualizes multiple k8s clusters in a k8s cluster. The cluster that is virtualized with multiple k8s clusters is called host-k8s, 12 | and the virtualized cluster is called virtual-k8s, as shown in the figure below. 13 | 14 | ![kubenest](img/Kubenest-Host-K8s.png) 15 | 16 | ### Why do you need kubenest? 17 | In some cases, cloud vendors will provide a large k8s cluster as infrastructure at the bottom layer to provide computing resources such as CPU and GPU to different tenants. 18 | In order to achieve the isolation of permissions and data of different tenants, multiple small k8s clusters need to be virtualized in the large k8s cluster for tenants to use. 19 | Tenants do not perceive the existence of the large k8s cluster or other tenants, and use the virtualized k8s cluster like a normal k8s cluster. 20 | Based on this background, we designed kubenest. 21 | 22 | ## Solution 23 | 24 | ### Overview 25 | 26 | ![kubenest](img/Kubenest-Solution-Overview.png) 27 | 28 | As shown in the above figure, when a tenant accesses a virtual cluster, they first access the api-server in the virtual control plane on the hosted large cluster (Host k8s), 29 | expose the network externally through nodeport, and then interact with the virtual-k8s cluster created by the api-server in the virtual control plane. 30 | Users can deploy their own services in Virtual k8s. The nodes in Virtual k8s are separated from the hosted large cluster and then joined to virtual-k8s. 31 | Because all control plane services of k8s are installed in the control plane, metadata and network are completely isolated for tenants, 32 | ensuring the security of their data. At the same time, due to the automated disassembly and assembly of nodes, 33 | tenants are provided with the ability to scale elastically in seconds on demand. 34 | In order to virtualize a virtual-k8s cluster in a host-k8s cluster, first, virtual-k8s needs a control plane, in which api-server, 35 | controller-manager, etcd, scheduler, and core-dns components need to be installed. The control plane of this virtual-k8s needs to be installed on the master node of host-k8s. 36 | Then, in virtual-k8s, components such as calico and kube-proxy need to be installed. Among them, api-server, controller-manager, etcd, scheduler, 37 | and core-dns are installed under a namespace in the host-k8s cluster, which is specified by the tenant in CustomResource when creating a virtualcluster. 38 | CustomResource is an entry point for the virtual control plane controller. After the control plane of virtual-k8s is installed, 39 | it is necessary to install the worker node of virtual-k8s. Here we consider removing the worker node in the host-k8s cluster and installing it in the virtual-k8s cluster. 40 | This also explains why the control plane of virtual-k8s should be installed on the master node of host-k8s, because if it is installed in the worker node, 41 | the subsequent disassembly and installation of the node may use this worker node. In this way, the node in virtual-k8s is actually a node in host-k8s. 42 | Once it is removed and installed in virtual-k8s, the get node in host-k8s cannot get the removed node. Therefore, we need a place to "store" the node information in host-k8s. 43 | We designed the Custom Resource Definition globalnode to synchronize label changes on host-k8s, and also store node status (such as whether it belongs to a shared cluster node and whether it is already occupied by virtual-k8s). 44 | When creating virtual-k8s, the controller will select nodes based on CR and globalnode. 45 | 46 | ### Architecture Diagram 47 | 48 | ![Architecture Diagram](img/Kubenest-Architecture.png) 49 | 50 | #### Virtual Cluster Custom Resource 51 | Virtual cluster CR is the API of the virtual control plane operator, which is the entrance to create a virtual control plane. 52 | After the upper-level service creates this CR, the virtual cluster controller performs a series of operations such as creating 53 | a virtual control plane, disassembling nodes, and installing components. The following information needs to be paid attention to in the CR: 54 | 55 | - Metadata.namespace: the ns of the virtual-k8s created. This ns is the ns in host-k8s, used to create the virtual-k8s control plane in this ns 56 | - metadata.name: Name of virtual-k8s created 57 | - Spec.externalIP: This IP will be added to the certificate of virtual-k8s kubeconfig, and the api-server of virtual-k8s 58 | can be accessed externally through this IP 59 | - promotePolicies' labelSelector: Select the label of the worker node in host-k8s, and select the node based on the node label 60 | - promotePolicies.nodeCount: The number of selected nodes 61 | - Spec.kubeconfig: virtual-k8s' kubeconfig, encrypted with base64. No need to fill in on the client side, backfill to CR 62 | after the virtual-k8s virtual control plane is created 63 | - Status.phase: The state created by the virtual control plane that does not need to be filled in on the client side. 64 | It is also the state monitored by the two controllers of the virtual control plane and the disassembly node, and the corresponding 65 | logic is processed according to this state 66 | 67 | An example of CR is as follows: 68 | ```yaml 69 | apiVersion: kosmos.io/v1alpha1 70 | kind: VirtualCluster 71 | metadata: 72 | namespace: test-521-8 73 | name: test-521-8 74 | spec: 75 | externalIP: "192.168.0.1" #dummy ip 76 | promotePolicies: 77 | - labelSelector: 78 | matchLabels: 79 | kubernetes.io/hostname: kc-aaaaa-s-x6fn2 80 | nodeCount: 1 81 | ``` 82 | 83 | #### Virtual Cluster Controller 84 | The virtual cluster controller is used to listen to the virtual cluster CR, create a virtual control plane for virtual-k8s, 85 | disassemble worker nodes, install components in virtual-k8s, and perform a series of operations. 86 | This controller is divided into three specific controllers based on its functions. 87 | - GlobalNodeController: Used to listen to the node node and update the label value on the node. At the same time, 88 | maintain the disassembled state of the node. This information is needed in the CR created by the upper layer service 89 | - VirtualClusterInitController: Create a virtual control plane. After creating the virtual control plane, 90 | wait for the node to be added to the virtual cluster, and then start installing the components 91 | - NodeController: listens to the virtual cluster CR, and performs the operation of disassembling and assembling nodes 92 | when the virtual control plane is ready. 93 | 94 | #### Node-agent 95 | In order to automate node disassembly and provide on-demand second-level elastic scalability, 96 | we have developed a node-agent that can operate nodes through websocket without logging in to the node through ansible or ssh. 97 | We will install an agent on the node, which is a remote operator executor. 98 | The Virtual Cluster Controller implements some operations on the worker-node by calling the agent's interface, 99 | deploys the systemd service through daemonset, and needs to escape to the host user to execute the installation command. 100 | The flowchart of the node joining virtual-k8s is shown in the figure below. The dotted line part is the capability provided 101 | by the node-agent. 102 | 103 | ![flowchart](img/Kubenest-NodeAgent-Flowchart.png) 104 | 105 | ## How to use 106 | We defined the Virtual Cluster through the k8s CRD. Therefore, from a usage perspective, we only need to issue a CR, 107 | and the controller will create virtual-k8s based on the issued CR. At the same time, after the control plane is successfully installed, 108 | the kubeconfig file will be rewritten back to the CR. After the user obtains the kubeconfig, 109 | they can access the virtual-k8s cluster through base64 decryption. A complete CR is shown in the following figure. 110 | 111 | ![CR](img/Kubenest-CR.png) 112 | 113 | ## Technical advantages 114 | - Security isolation: tenant metadata and network are completely isolated to ensure tenant security 115 | - Second-level elastic expansion: quickly pull up the tenant control surface, providing node on-demand second-level elastic expansion 116 | - Efficient operation and maintenance: Create control surfaces through Cloud Native CRE to ensure that management does not interfere with existing resources 117 | 118 | ## Differences from standard k8s 119 | - Because the virtual-k8s control plane is hosted in host-k8s, the tenant cannot change the configuration of the k8s system service 120 | - The tenant's k8s control plane service is deployed in a k8s cluster mixed with other tenants, and is isolated by an exclusive etcd at the logical level 121 | - Control plane services are isolated from container networks on tenant nodes 122 | - The control plane service is exposed through the nodeport of host-k8s, and the tenant service is exposed through the nodeport of the node on virtual-k8s -------------------------------------------------------------------------------- /docs/tutorials/node-not-ready.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: node-not-ready 3 | title: 'Kosmos Node NotReady' 4 | --- 5 | 6 | # Kosmos Node NotReady 7 | 8 | ## Kosmos Node NotReady Solution 9 | 10 | ### Introduction 11 | Assuming that we have registered the cluster `cluster7` on the master cluster: 12 | ````shell script 13 | $ kubectl get node 14 | NAME STATUS ROLES AGE VERSION 15 | ecs-54004033-001 Ready worker 50d v1.21.5 16 | ecs-54004033-002 Ready control-plane,master,worker 50d v1.21.5 17 | kosmos-cluster7 Ready agent 5d22h v1.21.5 18 | ```` 19 | 20 | The clustertree-cluster-manager of Kosmos will continuously monitor the resource usage and cluster status of the `cluster7` cluster, and update it to the leaf node `kosmos-cluster7` on the master cluster. 21 | ````shell script 22 | $ kubectl get deploy -nkosmos-system 23 | NAME READY UP-TO-DATE AVAILABLE AGE 24 | clusterlink-controller-manager 1/1 1 1 5d22h 25 | clusterlink-elector 2/2 2 2 5d22h 26 | clusterlink-network-manager 1/1 1 1 5d22h 27 | clustertree-cluster-manager 1/1 1 1 5d22h 28 | kosmos-operator 1/1 1 1 5d22h 29 | kosmos-webhook 1/1 1 1 11 30 | ```` 31 | If there is a network fluctuation between the master cluster and the `cluster7` cluster, Kosmos will detect this anomaly and set the status of the leaf node `kosmos-cluster7` on the master cluster to "not ready". This will trigger the pod eviction behavior in Kubernetes, meaning that the pods on the "not ready" node will be evicted to other ready nodes. 32 | 33 | However, due to network fluctuations, the status of `kosmos-cluster7` may become "ready" again during the eviction process. But the events of the originally evicted pods will still be sent to the "cluster7" cluster, causing normal running pods on the "cluster7" cluster to be deleted or restarted, thus affecting the business. 34 | 35 | ### Solution: Integrating Kyverno to solve the kosmos node is not ready 36 | [Kyverno](https://kyverno.io/) validate, mutate, generate, and cleanup configurations using Kubernetes admission webhook, background scans, and source code repository scans. Kyverno policies can be managed as Kubernetes resources. 37 | 38 | Its main functions are as follows: 39 | 40 | - validate, mutate, generate, or cleanup (remove) any resource 41 | - verify container images for software supply chain security 42 | - match resources using label selectors and wildcards 43 | - synchronize configurations across Namespaces 44 | - block non-conformant resources using admission controls, or report policy violations 45 | 46 | This article explains how to use Kyverno's admission webhook to prevent pod expulsion when the kosmos node is not ready. 47 | #### What is an admission webhook? 48 | An "admission webhook" is a piece of code that intercepts requests to the Kubernetes API Server before object persistence. It allows requests to pass through after authentication and authorization. Admission controllers can perform validation, mutation, or both. Mutating controllers modify the resource objects they handle, while Validating controllers do not. If any controller in any phase rejects a request, the entire request will be immediately rejected, and the error will be returned to the end user. 49 | 50 | ![K8s_Admission_Webhook.png](img/K8s_Admission_Webhook.png) 51 | 52 | #### Solution 53 | ##### install Kyverno 54 | [Install Kyverno](https://kyverno.io/docs/installation/methods/) 55 | 56 | ```shell script 57 | kubectl create -f https://github.com/kyverno/kyverno/releases/download/v1.10.0/install.yaml 58 | ``` 59 | 60 | #### Configuring clusterpolicy 61 | There are four scenarios in which k8s evicts the pod: 62 | 63 | - **User initiated** : The user initiates the evict request initiated by the API. For example, all Pods on the node are evicted during node maintenance to avoid the impact on services caused by the node going offline suddenly. 64 | - **Kubelet initiated** : Periodically checks the resources of the node. When the resources are insufficient, some Pods are evicted based on the priority. 65 | - **kube-controller-manager Initiate** : Periodically detects all nodes. When a node is in the NotReady state for more than a period of time, all Pods on the node are evicted so that they are scheduled to other normal nodes for re-running. When taint evict is enabled, the pod that cannot tolerate the taint is exicted immediately after there is a 'NoExecute' taint on node. For the pod that can tolerate the taint, the pod will be evicted after the minimum taint tolerance time configured on the pod. 66 | - **kube-scheduler Initiating** : When preemptive scheduling is implemented, the low-priority Pod may be evicted to make room for the high-priority & preemptive Pod, so that the high-priority Pod can be scheduled normally 67 | 68 | With the following profile, we will only block pod deletion events that meet the following three conditions: 69 | 70 | (1) Node status is NotReady 71 | 72 | (2) Node is a KosmosNode 73 | 74 | (3) the Username is system: serviceaccount: kube-system:node-controller (belong to kube-controller-manager of node-controller ) 75 | 76 | ```yaml 77 | apiVersion: kyverno.io/v1 78 | kind: ClusterPolicy 79 | metadata: 80 | name: kosmos-node-not-ready 81 | spec: 82 | validationFailureAction: Enforce 83 | background: false 84 | rules: 85 | - match: 86 | any: 87 | - resources: 88 | kinds: 89 | - Pod 90 | operations: 91 | - DELETE 92 | name: kosmos-node-not-read 93 | context: 94 | - name: nodeStatus 95 | apiCall: 96 | urlPath: /api/v1/nodes/{{request.oldObject.spec.nodeName}} 97 | jmesPath: status.conditions[?type=='Ready'].status | [0] 98 | - name: isKosmosNode 99 | apiCall: 100 | urlPath: /api/v1/nodes/{{request.oldObject.spec.nodeName}} 101 | jmesPath: metadata.labels."kosmos.io/node" 102 | preconditions: 103 | all: 104 | - key: "{{ request.userInfo.username }}" 105 | operator: Equals 106 | value: "system:serviceaccount:kube-system:node-controller" 107 | - key: "{{ nodeStatus }}" 108 | operator: NotEquals 109 | value: "True" 110 | - key: "{{ length(isKosmosNode) }}" 111 | operator: GreaterThan 112 | value: 0 113 | validate: 114 | message: " {{ request.userInfo.username }} delete pod {{request.oldObject.metadata.name}} of NotReady Kosmos {{request.oldObject.spec.nodeName}} Node is not allowed. " 115 | deny: {} 116 | ``` 117 | 118 | When the status of Kosmos node is notready, the Pods on this node are blocked. You can view the following logs by viewing the kyverno-admission-controller 119 | 120 | ```shell script 121 | handlers.go:139] webhooks/resource/validate "msg"="admission request denied" "clusterroles"=["system:basic-user","system:controller:node-controller","system:discovery","system:public-info-viewer","system:service-account-issuer-discovery"] "gvk"={"group":"","version":"v1","kind":"Pod"} "gvr"={"group":"","version":"v1","resource":"pods"} "kind"="Pod" "name"="example-deployment-6cc4fd9bd7-kkm8z" "namespace"="default" "operation"="DELETE" "resource.gvk"={"Group":"","Version":"v1","Kind":"Pod"} "roles"=null "uid"="7f25ee88-4522-45fd-a6ba-38733122b443" "user"={"username":"system:serviceaccount:kube-system:node-controller","uid":"5a13be66-71fd-40e3-9553-00eb0825fbb0","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"]} 122 | event.go:307] "Event occurred" object="kosmos-node-not-ready" fieldPath="" kind="ClusterPolicy" apiVersion="kyverno.io/v1" type="Warning" reason="PolicyViolation" message="Pod default/example-deployment-6cc4fd9bd7-kkm8z: [kosmos-node-not-ready] fail (blocked); system:serviceaccount:kube-system:node-controller delete pod example-deployment-6cc4fd9bd7-kkm8z of NotReady Kosmos kosmos-cluster2 Node is not allowed. " 123 | validation.go:103] webhooks/resource/validate "msg"="validation failed" "action"="Enforce" "clusterroles"=["system:basic-user","system:controller:node-controller","system:discovery","system:public-info-viewer","system:service-account-issuer-discovery"] "failed rules"=["kosmos-node-not-ready"] "gvk"={"group":"","version":"v1","kind":"Pod"} "gvr"={"group":"","version":"v1","resource":"pods"} "kind"="Pod" "name"="example-deployment-6cc4fd9bd7-sb7m7" "namespace"="default" "operation"="DELETE" "policy"="kosmos-node-not-ready" "resource"="default/Pod/example-deployment-6cc4fd9bd7-sb7m7" "resource.gvk"={"Group":"","Version":"v1","Kind":"Pod"} "roles"=null "uid"="251f1877-4f2c-40ec-9bca-8ceb7c9c845f" "user"={"username":"system:serviceaccount:kube-system:node-controller","uid":"5a13be66-71fd-40e3-9553-00eb0825fbb0","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"]} 124 | ``` 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/node-not-ready.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: node-not-ready 3 | title: 'Kosmos Node NotReady' 4 | --- 5 | 6 | # Kosmos Node NotReady 7 | 8 | ## Kosmos Node NotReady Solution 9 | 10 | ### Introduction 11 | Assuming that we have registered the cluster `cluster7` on the master cluster: 12 | ````shell script 13 | $ kubectl get node 14 | NAME STATUS ROLES AGE VERSION 15 | ecs-54004033-001 Ready worker 50d v1.21.5 16 | ecs-54004033-002 Ready control-plane,master,worker 50d v1.21.5 17 | kosmos-cluster7 Ready agent 5d22h v1.21.5 18 | ```` 19 | 20 | The clustertree-cluster-manager of Kosmos will continuously monitor the resource usage and cluster status of the `cluster7` cluster, and update it to the leaf node `kosmos-cluster7` on the master cluster. 21 | ````shell script 22 | $ kubectl get deploy -nkosmos-system 23 | NAME READY UP-TO-DATE AVAILABLE AGE 24 | clusterlink-controller-manager 1/1 1 1 5d22h 25 | clusterlink-elector 2/2 2 2 5d22h 26 | clusterlink-network-manager 1/1 1 1 5d22h 27 | clustertree-cluster-manager 1/1 1 1 5d22h 28 | kosmos-operator 1/1 1 1 5d22h 29 | kosmos-webhook 1/1 1 1 11 30 | ```` 31 | If there is a network fluctuation between the master cluster and the `cluster7` cluster, Kosmos will detect this anomaly and set the status of the leaf node `kosmos-cluster7` on the master cluster to "not ready". This will trigger the pod eviction behavior in Kubernetes, meaning that the pods on the "not ready" node will be evicted to other ready nodes. 32 | 33 | However, due to network fluctuations, the status of `kosmos-cluster7` may become "ready" again during the eviction process. But the events of the originally evicted pods will still be sent to the "cluster7" cluster, causing normal running pods on the "cluster7" cluster to be deleted or restarted, thus affecting the business. 34 | 35 | ### Solution: Integrating Kyverno to solve the kosmos node is not ready 36 | [Kyverno](https://kyverno.io/) validate, mutate, generate, and cleanup configurations using Kubernetes admission webhook, background scans, and source code repository scans. Kyverno policies can be managed as Kubernetes resources. 37 | 38 | Its main functions are as follows: 39 | 40 | - validate, mutate, generate, or cleanup (remove) any resource 41 | - verify container images for software supply chain security 42 | - match resources using label selectors and wildcards 43 | - synchronize configurations across Namespaces 44 | - block non-conformant resources using admission controls, or report policy violations 45 | 46 | This article explains how to use Kyverno's admission webhook to prevent pod expulsion when the kosmos node is not ready. 47 | #### What is an admission webhook? 48 | An "admission webhook" is a piece of code that intercepts requests to the Kubernetes API Server before object persistence. It allows requests to pass through after authentication and authorization. Admission controllers can perform validation, mutation, or both. Mutating controllers modify the resource objects they handle, while Validating controllers do not. If any controller in any phase rejects a request, the entire request will be immediately rejected, and the error will be returned to the end user. 49 | 50 | ![K8s_Admission_Webhook.png](img/K8s_Admission_Webhook.png) 51 | 52 | #### Solution 53 | ##### install Kyverno 54 | [Install Kyverno](https://kyverno.io/docs/installation/methods/) 55 | 56 | ```shell script 57 | kubectl create -f https://github.com/kyverno/kyverno/releases/download/v1.10.0/install.yaml 58 | ``` 59 | 60 | #### Configuring clusterpolicy 61 | There are four scenarios in which k8s evicts the pod: 62 | 63 | - **User initiated** : The user initiates the evict request initiated by the API. For example, all Pods on the node are evicted during node maintenance to avoid the impact on services caused by the node going offline suddenly. 64 | - **Kubelet initiated** : Periodically checks the resources of the node. When the resources are insufficient, some Pods are evicted based on the priority. 65 | - **kube-controller-manager Initiate** : Periodically detects all nodes. When a node is in the NotReady state for more than a period of time, all Pods on the node are evicted so that they are scheduled to other normal nodes for re-running. When taint evict is enabled, the pod that cannot tolerate the taint is exicted immediately after there is a 'NoExecute' taint on node. For the pod that can tolerate the taint, the pod will be evicted after the minimum taint tolerance time configured on the pod. 66 | - **kube-scheduler Initiating** : When preemptive scheduling is implemented, the low-priority Pod may be evicted to make room for the high-priority & preemptive Pod, so that the high-priority Pod can be scheduled normally 67 | 68 | With the following profile, we will only block pod deletion events that meet the following three conditions: 69 | 70 | (1) Node status is NotReady 71 | 72 | (2) Node is a KosmosNode 73 | 74 | (3) the Username is system: serviceaccount: kube-system:node-controller (belong to kube-controller-manager of node-controller ) 75 | 76 | ```yaml 77 | apiVersion: kyverno.io/v1 78 | kind: ClusterPolicy 79 | metadata: 80 | name: kosmos-node-not-ready 81 | spec: 82 | validationFailureAction: Enforce 83 | background: false 84 | rules: 85 | - match: 86 | any: 87 | - resources: 88 | kinds: 89 | - Pod 90 | operations: 91 | - DELETE 92 | name: kosmos-node-not-read 93 | context: 94 | - name: nodeStatus 95 | apiCall: 96 | urlPath: /api/v1/nodes/{{request.oldObject.spec.nodeName}} 97 | jmesPath: status.conditions[?type=='Ready'].status | [0] 98 | - name: isKosmosNode 99 | apiCall: 100 | urlPath: /api/v1/nodes/{{request.oldObject.spec.nodeName}} 101 | jmesPath: metadata.labels."kosmos.io/node" 102 | preconditions: 103 | all: 104 | - key: "{{ request.userInfo.username }}" 105 | operator: Equals 106 | value: "system:serviceaccount:kube-system:node-controller" 107 | - key: "{{ nodeStatus }}" 108 | operator: NotEquals 109 | value: "True" 110 | - key: "{{ length(isKosmosNode) }}" 111 | operator: GreaterThan 112 | value: 0 113 | validate: 114 | message: " {{ request.userInfo.username }} delete pod {{request.oldObject.metadata.name}} of NotReady Kosmos {{request.oldObject.spec.nodeName}} Node is not allowed. " 115 | deny: {} 116 | ``` 117 | 118 | When the status of Kosmos node is notready, the Pods on this node are blocked. You can view the following logs by viewing the kyverno-admission-controller 119 | 120 | ```shell script 121 | handlers.go:139] webhooks/resource/validate "msg"="admission request denied" "clusterroles"=["system:basic-user","system:controller:node-controller","system:discovery","system:public-info-viewer","system:service-account-issuer-discovery"] "gvk"={"group":"","version":"v1","kind":"Pod"} "gvr"={"group":"","version":"v1","resource":"pods"} "kind"="Pod" "name"="example-deployment-6cc4fd9bd7-kkm8z" "namespace"="default" "operation"="DELETE" "resource.gvk"={"Group":"","Version":"v1","Kind":"Pod"} "roles"=null "uid"="7f25ee88-4522-45fd-a6ba-38733122b443" "user"={"username":"system:serviceaccount:kube-system:node-controller","uid":"5a13be66-71fd-40e3-9553-00eb0825fbb0","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"]} 122 | event.go:307] "Event occurred" object="kosmos-node-not-ready" fieldPath="" kind="ClusterPolicy" apiVersion="kyverno.io/v1" type="Warning" reason="PolicyViolation" message="Pod default/example-deployment-6cc4fd9bd7-kkm8z: [kosmos-node-not-ready] fail (blocked); system:serviceaccount:kube-system:node-controller delete pod example-deployment-6cc4fd9bd7-kkm8z of NotReady Kosmos kosmos-cluster2 Node is not allowed. " 123 | validation.go:103] webhooks/resource/validate "msg"="validation failed" "action"="Enforce" "clusterroles"=["system:basic-user","system:controller:node-controller","system:discovery","system:public-info-viewer","system:service-account-issuer-discovery"] "failed rules"=["kosmos-node-not-ready"] "gvk"={"group":"","version":"v1","kind":"Pod"} "gvr"={"group":"","version":"v1","resource":"pods"} "kind"="Pod" "name"="example-deployment-6cc4fd9bd7-sb7m7" "namespace"="default" "operation"="DELETE" "policy"="kosmos-node-not-ready" "resource"="default/Pod/example-deployment-6cc4fd9bd7-sb7m7" "resource.gvk"={"Group":"","Version":"v1","Kind":"Pod"} "roles"=null "uid"="251f1877-4f2c-40ec-9bca-8ceb7c9c845f" "user"={"username":"system:serviceaccount:kube-system:node-controller","uid":"5a13be66-71fd-40e3-9553-00eb0825fbb0","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"]} 124 | ``` 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | -------------------------------------------------------------------------------- /i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/quickstart.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: quick-start 3 | title: 'Quick Start' 4 | --- 5 | 6 | # 快速开始: 7 | 本指南涵盖了 8 | - 通过Kind创建三套集群，一套作为`host`集群，另外两套作为`member`集群 9 | - 在`host`集群中安装Kosmos控制面组件 10 | - 在`host`集群中将`member`集群进行纳管 11 | - 基于搭建完成的Kosmos环境实现应用的跨集群部署 12 | 13 | ## 前提条件 14 | - [Go](https://golang.org/) version v1.20+ 15 | - [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) version v1.19+ 16 | - [kind](https://kind.sigs.k8s.io/) version v0.14.0+ 17 | - [helm](https://github.com/helm/helm/releases) version v3.11.0+ 18 | 19 | ## 通过脚本一键化部署和运行Kosmos(可选) 20 | 运行如下脚本: 21 | 22 | ```bash 23 | git clone https://github.com/kosmos-io/kosmos.git && cd kosmos 24 | hack/local-up-kosmos.sh 25 | ``` 26 | 27 | 28 | ## 通过`kosmosctl`部署和运行Kosmos（可选） 29 | 30 | ### 1. 通过Kind创建集群 31 | - 可参考如下配置进行Kind集群创建，具体信息可按实际情况修改 32 | ```yaml 33 | kind: Cluster 34 | apiVersion: kind.x-k8s.io/v1alpha4 35 | networking: 36 | # WARNING: It is _strongly_ recommended that you keep this the default 37 | # (127.0.0.1) for security reasons. However it is possible to change this. 38 | apiServerAddress: "192.168.200.112" 39 | # By default the API server listens on a random open port. 40 | # You may choose a specific port but probably don't need to in most cases. 41 | # Using a random port makes it easier to spin up multiple clusters. 42 | apiServerPort: 1443 43 | nodes: 44 | - role: control-plane 45 | extraPortMappings: 46 | - containerPort: "{{container_port}}" 47 | hostPort: "{{host_port}}" 48 | protocol: TCP 49 | listenAddress: "{{host_ipaddress}}" 50 | #- role: worker 51 | #- role: worker 52 | ``` 53 | - 创建集群 Cluster1 54 | 55 | `kind create cluster -n kind-cluster1 --config /path/to/kind-config` 56 | - 创建集群 Cluster2 57 | 58 | `kind create cluster -n kind-cluster2 --config /path/to/kind-config` 59 | - 创建集群 Cluster3 60 | 61 | `kind create cluster -n kind-cluster3 --config /path/to/kind-config` 62 | 63 | ### 2. 安装 `kosmosctl` 客户端 64 | Kosmosctl 是 Kosmos 的一款工具，可快速部署 Kosmos 组件、添加集群以及测试网络连接。安装 `kosmosctl`, 可通过 发布页面下载 或者从源代码进行构建。 65 | 66 | #### 2.1 方式一：下载二进制文件 67 | - 通过 发布页面下载, 仅支持 macOS 系统和 linux系统 68 | - 将`kosmosctl` 客户端放置指定路径，通过配置可不通过绝对路径直接使用`kosmosctl` 客户端进行操作 69 | - 建议在linux系统中使用`kosmosctl` 客户端 70 | ```bash 71 | wget -cO kosmosctl-linux-amd64 https://github.com/kosmos-io/kosmos/releases/download/v0.2.0-lts/kosmosctl-linux-amd64 72 | chmod +x kosmosctl-linux-amd64 && sudo install -m 755 kosmosctl-linux-amd64 /usr/local/bin/kosmosctl 73 | ``` 74 | #### 2.2 方式二：源码构建 75 | - 下载Kosmos源代码 76 | `git clone https://github.com/kosmos-io/kosmos.git` 77 | - 构建代码, 输出内容在` /_output/bin/linux/amd64/kosmosctl` 78 | `make kosmosctl VERSION=v0.1.9`] 79 | - 可通过链接查到可用版本及[Tags](https://github.com/kosmos-io/kosmos/tags) 80 | 81 | ### 3. 安装 Kosmos 控制面组件 82 | - 在 `host`集群中进行 Kosmos 控制面安装，需保证在Kind环境中Pod能够连接集群的ApiServer, 避免`kosmos-operator` 组件出现CrashLoopBackOff情况 83 | ```Shell 84 | kosmosctl install --cni calico --default-nic eth0 // We build a network tunnel based the network interface value passed by the arg default-nic 85 | ``` 86 | ### 4. `host`集群纳管`member`集群 87 | - 将`member`集群的所有kubeconfig文件放置在`host`集群的~/kubeconfig/目录下（亦可自行指定），通过如下命令操作纳管： 88 | ```Shell 89 | kosmosctl join cluster --name cluster2 --kubeconfig ~/kubeconfig/cluster1-kubeconfig --cni calico --default-nic eth0 --enable-all 90 | kosmosctl join cluster --name cluster3 --kubeconfig ~/kubeconfig/cluster2-kubeconfig --cni calico --default-nic eth0 --enable-all 91 | ``` 92 | ### 5. 查看集群纳管情况（像使用单集群一样使用多集群） 93 | - 查看纳管集群 94 | ```shell 95 | kubectl get nodes 96 | NAME STATUS ROLES AGE VERSION 97 | kosmos-cluster1-control-plane Ready control-plane 9d v1.27.3 98 | kosmos-cluster2 Ready agent 9d v1.27.3 99 | kosmos-cluster3 Ready agent 9d v1.27.3 100 | ``` 101 | - 查看集群状态 102 | ```bash 103 | kubectl get clusters 104 | NAME NETWORK_TYPE IP_FAMILY 105 | cluster2 gateway ipv4 106 | cluster3 gateway ipv4 107 | kosmos-control-cluster gateway ipv4 108 | ``` 109 | 110 | ## 通过`Helm`部署和运行Kosmos（可选） 111 | ### 1. 通过Kind创建集群 112 | - 可参考如下配置进行Kind集群创建，具体信息可按实际情况修改 113 | ```yaml 114 | kind: Cluster 115 | apiVersion: kind.x-k8s.io/v1alpha4 116 | networking: 117 | # WARNING: It is _strongly_ recommended that you keep this the default 118 | # (127.0.0.1) for security reasons. However it is possible to change this. 119 | apiServerAddress: "192.168.200.112" 120 | # By default the API server listens on a random open port. 121 | # You may choose a specific port but probably don't need to in most cases. 122 | # Using a random port makes it easier to spin up multiple clusters. 123 | apiServerPort: 1443 124 | nodes: 125 | - role: control-plane 126 | extraPortMappings: 127 | - containerPort: "{{container_port}}" 128 | hostPort: "{{host_port}}" 129 | protocol: TCP 130 | listenAddress: "{{host_ipaddress}}" 131 | #- role: worker 132 | #- role: worker 133 | ``` 134 | - 创建集群 Cluster1 135 | 136 | `kind create cluster -n kind-cluster1 --config /path/to/kind-config` 137 | - 创建集群 Cluster2 138 | 139 | `kind create cluster -n kind-cluster2 --config /path/to/kind-config` 140 | - 创建集群 Cluster3 141 | 142 | `kind create cluster -n kind-cluster3 --config /path/to/kind-config` 143 | 144 | ### 2. Kosmos 的 Helm 部署文件下载 145 | - 下载Kosmos源代码 146 | 147 | `git clone https://github.com/kosmos-io/kosmos.git` 148 | 149 | - 将源代码中charts模块的kosmos文件夹放置`host`集群中 150 | 151 | ### 3. 安装 Kosmos 控制面组件 152 | - 在 `host`集群中进行 Kosmos 控制面安装，需保证在Kind环境中Pod能够连接集群的ApiServer, 避免`kosmos-operator` 组件出现CrashLoopBackOff情况,修改上述下载的kosmos文件夹中value.yaml中的配置值(特别是ns、主集群中kubeconfig信息、镜像等)，使配置值符合当前环境要求，具体配置信息见注释; 153 | ```Shell 154 | kubectl create ns kosmos-system 155 | 156 | helm install kosmos -n kosmos-system kosmos 157 | 158 | ``` 159 | 160 | ### 4. 查看 Kosmos 控制面组件安装情况 161 | ```Shell 162 | kubectl get all -n kosmos-system 163 | 164 | ``` 165 | ![manager-list.png](img/manager-list.png) 166 | 167 | ### 5. Kosmos纳管集群 168 | Kosmos纳管集群操作均在 `host`集群中进行，需要分别对`host`集群、`member`集群均纳管，纳管`host`集群便于后续纳管`member`集群管理操作及跨集群容器网络打通工作，将源代码中deploy模块的cluster_yaml文件夹放置`host`集群中 169 | - 纳管`host`集群 170 | 171 | 修改cluster_yaml中kosmos-control-cluster.yaml文件中注释的内容 172 | ```Shell 173 | kubectl apply -f kosmos-control-cluster.yaml 174 | 175 | ``` 176 | 纳管`host`集群成功后，检查`host`集群cluster对象是否创建,可以看到cluster对象中详细信息，特别是看到status处有新增的clusterLinkStatus信息包括了podCIDRS和serviceCIDRS 177 | ```Shell 178 | kubectl get cluster kosmos-control-cluster -oyaml 179 | 180 | ``` 181 | - 纳管`member`集群 182 | 183 | 修改member-cluster.yaml注释的内容（如果有多个`member`集群，需要有多个文件），使其与所纳管的子集群中的信息完全一致（包含节点网卡信息等） 184 | ```Shell 185 | kubectl apply -f member-cluster.yaml 186 | 187 | ``` 188 | 纳管`member`集群成功后，检查`member`集群cluster对象是否创建,可以看到cluster对象中详细信息，特别是看到status处有新增的clusterLinkStatus信息包括了podCIDRS和serviceCIDRS 189 | ```Shell 190 | kubectl get cluster member-cluster -oyaml 191 | 192 | ``` 193 | 在`host`集群查看`member`集群是否以虚拟节点的方式进行纳管成功 194 | ```Shell 195 | kubectl get nodes 196 | 197 | ``` 198 | kosmos支持一对一个集群维度纳管，亦可支持一对一个集群中所有节点或者部分节点纳管，支持模式在cluster对象中修改 199 | 200 | ### 6. Kosmos集群纳管成功后检查 201 | 202 | - `host`集群检查 203 | 204 | ![host_ippools_check.png](img/host_ippools_check.png) 205 | 206 | 207 | ![host_clusternodes_check.png](img/host_clusternodes_check.png) 208 | 209 | 210 | ![host_nodeconfigs_check.png](img/host_nodeconfigs_check.png) 211 | 212 | 当出现跨集群容器网络问题时，不管是p2p模式还是gateway模式，均可围绕kosmos创建的vx开头的网卡进行相关问题排查(抓包等方式) 213 | 214 | ![host_vx_check.png](img/host_vx_check.png) 215 | 216 | 217 | 218 | - `member`集群检查 219 | 220 | ![member_service_check.png](img/member_service_check.png) 221 | 222 | 223 | ![member_vx_check.png](img/member_vx_check.png) 224 | 225 | 226 | ![member_ippools_check.png](img/member_ippools_check.png) 227 | 228 | 229 | ## 基于搭建的Kosmos环境部署Nginx应用 230 | 231 | - 编辑Nginx服务Yaml 232 | 233 | ```yaml 234 | apiVersion: apps/v1 235 | kind: Deployment 236 | metadata: 237 | name: nginx-deployment 238 | labels: 239 | app: nginx 240 | spec: 241 | replicas: 3 242 | selector: 243 | matchLabels: 244 | app: nginx 245 | template: 246 | metadata: 247 | labels: 248 | app: nginx 249 | spec: 250 | tolerations: 251 | - key: kosmos.io/node 252 | operator: Equal 253 | value: "true" 254 | containers: 255 | - name: nginx 256 | image: nginx:1.14.2 257 | ports: 258 | - containerPort: 80 259 | 260 | ``` 261 | - 执行Nginx服务Yaml 262 | ```bash 263 | kubectl apply -f nginx-deploy.yml 264 | ``` 265 | - 查看Nginx服务Pod状态 266 | 267 | ```bash 268 | kubectl get pods -o wide 269 | NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES 270 | nginx-deployment-887b5c6bb-jx9kq 1/1 Running 0 18h 10.244.0.8 kosmos-cluster3 271 | nginx-deployment-887b5c6bb-kc9ff 1/1 Running 0 41h 10.244.0.7 kosmos-cluster2 272 | nginx-deployment-887b5c6bb-vz8vk 1/1 Running 0 41h 10.244.0.7 kosmos-cluster3 273 | ``` 274 | 综上，基于Kosmos进行Nginx服务实现跨集群部署完成，在子集群中运行Running -------------------------------------------------------------------------------- /docs/proposals/performance-testing.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: performance-testing 3 | title: 'Kosmos Performance Testing' 4 | --- 5 | 6 | # Kosmos Performance Testing 7 | 8 | ## Network Performance Testing 9 | 10 | The target of Kosmos networking is to establish connectivity between multiple Kubernetes clusters. This module can be deployed and used independently. In this document, we do performance testing on cross-cluster pod. 11 | 12 | Bandwidth and latency are two import performance metrics. For bandwidth, we used iperf3 for testing, while ping was used to measure latency. 13 | 14 | The test environment consisted of three physical machines. Among these physical machines, two nodes on same subnet were deployed as a k8s cluster, and the other node on another subnet was separately deployed a k8s cluster. We established overley pod network connectivity between the two clusters using Kosmos. 15 | 16 | environment： 17 | 18 | + kubernetes: `v1.21.5` 19 | 20 | + OS: `BigCloud Enterprise Linux For Euler release 21.10 (LTS-SP2)` 21 | 22 | + kernel: `4.19.90-2107.6.0.0192.8.oe1.bclinux.x86_64` 23 | 24 | + calico: `v3.23.2` 25 | 26 | result： 27 | 28 | | Test scenario | bandwidth | latency | 29 | | --------------------------------------------- | -------------- | ---------- | 30 | | physical machines in same cluster | 9.37 Gbits/sec | 0.091550ms | 31 | | kubernetes overlay network（calico bgp) | 8.15 Gbits/sec | 0.128268ms | 32 | | kubernetes overlay network（calico tunl0） | 6.34 Gbits/sec | 0.170805ms | 33 | | kubernetes overlay network（calico vxlan） | 6.25 Gbits/sec | 0.193439ms | 34 | | physical machines in different cluster | 9.35 Gbits/sec | 0.108512ms | 35 | | cross-cluster overlay network（kosmos gateway） | 6.22 Gbits/sec | 0.332878ms | 36 | | cross-cluster overlay network（kosmos p2p） | 6.26 Gbits/sec | 0.218075ms | 37 | 38 | It can be observed that the transmission through overlay container networks incurs certain losses compared to the transmission through underlay physical machine networks. It is also evident that the performance of our Kosmos network is essentially consistent with the network transmission performance within the Calico cluster. 39 | 40 | By the way, when using kosmos to connect multiple Kubernetes clusters, network performance is better in p2p mode compared to gateway mode. 41 | 42 | 43 | 44 | ## Large scale cluster performance testing 45 | 46 | Based on the standard configuration of Kubernetes large-scale clusters and user production implementation practices, the usage scenario of Kosmos managing 100000 nodes and 200000 Pods simultaneously was tested. Due to limitations in testing environment and tools, we have not pursued the upper limit of testing Kosmos multi cluster systems. 47 | 48 | #### Performance indicators 49 | 50 | How to determine if Kubernetes supports 100000+nodes? Is it supported even if 100000 nodes can be successfully registered in the cluster? Or what conditions? 51 | 52 | The Kubernetes community has provided a series of SLIs (Service Level Indicator)/SLOs (service level objective) indicators to measure the quality of cluster services based on the judgment criteria. In a cluster, when the SLO of these SLIs is met, we say that the current Kubernetes cluster supports this performance metric.https://github.com/kubernetes/community/blob/master/sig-scalability/slos/slos.md 53 | 54 | There are currently three official SLIs: 55 | 56 | **API Call Latency** 57 | 58 | | **Status** | **SLI** | **SLO** | 59 | |------------|---------------------------------------------------------------|-----------------------------------------------| 60 | | Official | Single Object Mutating API P99 latency in the last 5 minutes | Except for aggregation API and CRD, P99 < 1s | 61 | | Official | Non streaming read only API P99 latency in the last 5 minutes | Except for aggregation API and CRD, P99 < 30s | 62 | 63 | **Pod Startup Latency** 64 | 65 | | Status | **SLI** | SLO | 66 | |----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------| 67 | | Official | Stateless Pod startup time (excluding mirror removal and InitContainer), reporting startup time from Pod createTimestamp to all containers, and P99 time observed by the watch | P99 < 5s | 68 | 69 | 70 | 71 | #### Testing tools and test cases 72 | 73 | ##### 1. ClusterLoader2 74 | 75 | ClusterLoader2 is an official open-source cluster load testing tool provided by Kubernetes, which can test the SLIs/SLOs indicators defined by Kubernetes to verify whether the cluster meets various service quality standards. In addition, Clusterloader2 provides visual data for cluster problem localization and cluster performance optimization. ClusterLoader2 will ultimately output a Kubernetes cluster performance report, displaying a series of performance indicator test results. 76 | 77 | ##### 2. Kwok 78 | 79 | Kwok is a simulation testing toolkit officially released by Kubernetes, which allows users to create clusters with thousands of nodes in low resource simulation within seconds, and conduct large-scale testing of Kubernetes controllers without occupying a large amount of infrastructure. This report used the Kwok tool to simulate the creation of 20 Kubernetes clusters, each with 5000 nodes. 80 | 81 | ##### 3. Test cases 82 | 83 | Use the Density test cases provided by ClusterLoader2( https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/testing/density/config.yaml)。The detailed configuration of this report is shown in the table below: 84 | 85 | | Type | Value | 86 | |------------------------------|---------| 87 | | Number of Nodes | 100,000 | 88 | | Number of Pods | 200,000 | 89 | | Number of Pods per node | 2 | 90 | | Number of Namespaces | 200 | 91 | | Number of Pods per Namespace | 1000 | 92 | 93 | ##### 4. Test clusters deploy architecture 94 | 95 | ![img](img/cluster-deploy.jpg) 96 | 97 | 98 | 99 | #### Testing environment detail 100 | 101 | The host Kubernetes is deployed using a single master, with Etcd, Kube apiserver, Kube Schedule, and Kube Controller all deployed as single instances. 102 | 103 | ##### 1. OS version 104 | 105 | BigCloud Enterprise Linux For LDK release 7.6.1906 (Core) 106 | 107 | kernel versions: 4.19.0-240.23.19.el7_6.bclinux.x86_64 108 | 109 | Kubernetes version 110 | 111 | Major:"1",Minor:"21+",GitVersion:"v1.21.5-eki.0",GitCommit:"83c85a6e50757c6bbb81c6ceaac60234c1161817",GitTreeState:"clean",BuildDate:"2021-10-19T08:28:00Z",GoVersion:"go1.16.8",Compiler:"gc", Platform:"linux/amd64" 112 | 113 | 114 | 115 | ##### 2. Master node 116 | 117 | **CPU** 118 | 119 | Architecture: x86_64 120 | 121 | CPU op-mode(s): 32-bit, 64-bit 122 | 123 | Byte Order: Little Endian 124 | 125 | CPU(s): 48 126 | 127 | On-line CPU(s) list: 0-47 128 | 129 | Thread(s) per core: 2 130 | 131 | Core(s) per socket: 12 132 | 133 | Socket(s): 2 134 | 135 | NUMA node(s): 2 136 | 137 | Vendor ID: GenuineIntel 138 | 139 | CPU family: 6 140 | 141 | Model: 85 142 | 143 | Model name: Intel(R) Xeon(R) Silver 4214 CPU @ 2.20GHz 144 | 145 | Stepping: 7 146 | 147 | CPU MHz: 2482.416 148 | 149 | **MEMORY**： 187GB 150 | 151 | 152 | 153 | ##### 3. Component configuration 154 | 155 | **kube-apiserver** 156 | 157 | --max-requests-inflight=1000 158 | 159 | --max-mutating-requests-inflight=1000 160 | 161 | 162 | 163 | **kube-controller-manager ** 164 | 165 | --kube-api-qps=100 166 | 167 | --kube-api-burst=100 168 | 169 | 170 | 171 | **kube-scheduler ** 172 | 173 | --kube-api-qps=100 174 | 175 | --kube-api-burst=200 176 | 177 | 178 | 179 | **etcd ** 180 | 181 | --auto-compaction-retention=4 182 | 183 | --quota-backend-bytes=8589934592 184 | 185 | 186 | 187 | #### Test Results 188 | 189 | **API Call Latency** 190 | 191 | 1. mutating API latency（threshold=1s） 192 | 193 | ![img](img/mutaing_api_latency.jpg) 194 | 195 | 196 | 197 | 2. Read-only API call latency(scope=resource, threshold=1s) 198 | 199 | ![img](img/readonly_api_latency.jpg) 200 | 201 | 202 | 203 | 3. Read-only API call latency(scope=namespace, threshold=5s) 204 | 205 | ![img](img/readonly_api_latency5.jpg) 206 | 207 | 208 | 209 | 4. Read-only API call latency(scope=cluster, threshold=30s) 210 | 211 | ![img](img/readonly_api_latency30.jpg) 212 | 213 | 214 | 215 | ##### 2. **Pod Startup Latency** 216 | 217 | | **metric** | p50(ms) | p90(ms) | p99(ms) | **SLO(ms)** | 218 | |--------------------|---------|---------|---------|-------------| 219 | | pod_startup | 780 | 1189 | 1403 | 5000 | 220 | | create_to_schedule | 0 | 0 | 1000 | N/A | 221 | | schedule_to_run | 0 | 0 | 1000 | N/A | 222 | | run_to_watch | 899 | 1375 | 1512 | N/A | 223 | | schedule_to_watch | 764 | 1176 | 1376 | N/A | 224 | 225 | Note: In cases where the delay is relatively small, due to accuracy loss, certain values recorded by ClusterLoader2 are 0. 226 | 227 | 228 | 229 | #### **Conclusion and Analysis** 230 | 231 | In the above test results, both API Call Latency and Pod Startup Latency meet the SLIs/SLOs metrics defined by the Kubernetes community. Therefore, a multi cluster system with Kosmos as its core can stably support 100000 nodes online simultaneously and manage over 200000 pods. --------------------------------------------------------------------------------