├── .gitattributes ├── .github └── workflows │ ├── golangci-lint.yml │ └── tests.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── OWNERS ├── OWNERS_ALIASES ├── README.md ├── README_VMWARE.md ├── RELEASE.md ├── SECURITY.md ├── SECURITY_CONTACTS ├── Vagrantfile ├── WINDOWS.md ├── build.sh ├── code-of-conduct.md ├── docs └── quo-vadis-swdt.md ├── experiments └── swdt │ ├── .gitignore │ ├── LICENSE │ ├── Makefile │ ├── README.md │ ├── apis │ └── config │ │ └── v1alpha1 │ │ ├── config_types.go │ │ ├── defaults.go │ │ ├── groupversion_info.go │ │ ├── zz_generated.deepcopy.go │ │ └── zz_generated.defaults.go │ ├── cmd │ ├── kubernetes.go │ ├── root.go │ └── setup.go │ ├── go.mod │ ├── go.sum │ ├── hack │ └── boilerplate.go.txt │ ├── main.go │ ├── packer │ ├── .gitignore │ ├── Makefile │ ├── README.md │ └── kvm │ │ ├── floppy │ │ ├── autounattend.xml │ │ └── openssh.ps1 │ │ └── win2022.pkr.hcl │ ├── pkg │ ├── config │ │ ├── node.go │ │ └── node_test.go │ ├── connections │ │ ├── interface.go │ │ ├── ssh.go │ │ ├── ssh_test.go │ │ └── tests │ │ │ └── server.go │ └── pwsh │ │ ├── executor │ │ ├── runner.go │ │ └── runner_test.go │ │ ├── kubernetes │ │ ├── provisioners.go │ │ └── provisioners_test.go │ │ └── setup │ │ ├── setup.go │ │ └── setup_test.go │ └── samples │ ├── config.yaml │ └── mloskot │ ├── .gitignore │ ├── README.windows.md │ ├── ssh.id_rsa │ ├── ssh.id_rsa.pub │ └── winworker.yaml ├── fetch.sh ├── forked ├── StartKubelet.ps1 ├── calico.yaml ├── config.ps1 └── node-service.ps1 ├── swdt.ps1 ├── swdt.sh ├── sync ├── linux │ ├── antrea-0.sh │ ├── calico-0.sh │ ├── controlplane.sh │ ├── e2e.sh │ ├── smoke-test.yaml │ ├── sonobuoyconfig.json │ └── wine2e.yaml ├── shared │ └── README └── windows │ ├── 0-antrea.ps1 │ ├── 0-calico.ps1 │ ├── 0-containerd.ps1 │ ├── 0-kubelet.ps1 │ ├── 1-antrea.ps1 │ ├── 1-calico.ps1 │ ├── forked.ps1 │ └── ssh.ps1 └── variables.yaml /.gitattributes: -------------------------------------------------------------------------------- 1 | *.sh text eol=lf 2 | *.yaml text eol=lf -------------------------------------------------------------------------------- /.github/workflows/golangci-lint.yml: -------------------------------------------------------------------------------- 1 | name: golangci-lint 2 | on: 3 | push: 4 | tags: 5 | - v* 6 | branches: 7 | - 'master' 8 | - '*' 9 | pull_request: 10 | permissions: 11 | contents: read 12 | pull-requests: read 13 | jobs: 14 | golangci: 15 | name: lint 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/setup-go@v3 19 | with: 20 | go-version: 1.21 21 | - uses: actions/checkout@v3 22 | - name: golangci-lint 23 | uses: golangci/golangci-lint-action@v3 24 | with: 25 | working-directory: experiments/swdt/ 26 | version: latest 27 | args: -v 28 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: unit-tests 2 | on: 3 | push: 4 | tags: 5 | - v* 6 | branches: 7 | - 'master' 8 | - '*' 9 | pull_request: 10 | permissions: 11 | contents: read 12 | pull-requests: read 13 | jobs: 14 | unit-test: 15 | name: unit-test 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/setup-go@v3 19 | with: 20 | go-version: 1.21 21 | - uses: actions/checkout@v3 22 | - name: Run unit tests 23 | run: | 24 | pushd experiments/swdt 25 | make test 26 | make build 27 | popd 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | sync/config 2 | sync/windows/bin/ 3 | sync/linux/bin/ 4 | sync/shared/config 5 | sync/shared/kubejoin.ps1 6 | sync/shared/kubeadm.yaml 7 | sync/shared/variables.yaml 8 | .vagrant/ 9 | 10 | # semaphores during build 11 | .lock/ 12 | .idea/ 13 | 14 | # Go 15 | go.work 16 | go.work.sum 17 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.5.0 4 | hooks: 5 | - id: trailing-whitespace 6 | - id: end-of-file-fixer 7 | - id: check-added-large-files 8 | - id: check-json 9 | - id: check-yaml 10 | args: 11 | - --unsafe 12 | - id: check-symlinks 13 | - repo: https://github.com/dnephin/pre-commit-golang 14 | rev: v0.5.1 15 | hooks: 16 | - id: go-fmt 17 | files: ^experiments/swdt/ 18 | - id: go-imports 19 | files: ^experiments/swdt/ 20 | - repo: https://github.com/golangci/golangci-lint 21 | rev: v1.55.2 22 | hooks: 23 | - id: golangci-lint 24 | name: golangci-lint 25 | description: Fast linters runner for Go. 26 | entry: golangci-lint run ./experiments/swdt/... 27 | types: [go] 28 | language: golang 29 | pass_filenames: false 30 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Welcome to Kubernetes. We are excited about the prospect of you joining our [community](https://git.k8s.io/community)! The Kubernetes community abides by the CNCF [code of conduct](code-of-conduct.md). Here is an excerpt: 4 | 5 | _As contributors and maintainers of this project, and in the interest of fostering an open and welcoming community, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities._ 6 | 7 | ## Getting Startedalice 8 | 9 | We have full documentation on how to get started contributing here: 10 | 11 | 14 | 15 | - [Contributor License Agreement](https://git.k8s.io/community/CLA.md) Kubernetes projects require that you sign a Contributor License Agreement (CLA) before we can accept your pull requests 16 | - [Kubernetes Contributor Guide](https://git.k8s.io/community/contributors/guide) - Main contributor documentation, or you can just jump directly to the [contributing section](https://git.k8s.io/community/contributors/guide#contributing) 17 | - [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet) - Common resources for existing developers 18 | 19 | ## Mentorship 20 | 21 | - [Mentoring Initiatives](https://git.k8s.io/community/mentoring) - We have a diverse set of mentorship programs available that are always looking for volunteers! 22 | 23 | 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | 3 | Version 2.0, January 2004 4 | 5 | http://www.apache.org/licenses/ 6 | 7 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 8 | 9 | 1. Definitions. 10 | 11 | "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. 16 | 17 | "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. 18 | 19 | "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. 20 | 21 | "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. 22 | 23 | "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). 24 | 25 | "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. 26 | 27 | "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." 28 | 29 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 30 | 31 | 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 32 | 33 | 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 34 | 35 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: 36 | 37 | You must give any other recipients of the Work or Derivative Works a copy of this License; and 38 | You must cause any modified files to carry prominent notices stating that You changed the files; and 39 | You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 40 | If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. 41 | 42 | You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 43 | 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 44 | 45 | 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 46 | 47 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 48 | 49 | 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 50 | 51 | 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. 52 | 53 | END OF TERMS AND CONDITIONS 54 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | VAGRANT?="vagrant" 16 | 17 | .SILENT: clean 18 | 19 | all: 0-fetch-k8s 1-build-binaries 2-vagrant-up 3-smoke-test 4-e2e-test 20 | 21 | 0: 0-fetch-k8s 22 | 1: 1-build-binaries 23 | 2: 2-vagrant-up 24 | 3: 3-smoke-test 25 | 4: 4-e2e-test 26 | 27 | 0-fetch-k8s: clean 28 | @echo "clean" 29 | @chmod +x fetch.sh 30 | @./fetch.sh 31 | 32 | 1-build-binaries: 33 | @echo "build" 34 | @chmod +x build.sh 35 | @./build.sh $(PWD)/kubernetes 36 | 37 | 2-vagrant-up: 38 | @echo "vagrant phase" 39 | @rm -rf .lock/ 40 | @mkdir -p .lock/ 41 | @echo "making mock kubejoin file to keep Vagrantfile happy in sync/shared" 42 | @touch ./sync/shared/kubejoin.ps1 43 | @echo "######################################" 44 | @echo "Retry vagrant up if the first time the windows node failed" 45 | @echo "Starting the control plane" 46 | @echo "######################################" 47 | @$(VAGRANT) up controlplane 48 | 49 | @echo "*********** vagrant up first run done ~~~~ ENTERING WINDOWS BRINGUP LOOP ***" 50 | @until `$(VAGRANT) status | grep winw1 | grep -q "running"` ; do $(VAGRANT) up winw1 || echo failed_win_up ; done 51 | @until `$(VAGRANT) ssh controlplane -c "kubectl get nodes" | grep -q winw1` ; do $(VAGRANT) provision winw1 || echo failed_win_join; done 52 | @touch .lock/joined 53 | @$(VAGRANT) provision winw1 54 | @touch .lock/cni 55 | 56 | 3-smoke-test: 57 | @$(VAGRANT) ssh controlplane -c "kubectl apply -f /var/sync/linux/smoke-test.yaml" 58 | @$(VAGRANT) ssh controlplane -c "kubectl scale deployment whoami-windows --replicas 0" 59 | @$(VAGRANT) ssh controlplane -c "kubectl scale deployment whoami-windows --replicas 3" 60 | @$(VAGRANT) ssh controlplane -c "kubectl wait --for=condition=Ready=true pod -l 'app=whoami-windows' --timeout=600s" 61 | @$(VAGRANT) ssh controlplane -c "kubectl exec -it netshoot -- curl http://whoami-windows:80/" 62 | 63 | 4-e2e-test: 64 | @$(VAGRANT) ssh controlplane -c "cd /var/sync/linux && chmod +x ./e2e.sh && ./e2e.sh" 65 | 66 | clean: 67 | @touch sync/shared/kubejoin.ps1 68 | $(VAGRANT) destroy --force 69 | rm -rf sync/linux/bin/ 70 | rm -rf sync/windows/bin/ 71 | rm -f sync/shared/config 72 | rm -f sync/shared/kubeadm.yaml 73 | rm -f sync/shared/kubejoin.ps1 74 | rm -rf .lock/ 75 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs at https://go.k8s.io/owners 2 | 3 | approvers: 4 | - aravindhp 5 | - friedrichwilken 6 | - jayunit100 7 | - jsturtevant 8 | - knabben 9 | - marosset 10 | - mloskot 11 | -------------------------------------------------------------------------------- /OWNERS_ALIASES: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs at https://go.k8s.io/owners#owners_aliases 2 | 3 | aliases: 4 | sig-windows-dev-tools: 5 | - friedrichwilken 6 | - jayunit100 7 | - jsturtevant 8 | - knabben 9 | - marosset 10 | - aravindhp 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Welcome to the SIG Windows Development Environment! 2 | 3 | This is a fully batteries-included development environment for Windows on Kubernetes, including: 4 | - Vagrant file for launching a two-node cluster 5 | - containerd 1.6.15 6 | - Support for two CNIs: antrea, or calico on containerd: configure your CNI option in variables.yml 7 | - Calico 3.25.0 on containerd runs containers out of the box 8 | - Antrea 0.13.2 runs but requires running with a patch for https://github.com/antrea-io/antrea/issues/2344 which was recently made available 9 | - NetworkPolicy support for Windows and Linux provided by [Antrea](https://antrea.io) and [Calico](https://www.tigera.io/project-calico/) 10 | - Windows binaries for kube-proxy.exe and kubelet.exe that are either built from source (K8s main branch) or releases 11 | - Kubeadm installation that can put the latest Linux control plane in place 12 | 13 | ## Quick Start 14 | 15 | ### Prerequisites 16 | - Linux host - [Fedora 38](#fedora). 17 | - Experimental support for Windows host with WSL as environment providing `make`, see [Windows with WSL](#windows-with-wsl-experimental). 18 | - [make](https://www.gnu.org/software/make/) 19 | - [Vagrant](https://www.vagrantup.com/downloads) 20 | - [VirtualBox](https://www.virtualbox.org/wiki/Downloads) (we only have VirtualBox automated here, but these recipes have been used with others, like Microsoft HyperV and VMware Fusion). 21 | - [Kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) 22 | 23 | ### Getting a cluster up and running 24 | 25 | Simple steps to a Windows Kubernetes cluster, from scratch, built from source... 26 | 27 | - `vagrant plugin install vagrant-reload vagrant-vbguest winrm winrm-elevated`, vagrant-reload needed to easily reboot windows VMs during setup of containers features. 28 | - `make all`, this will create the entire cluster for you. To compile k/k/ from local source, see instructions later in this doc. 29 | - *If the above failed, run `vagrant provision winw1`, just in case you have a flake during windows installation.* 30 | - `vagrant ssh controlplane` and run `kubectl get nodes` to see your running dual-os linux+windows k8s cluster. 31 | 32 | ## Windows with WSL (experimental) 33 | 34 | All the above Quick Start steps apply, except you have to run the `Makefile` targets in WSL 35 | - using `vagrant.exe` on the host 36 | - while inside clone of this repo on Windows filesystem, not WSL filesystem. 37 | 38 | First, get the path for your `vagrant.exe` on the host use `Get-Command vagrant` in PowerShell like the following example. 39 | 40 | ```powershell 41 | ~ > $(get-command vagrant).Source.Replace("\","/").Replace("C:/", "/mnt/c/") 42 | /mnt/c/HashiCorp/Vagrant/bin/vagrant.exe 43 | ``` 44 | 45 | Next, pass the mount path to the executable on the Windows host with the `VAGRANT` environment variable exported in WSL. 46 | 47 | Then, ensure you clone this repository onto filesystem inside `/mnt` and not the WSL filesystem, in order to avoid failures similar to this one: 48 | 49 | ``` 50 | The host path of the shared folder is not supported from WSL. 51 | Host path of the shared folder must be located on a file system with 52 | DrvFs type. Host path: ./sync/shared 53 | ``` 54 | 55 | Finally, steps to a Windows Kubernetes cluster on Windows host in WSL is turn into the following sequence: 56 | 57 | ```bash 58 | export VAGRANT=/mnt/c/HashiCorp/Vagrant/bin/vagrant.exe 59 | cd /mnt/c/Users/joe 60 | git clone https://github.com/kubernetes-sigs/sig-windows-dev-tools.git 61 | make all 62 | # ... 63 | make clean 64 | ``` 65 | 66 | ## Fedora 67 | 68 | Follow the steps presented below to prepare the Linux host environment and create the two-node cluster: 69 | 70 | **1.** Install essential tools for build and vagrant/virtualbox packages. 71 | 72 | *Example*: 73 | 74 | [Adding](https://developer.hashicorp.com/vagrant/downloads?product_intent=vagrant) 75 | hashicorp repo for most recent vagrant bits: 76 | ``` 77 | sudo dnf install -y dnf-plugins-core 78 | sudo dnf config-manager --add-repo https://rpm.releases.hashicorp.com/fedora/hashicorp.repo 79 | sudo dnf -y install vagrant 80 | ``` 81 | 82 | Installing packages: 83 | ``` 84 | sudo dnf install -y vagrant VirtualBox 85 | sudo vagrant plugin install vagrant-reload vagrant-vbguest winrm winrm-elevated vagrant-ssh 86 | ``` 87 | 88 | **2.** Create `/etc/vbox/networks.conf` to set the [network bits](https://www.virtualbox.org/manual/ch06.html#network_hostonly): 89 | 90 | *Example*: 91 | ``` 92 | sudo mkdir /etc/vbox 93 | sudo vi /etc/vbox/networks.conf 94 | 95 | * 10.0.0.0/8 192.168.0.0/16 96 | * 2001::/64 97 | ``` 98 | 99 | **3.** Clone the repo and build 100 | 101 | If you are building Kubernetes components from source, please follow the 102 | [development guide](https://github.com/kubernetes/community/blob/master/contributors/devel/development.md). 103 | 104 | ``` 105 | git clone https://github.com/kubernetes-sigs/sig-windows-dev-tools.git 106 | cd sig-windows-dev-tools 107 | touch tools/sync/shared/kubejoin.ps1 108 | make all 109 | ``` 110 | 111 | **4.** ssh to the virtual machines 112 | 113 | - Control Plane node (Linux): 114 | ``` 115 | vagrant ssh controlplane 116 | kubectl get pods -A 117 | ``` 118 | 119 | - Windows node: 120 | ``` 121 | vagrant ssh winw1 122 | ``` 123 | 124 | # Goal 125 | 126 | Our goal is to make Windows ridiculously easy to contribute to, play with, and learn about for anyone interested 127 | in using or contributing to the ongoing Kubernetes-on-Windows story. Windows is rapidly becoming an increasingly 128 | viable alternative to Linux thanks to the recent introduction of Windows HostProcess containers and Windows support for NetworkPolicies + Containerd integration. 129 | 130 | 131 | 132 | # Lets run it! 133 | 134 | Ok let's get started... 135 | 136 | ## 1) Pre-Flight checks... 137 | 138 | For the happy path, just: 139 | 140 | 0) Start Docker so that you can build K8s from source as needed. 141 | 1) Install Vagrant, and then vagrant-reload 142 | ``` 143 | vagrant plugin install vagrant-reload vagrant-vbguest winrm winrm-elevated 144 | ``` 145 | 2) Modify CPU/memory in the variables.yml file. We recommend four cores 8G+ for your Windows node if you can spare it, and two cores 8G for your Linux node as well. 146 | 147 | ## 2) Run it! 148 | 149 | There are two use cases for these Windows K8s dev environments: Quick testing, and testing K8s from source. 150 | 151 | ## 3) Testing from source? `make all` 152 | 153 | To test from source, run `vagrant destroy --force ; make all`. This will 154 | 155 | - destroy your existing dev environment (destroying the existent one, and removing binaries folder) 156 | - clone down K8s from GitHub. If you have the k/k repo locally, you can `make path=path_to_k/k all` 157 | - compile the K8s proxy and kubelet (for linux and windows) 158 | - inject them into the Linux and Windows vagrant environment at the /usr/bin and C:/k/bin/ location 159 | - start up the Linux and Windows VMs 160 | 161 | AND THAT'S IT! Your machines should come up in a few minutes... 162 | 163 | NOTE: Do not run the middle Makefile targets, they depend of the sequence to give the full cluster experience. 164 | 165 | ## IMPORTANT 166 | Do not log into the VMs until the provisioning is done. That is especially true for Windows because it will prevent the reboots. 167 | 168 | ## Other notes 169 | 170 | If you still have an old instance of these VMs running for the same dir: 171 | ``` 172 | vagrant destroy -f && vagrant up 173 | ``` 174 | after everything is done (can take 10 min+), ssh' into the Linux VM: 175 | ``` 176 | vagrant ssh controlplane 177 | ``` 178 | and get an overview of the nodes: 179 | ``` 180 | kubectl get nodes 181 | ``` 182 | The Windows node might stay 'NotReady' for a while, because it takes some time to download the Flannel image. 183 | ``` 184 | vagrant@controlplane:~$ kubectl get nodes 185 | NAME STATUS ROLES AGE VERSION 186 | controlplane Ready control-plane,controlplane 8m4s v1.20.4 187 | winw1 NotReady 64s v1.20.4 188 | ``` 189 | ... 190 | ``` 191 | NAME STATUS ROLES AGE VERSION 192 | controlplane Ready control-plane,controlplane 16m v1.20.4 193 | winw1 Ready 9m11s v1.20.4 194 | ``` 195 | 196 | ## Accessing the Windows box 197 | 198 | You'll obviously want to run commands on the Windows box. The easiest way is to SSH into the Windows machine and use powershell from there: 199 | 200 | ``` 201 | vagrant ssh winw1 202 | C:\ > powershell 203 | ``` 204 | 205 | Optionally, you can do this by noting the IP address during `vagrant provision` and running *any* RDP client (vagrant/vagrant for username/password, works for SSH). 206 | To run a *command* on the Windows boxes without actually using the UI, you can use `winrm`, which is integrated into Vagrant. For example, you can run: 207 | 208 | ``` 209 | vagrant winrm winw1 --shell=powershell --command="ls" 210 | ``` 211 | 212 | IF you want to debug on the windows node, you can also run crictl: 213 | 214 | ``` 215 | .\crictl config --set runtime-endpoint=npipe:////./pipe/containerd-containerd 216 | ``` 217 | 218 | ## Where we derived these recipes from 219 | 220 | - This guide is based on [this very nice Vagrantfile](https://gist.github.com/danielepolencic/ef4ddb763fd9a18bf2f1eaaa2e337544) 221 | - this very good [guide on how to install Kubernetes on Ubuntu Focal (20.04)](https://github.com/mialeevs/kubernetes_installation). 222 | - The Windows part is informed by this [guide on how to install Docker on Win Server 2019](https://www.hostafrica.co.za/blog/new-technologies/how-to-install-docker-on-linux-and-windows/#win), [this guide on adding Windows nodes](https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/adding-windows-nodes/), and [this guide](https://www.hostafrica.co.za/blog/new-technologies/install-kubernetes-cluster-windows-server-worker-nodes/) on how to install Kubernetes on Win Server 2019. 223 | - We've also borrowed ideas from cluster api, kubeadm, and the antrea project too bootstrap how we manage CNI and containerd support. 224 | 225 | # Contributing 226 | 227 | Working on Windows Kubernetes is a great way to learn about Kubernetes internals and how Kubernetes works in a multi-OS environment. 228 | 229 | So, even if you aren't a Windows user, we encourage Kubernetes users of all types to try to get involved and contribute! 230 | 231 | We are a new project and we need help with... 232 | 233 | - contributing / testing recipes on different Vagrant providers 234 | - docs of existing workflows 235 | - CSI support and testing 236 | - privileged container support 237 | - recipes with active directory 238 | - any other ideas! 239 | 240 | If nothing else, filing an issue with your bugs or experiences will be helpful long-term. If interested in pairing with us to do your first contribution, just reach out in #sig-windows (https://slack.k8s.io/). We understand that developing on Kubernetes with Windows is new to many folks, and we're here to help you get started. 241 | -------------------------------------------------------------------------------- /README_VMWARE.md: -------------------------------------------------------------------------------- 1 | Instructions to use vmware fusion 2 | 3 | # Install the vagrant plugin 4 | 5 | ``` 6 | vagrant plugin install vagrant-vmware-desktop 7 | ``` 8 | 9 | # Install system package for vmware utils 10 | 11 | ``` 12 | https://www.vagrantup.com/docs/providers/vmware/vagrant-vmware-utility 13 | ``` 14 | 15 | # Run with vsphere: 16 | 17 | ``` 18 | vagrant up --provider= 19 | 20 | ``` 21 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | # Release Process 2 | 3 | The SIG Windows Dev Tools Project is released on an as-needed basis. The process is as follows: 4 | 5 | 1. An issue is proposing a new release with a changelog since the last release 6 | 2. All [OWNERS](OWNERS) must LGTM this release 7 | 3. An OWNER runs `git tag -s $VERSION` and inserts the changelog and pushes the tag with `git push $VERSION` 8 | 4. The release issue is closed 9 | 5. An announcement email is sent to `kubernetes-dev@googlegroups.com` with the subject `[ANNOUNCE] kubernetes-template-project $VERSION is released` 10 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Security Announcements 4 | 5 | Join the [kubernetes-security-announce] group for security and vulnerability announcements. 6 | 7 | You can also subscribe to an RSS feed of the above using [this link][kubernetes-security-announce-rss]. 8 | 9 | ## Reporting a Vulnerability 10 | 11 | Instructions for reporting a vulnerability can be found on the 12 | [Kubernetes Security and Disclosure Information] page. 13 | 14 | ## Supported Versions 15 | 16 | Information about supported Kubernetes versions can be found on the 17 | [Kubernetes version and version skew support policy] page on the Kubernetes website. 18 | 19 | [kubernetes-security-announce]: https://groups.google.com/forum/#!forum/kubernetes-security-announce 20 | [kubernetes-security-announce-rss]: https://groups.google.com/forum/feed/kubernetes-security-announce/msgs/rss_v2_0.xml?num=50 21 | [Kubernetes version and version skew support policy]: https://kubernetes.io/docs/setup/release/version-skew-policy/#supported-versions 22 | [Kubernetes Security and Disclosure Information]: https://kubernetes.io/docs/reference/issues-security/security/#report-a-vulnerability 23 | -------------------------------------------------------------------------------- /SECURITY_CONTACTS: -------------------------------------------------------------------------------- 1 | # Defined below are the security contacts for this repo. 2 | # 3 | # They are the contact point for the Product Security Committee to reach out 4 | # to for triaging and handling of incoming issues. 5 | # 6 | # The below names agree to abide by the 7 | # [Embargo Policy](https://git.k8s.io/security/private-distributors-list.md#embargo-policy) 8 | # and will be removed and replaced if they violate that agreement. 9 | # 10 | # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE 11 | # INSTRUCTIONS AT https://kubernetes.io/security/ 12 | 13 | sladyn98 14 | jayunit100 15 | friedrichwilken -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | require 'yaml' 4 | require 'fileutils' 5 | 6 | # Modify these in the variables.yaml file... they are described there in gory detail... 7 | # This will get copied down later to synch/shared/variables... and read by the controlplane.sh etc... 8 | settingsFile = "variables.yaml" || ENV["VAGRANT_VARIABLES"] 9 | puts "[Vagrantfile] Loading settings from #{settingsFile}" 10 | FileUtils.cp(settingsFile, "sync/shared/variables.yaml") 11 | settings = YAML.load_file settingsFile 12 | 13 | 14 | kubernetes_version=settings["kubernetes_version"] 15 | k8s_linux_kubelet_nodeip=settings['k8s_linux_kubelet_nodeip'] 16 | pod_cidr=settings['pod_cidr'] 17 | calico_version=settings['calico_version'] 18 | containerd_version=settings['containerd_version'] 19 | 20 | 21 | linux_ram = settings['linux_ram'] 22 | linux_cpus = settings['linux_cpus'] 23 | windows_ram = settings['windows_ram'] 24 | windows_cpus = settings['windows_cpus'] 25 | windows_node_ip = settings['windows_node_ip'] 26 | 27 | cni = settings['cni'] 28 | 29 | Vagrant.configure(2) do |config| 30 | puts "[Vagrantfile] Using Kubernetes version: #{kubernetes_version}" 31 | puts "[Vagrantfile] Using Kubernetes CNI implementation: #{cni}" 32 | 33 | # LINUX Control Plane 34 | config.vm.define :controlplane do |controlplane| 35 | controlplane.vm.host_name = "controlplane" 36 | controlplane.vm.box = "roboxes/ubuntu2004" 37 | 38 | controlplane.vm.network :private_network, ip:"#{k8s_linux_kubelet_nodeip}" 39 | 40 | controlplane.vm.provider :virtualbox do |vb| 41 | vb.memory = linux_ram 42 | vb.cpus = linux_cpus 43 | end 44 | 45 | controlplane.vm.synced_folder "./sync/shared", "/var/sync/shared" 46 | controlplane.vm.synced_folder "./forked", "/var/sync/forked" 47 | controlplane.vm.synced_folder "./sync/linux", "/var/sync/linux" 48 | 49 | ### This allows the node to default to the right IP i think.... 50 | # 1) this seems to break the ability to get to the internet 51 | 52 | controlplane.vm.provision :shell, privileged: false, path: "sync/linux/controlplane.sh", args: "#{kubernetes_version} #{k8s_linux_kubelet_nodeip} #{pod_cidr}" 53 | 54 | # TODO shoudl we pass KuberneteVersion to calico agent exe? and also service cidr if needed? 55 | # dont run as priveliged cuz we need the kubeconfig from regular user 56 | if cni == "calico" then 57 | puts "[Vagrantfile] Provisioning controlplane with Calico: #{calico_version}" 58 | controlplane.vm.provision "shell", path: "sync/linux/calico-0.sh", args: "#{pod_cidr} #{calico_version}" 59 | else 60 | controlplane.vm.provision "shell", path: "sync/linux/antrea-0.sh" 61 | end 62 | end 63 | 64 | # WINDOWS WORKER (win server 2019) 65 | config.vm.define :winw1 do |winw1| 66 | winw1.vm.host_name = "winw1" 67 | winw1.vm.box = "sig-windows-dev-tools/windows-2019" 68 | winw1.vm.box_version = "1.0" 69 | 70 | winw1.vm.provider :virtualbox do |vb| 71 | vb.memory = windows_ram 72 | vb.cpus = windows_cpus 73 | vb.gui = false 74 | end 75 | 76 | winw1.vm.network :private_network, ip:"#{windows_node_ip}" 77 | winw1.vm.synced_folder ".", "/vagrant", disabled:true 78 | winw1.vm.synced_folder "./sync/shared", "C:/sync/shared" 79 | winw1.vm.synced_folder "./sync/windows/", "C:/sync/windows/" 80 | winw1.vm.synced_folder "./forked", "C:/forked/" 81 | 82 | winw1.winrm.username = "vagrant" 83 | winw1.winrm.password = "vagrant" 84 | 85 | if not File.file?(".lock/joined") then 86 | # Update containerd 87 | puts "[Vagrantfile] Provisioning winw1 node with Calico: #{calico_version}; containerd: #{containerd_version}" 88 | winw1.vm.provision "shell", path: "sync/windows/0-containerd.ps1", args: "#{calico_version} #{containerd_version}", privileged: true 89 | 90 | # Joining the controlplane 91 | winw1.vm.provision "shell", path: "sync/windows/forked.ps1", args: "#{kubernetes_version}", privileged: true 92 | winw1.vm.provision "shell", path: "sync/shared/kubejoin.ps1", privileged: true #, run: "never" 93 | else 94 | if not File.file?(".lock/cni") then 95 | if cni == "calico" then 96 | # we don't need to run Calico agents as service now, 97 | # calico will be installed as a HostProcess container 98 | # installs both felix and node 99 | #winw1.vm.provision "shell", path: "sync/windows/0-calico.ps1", privileged: true 100 | #winw1.vm.provision "shell", path: "sync/windows/1-calico.ps1", privileged: true 101 | else 102 | winw1.vm.provision "shell", path: "sync/windows/0-antrea.ps1", privileged: true #, run: "always" 103 | winw1.vm.provision "shell", path: "sync/windows/1-antrea.ps1", privileged: true, args: "#{windows_node_ip}" #, run: "always" 104 | end 105 | end 106 | end 107 | end 108 | end 109 | -------------------------------------------------------------------------------- /WINDOWS.md: -------------------------------------------------------------------------------- 1 | # sig-windows-dev-tools Dual Boot Setup 2 | 3 | ## Introduction 4 | 5 | Welcome to sig-windows-dev-tools, a suite of tools designed to help Windows developers create and maintain high-quality applications. 6 | 7 | This guide is here to provide instructions on how to setup a 2 node cluster environment utilizing your current Windows 11 PC. 8 | 9 | ## Features 10 | 11 | - Automated testing and debugging 12 | - Easy-to-use graphical user interface 13 | - Support for multiple languages 14 | - Comprehensive documentation 15 | 16 | ## Minimum Requirements 17 | 18 | - Windows PC (Windows 11 Used) 19 | - 100GB of free HDD space 20 | - 16GB of RAM 21 | 22 | ## Installation 23 | 24 | As we are utilizing our existing Windows 11 computer, we will start by shrinking the current hard drive. This will provide us the space we need to install Ubuntu and run the scripts necessary to automate the development 2 Node cluster setup. 25 | 26 | ## Documentation 27 | 28 | ### Windows Configuration Steps 29 | 30 | 1. Start by shrinking the current HDD and partitioning 100GB to install Ubuntu Desktop and additional tools. 31 | - Open disk management by searching `create and format hard disk partitions` within the start menu. 32 | - Right click on the hard drive you would like to shrink, select shrink, set the amount of space to shrink in MB (100GB Minimum) and select shrink. 33 | 34 | 2. These instructions were created using the latest Ubuntu LTS Desktop image (22.04 LTS) from [Ubuntu Desktop] and configured flash drive for installation boot. 35 | - Boot/ Install instructions can be found - [Ubuntu Tutorials] 36 | 3. Reboot your PC and install Ubuntu Desktop now we have an unallocated amount of space ready, and the boot drive configured from step 2. 37 | 4. Upon Reboot, insert the USB drive, and install Ubuntu desktop on the newly allocated space. 38 | - Boot/ Install instructions can be found - [Ubuntu Tutorials] 39 | 40 | ### Ubuntu Desktop Instructions 41 | 42 | 1. Install essential tools for build and vagrant/virtualbox packages. 43 | 44 | *Example*: 45 | 46 | Adding hashicorp repo for most recent vagrant bits: 47 | 48 | ``` 49 | curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add -; 50 | sudo apt-add-repository \ 51 | "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"; 52 | sudo apt-get update 53 | 54 | ``` 55 | 56 | Installing packages: 57 | ``` 58 | sudo apt install build-essential vagrant virtualbox virtualbox-ext-pack -y 59 | sudo vagrant plugin install vagrant-reload vagrant-vbguest winrm winrm-elevated vagrant-ssh 60 | ``` 61 | 62 | 2. Create `/etc/vbox/networks.conf` to set the [network bits](https://www.virtualbox.org/manual/ch06.html#network_hostonly): 63 | 64 | *Example*: 65 | ``` 66 | sudo mkdir /etc/vbox 67 | sudo vi /etc/vbox 68 | 69 | * 10.0.0.0/8 192.168.0.0/16 70 | * 2001::/64 71 | ``` 72 | 73 | 3. Clone the repo and build 74 | 75 | **Change directory to where you want the sig-windows-dev-tools to be installed*: 76 | 77 | ``` 78 | sudo apt install git; 79 | git clone https://github.com/kubernetes-sigs/sig-windows-dev-tools.git; 80 | cd sig-windows-dev-tools; 81 | mkdir -p tools/sync/shared; 82 | touch tools/sync/shared/kubejoin.ps1; 83 | sudo apt install make; 84 | make all 85 | 86 | ``` 87 | 88 | 4. ssh to the virtual machines 89 | 90 | - Control Plane node (Linux): 91 | ``` 92 | vagrant ssh controlplane 93 | kubectl get pods -A 94 | ``` 95 | 96 | - Windows node: 97 | ``` 98 | vagrant ssh winw1 99 | ``` 100 | 101 | Now [let's run it!](https://github.com/kubernetes-sigs/sig-windows-dev-tools#lets-run-it) 102 | 103 | [Ubuntu Tutorials]: https://ubuntu.com/tutorials/install-ubuntu-desktop#1-overview 104 | [Ubuntu Desktop]: https://ubuntu.com/download/desktop 105 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | : ' 3 | Copyright 2021 The Kubernetes Authors. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ' 17 | set -e 18 | 19 | START_DIR=`pwd` 20 | BUILD_FROM_SOURCE=0 21 | VARIABLES_FILE="sync/shared/variables.yaml" 22 | 23 | download_binaries () { 24 | for bin in kubelet kube-proxy kubeadm kubectl 25 | do 26 | # windows binaries 27 | curl "https://storage.googleapis.com/k8s-release-dev/ci/${KUBERNETES_VERSION}/bin/windows/amd64/${bin}.exe" -o $START_DIR/sync/windows/bin/${bin}.exe 28 | # linux binaries 29 | curl "https://storage.googleapis.com/k8s-release-dev/ci/${KUBERNETES_VERSION}/bin/linux/amd64/${bin}" -o $START_DIR/sync/linux/bin/${bin}; chmod +x $START_DIR/sync/linux/bin/${bin} 30 | done 31 | } 32 | 33 | build_binaries () { 34 | echo "building kube locally inside `pwd` from $1" 35 | echo "changing into directory $1 to start the build" 36 | pushd $1 37 | echo "running docker build ~ checking memory, make sure its > 4G!!!" 38 | if [[ `docker system info | grep Memory | cut -d' ' -f 4 |cut -d'.' -f 1` -gt 4 ]] || [[ `docker system info | grep memFree | cut -d' ' -f 4 |cut -d'.' -f 1` -gt 4000000000 ]]; then 39 | echo "Proceeding with build, docker daemon memory is ok" 40 | else 41 | echo "Insufficient LOCAL memory to build k8s before the vagrant builder starts" 42 | exit 1 43 | fi 44 | 45 | # use the kubernetes/build/run script to build specific targets... 46 | ./build/run.sh make kubelet KUBE_BUILD_PLATFORMS=windows/amd64 47 | ./build/run.sh make kube-proxy KUBE_BUILD_PLATFORMS=windows/amd64 48 | ./build/run.sh make kubeadm KUBE_BUILD_PLATFORMS=windows/amd64 49 | 50 | ./build/run.sh make kubelet KUBE_BUILD_PLATFORMS=linux/amd64 51 | ./build/run.sh make kubectl KUBE_BUILD_PLATFORMS=linux/amd64 52 | ./build/run.sh make kubeadm KUBE_BUILD_PLATFORMS=linux/amd64 53 | } 54 | 55 | copy_to_sync () { 56 | # TODO replace with https://github.com/kubernetes-sigs/sig-windows-tools/issues/152 at some point 57 | echo "Copying files to sync in ... $START_DIR" 58 | 59 | # Windows binaries 60 | cp -f ./_output/dockerized/bin/windows/amd64/kubelet.exe $START_DIR/sync/windows/bin 61 | cp -f ./_output/dockerized/bin/windows/amd64/kube-proxy.exe $START_DIR/sync/windows/bin 62 | cp -f ./_output/dockerized/bin/windows/amd64/kubeadm.exe $START_DIR/sync/windows/bin 63 | 64 | # Linux binaries 65 | cp -f ./_output/dockerized/bin/linux/amd64/kubelet $START_DIR/sync/linux/bin 66 | cp -f ./_output/dockerized/bin/linux/amd64/kubectl $START_DIR/sync/linux/bin 67 | cp -f ./_output/dockerized/bin/linux/amd64/kubeadm $START_DIR/sync/linux/bin 68 | popd 69 | } 70 | 71 | cleanup () { 72 | echo "Cleaning up the kubernetes directory" 73 | pwd 74 | rm -rf ../kubernetes 75 | } 76 | 77 | # Test if it should be build from source 78 | [[ 79 | $(awk '/build_from_source/ {print $2}' ${VARIABLES_FILE} | sed -e 's/^"//' -e 's/"$//' | head -1) =~ "true" 80 | ]] && BUILD_FROM_SOURCE=1 81 | 82 | mkdir -p $START_DIR/sync/windows/bin 83 | mkdir -p $START_DIR/sync/linux/bin 84 | 85 | # check if theres an input for the path 86 | if [[ $BUILD_FROM_SOURCE -eq 1 ]] ;then 87 | # Check if variable is set 88 | if [ -z "$1" ]; then 89 | echo "build_from_source is true but no path passed to the script, exiting." 90 | exit 1 91 | fi 92 | 93 | echo "Directory Kubernetes provided... building" 94 | build_binaries $1 95 | copy_to_sync 96 | cleanup 97 | else 98 | version=`awk '/kubernetes_version/ {print $2}' ${VARIABLES_FILE} | sed -e 's/^"//' -e 's/"$//' | head -1` 99 | KUBERNETES_VERSION=`curl -f https://storage.googleapis.com/k8s-release-dev/ci/latest-${version}.txt` 100 | download_binaries 101 | fi 102 | -------------------------------------------------------------------------------- /code-of-conduct.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Community Code of Conduct 2 | 3 | Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /docs/quo-vadis-swdt.md: -------------------------------------------------------------------------------- 1 | --- 2 | Author: Mateusz Loskot (aka mloskot) 3 | E-mail: mateusz@loskot.net 4 | --- 5 | 6 | # Quo Vadis SWDT? 7 | 8 | Follow-up to [SIG-Windows weekly meeting](https://docs.google.com/document/d/1aCcqSDBuXQ2gTbBg6QuGvLxCmzlSrAbTfflCTmo-960/edit#heading=h.la7dlzuh8o0o) from Jan 2, 2024, 9 | this is my brainstorm of ideas and issues about the state and future of SWDT. 10 | It turned out to be not as structured and systematic as I wished, apologies 11 | for the chaos of thoughts, but it is a brainstorm, or rather, a braindump. 12 | 13 | ## Promise 14 | 15 | The SWDT initial promise of the batteries-included local cluster with Windows 16 | worker nodes integration on variety of host operating system although very 17 | attractive turned out to be difficult to fulfill in manner that is reliably 18 | usable and maintainable at the same time. 19 | 20 | The SWDT goals need to be clarified and redefined, so they are achievable. 21 | 22 | ## Issues 23 | 24 | - Pre-built images used for node VM-s, especially Windows, impose significant 25 | maintenance burden and become outdated quickly. 26 | 27 | - Content of pre-built images is not easy to swap with latest releases of 28 | Kubernetes components. 29 | 30 | - Convenience tools like Vagrant require pre-built images and come with 31 | their own limitations and bugs, which may become frustrating blockers. 32 | 33 | - VirtualBox on Windows has a long history of performance issues and conflicts 34 | with other hypervisors like Windows-native Hyper-V or even with WSL. 35 | 36 | - Despite use of pre-built images, current implementation is still complex 37 | and hacky, based on numerous undocumented scripts and fragile flows. 38 | It also requires `make` which is neither Windows-native nor portable. 39 | 40 | - Although the project is supposed to be intrinsically quite simple and high level, 41 | its overall maintenance unfriendliness makes it not attractive to contributors. 42 | It is worth to acknowledge that SWDT is an auxiliary project rather than a product. 43 | 44 | - The end-user documentation is too long and fiddly. Not quite reflects, quote: 45 | "Our goal is to make Windows ridiculously easy to contribute to, play with". 46 | 47 | ## Value 48 | 49 | SWDT as a collection of current "Kubernetes on Windows" knowledge 50 | and practices typically scattered in numerous resources as well as 51 | user-friendly and configurable application to execute those locally 52 | for development, testing and learning purposes on Linux or Windows host. 53 | 54 | SWDT as a [facade](https://en.wikipedia.org/wiki/Facade_pattern) 55 | masking complexity of Kubernetes documentation and tools for the basic 56 | purpose of running a cluster with Windows-based workloads. 57 | 58 | Despite there is multitude of Kubernetes distributions available to 59 | run a local cluster, niche for a solution with simple minimal bare 60 | Kubernetes on Linux and Windows seems to remains vacant. That is, 61 | vanilla Kubernetes without using any amazing magic like the kind does, 62 | but plain `kubeadm` joining virtual or physical machines as nodes. 63 | 64 | **Ku(bare)netes**! [tm] ;) 65 | 66 | ## Dream 67 | 68 | Give users a command line tool that can **non-interactively** do: 69 | 70 | 1. Create Linux VM and install minimal Linux OS configured to become 71 | Kubernetes master node with control plane 72 | 2. Create Windows VM and install Windows Server OS configured to become 73 | Kubernetes worker node 74 | 3. Install container runtime, CNI and Kubernetes from official release 75 | or user-specified build 76 | 4. Initialise the control plane 77 | 5. Join the Windows node 78 | 79 | and that can do it: 80 | 81 | - on Linux or Windows host 82 | - with host-native (or close) virtualisation solution i.e. KVM/QEMU on Linux 83 | and Hyper-V on Windows 84 | - without using any pre-built SWDT-specific images. 85 | 86 | and such CLI is written in Go, so it can run on Linux and Windows host smoothly. 87 | 88 | Major challenges: 89 | 90 | 1. How to non-interactively create VM-s to build cluster nodes? 91 | 2. How to non-interactively install OS on VM-s to provision cluster nodes? 92 | 93 | Especially, how to manage it for Windows node as Windows OS is still not 94 | as friendly for non-human operators as Linux is. 95 | 96 | ### libvirt 97 | 98 | libvirt is highly capable, but is still Linux-oriented solution, so it would be only 99 | usable for managing Linux node on Linux host. 100 | 101 | Although there is Microsoft [Hyper-V driver](https://libvirt.org/drvhyperv.html) available, 102 | [it does not seem to be battle tested](https://lists.libvirt.org/archives/list/users@lists.libvirt.org/thread/IHVIAT72GD43DERY6FXIVNMMXHMNHLQ5/#IHVIAT72GD43DERY6FXIVNMMXHMNHLQ5). 103 | 104 | Choosing libvirt comes with risk of becoming a distraction due to getting involved in 105 | low-level work of fixing and maintaining the driver which, however beneficial for the 106 | greater community, would stand against the own goals of SWDT project. 107 | 108 | ### Microsoft Virtualization API-s for Windows and Linux 109 | 110 | Microsoft offers [plenty of options](https://learn.microsoft.com/en-us/virtualization/api/), 111 | but it looks like only the HCS API is feature-complete for management of VM lifecycle. 112 | Additionally, HCN API would help to initially setup VNet. 113 | 114 | This, however, is a low-level option which will increase complexity and skill requirements 115 | what in turn may work against making SWDT a project that is attractive and accessible to 116 | new Kubernetes contributors and testers. 117 | 118 | ### unattended.xml for Windows only 119 | 120 | Although it's an old school and tedious solution, it actually is fairly easy to reason about. 121 | This may solve handling of non-interactive OS installation for building Windows node. 122 | 123 | ### Windows Server as VHD 124 | 125 | Microsoft offers VHD - 9 GB to download - which is actually a pre-installed OS, 126 | so it could potentially solve the OS installation issue. 127 | 128 | ### PowerShell Direct for Windows only (?) 129 | 130 | A powerful high-level solution for day two provisioning of Windows VM, [regardless of network configuration](https://learn.microsoft.com/en-us/virtualization/hyper-v-on-windows/user-guide/powershell-direct) 131 | For example, setup container runtime, CNI, Kubernetes, etc. and this actually is what 132 | I used in my experiments in https://github.com/mloskot/swdt-nextgen 133 | 134 | My research suggests me, that PowerShell seems to be de-facto a communication of 135 | choice for managing VM-s from Go apps, for example: 136 | 137 | - https://github.com/taliesins/terraform-provider-hyperv/ 138 | - https://github.com/hashicorp/packer-plugin-hyperv 139 | 140 | ### SSH 141 | 142 | Similarly to PowerShell, SSH can be used for day two provisioning. 143 | In fact, I have used it in my `swdt-nextgen` experiments for setting up Linux node. 144 | I have successfully tested it with Windows too, as alternative to PowerShell Direct, 145 | but it is more fiddly and fragile than PowerShell (arguments handling, escaping,. 146 | outputs manipulation, etc. may become PITA) 147 | 148 | ### Ansible 149 | 150 | Whatever it can do, I, personally, would rather fail trying to fix 151 | libvirt driver for Hyper-V :) 152 | 153 | ### WSL 154 | 155 | I have no idea if and how it could help, but I thought it is worth to mention it. 156 | Perhaps WSL could be used as Linux node? History of WSL networking issues scares me. 157 | Regardless of what the WSL can do for SWDT, it will be best if SWDT aims for 158 | simple Linux and Windows native solutions, VM-s or bare metal hosts. 159 | 160 | ### Vagrant 3.0 161 | 162 | IFF we resist to keep VM management as one of features of SWDT, 163 | then, perhaps, we should revisit the use of Vagrant, but instead of 164 | relying on a custom image/box, we should ensure SWDT can work with 165 | vanilla Linux and Windows Server images available out there. 166 | 167 | The compelling reason to stay with Vagrant is their promise to deliver 168 | [Vagrant 3.0](https://www.hashicorp.com/resources/the-future-of-vagrant-toward-3-0) 169 | in Go. This would open opportunity to write SWDT-specific plugins, in Go, 170 | should we discover a need for that. The problem is that HashiCorp seems to 171 | be far from delivering the Vagrant 3.0. 172 | 173 | Vagrant is certainly a unified API and a facade removing lots of virtualization complexity. 174 | 175 | ## Dream Simplified 176 | 177 | *All credit for what follows below goes to Amim, for his idea of lean approach to SWDT.* 178 | 179 | 1. User creates VM-s however she likes, at least two, one Linux and one Windows, 180 | but according to certain well-documented basic requirements in order to make 181 | VM-s viable as nodes. 182 | For example: 183 | - Minimal installation of Linux Debian or Fedora 184 | - Minimal installation of Windows Server 2022 185 | - virtual switches created 186 | - static IP-s assigned 187 | - SSH servers installed on all nodes 188 | - public SSH keys deployed for password-less SSH communication nodes 189 | - password set for local `Administrator` user on Windows nodes, 190 | in case `swdt` needs to run PowerShell Direct (i.e. SSH comm runs short) 191 | - CNI configuration decided (e.g. pod CIDR) 192 | 193 | 2. User writes the details of the VM-s in form of simple YAML, in `my-awesome-cluster.yaml` 194 | 195 | Alternatively, a super friendly mode, user runs 196 | `swdt config create --output my-awesome-cluster.yaml` 197 | and a beautiful bubbly TUI asks the user sequence of questions, 198 | then generates the YAML ;) 199 | 200 | 3. User runs `swdt cluster create --config my-awesome-cluster.yaml` 201 | 202 | 4. The `swdt` takes over the nodes and does [what is necessary](https://github.com/mloskot/swdt-nextgen) 203 | to create control plane and join worker node(s): 204 | 205 | - optimises system level configuration: disables swap, loads kernel modules, 206 | enables iptables features, disables Windows Firewall 207 | - edits `hosts` files for hostname-based node-to-node communication 208 | - installs containerd or other CRI specified by user's configuration and supported by SWDT 209 | - installs CNI specified by user's configuration and supported by SWDT 210 | - installs Kubernetes 211 | - runs tests 212 | 213 | Of course, software components like containerd and Kubernetes can be specified as 214 | "build this from source from this tag for me, please", then. 215 | 216 | 5. The `swdt` offers day two commands too: 217 | 218 | - `swdt get kubeconfig --output ./.kubeconfig` 219 | - `swdt get nodes` and `swdt get pods` as convenient shortcuts that do not require host-local `.kubeconfig` 220 | - `swdt node stop|start` 221 | - `swdt cluster update --config my-new-node-spec-here.yaml` that is adding new things should be supported, but reconfiguring existing setup like network should not 222 | 223 | Stretching the dream further, it would be awesome if there was `swdt cluster destroy` 224 | reverting all the `swdt config create` changes leaving the VM-s cleaned up. 225 | If we figure a simple usable VM management API, then `swdt` could use snapshots/checkpoints, 226 | even if it does not provide a complete VM lifecycle management. 227 | 228 | The major benefits of the simplified (lean) approach: 229 | 230 | - Clears up almost all of the current issues discussed above. 231 | - Avoids over-engineering SWDT. 232 | - Potentially, may even become virtualization-agnostic and, in the step 1. above, 233 | could allow bare metal hosts for nodes - it is all about networking after all. 234 | 235 | ## Summary 236 | 237 | The SWDT, as explained above, falls into a category of auxiliary projects, 238 | so it has slim chances to become a rockstar of Kubernetes distributors. 239 | However, there are plenty of reasons to make SWDT a well-designed product, 240 | useful to attack real problems and pleasant to work with and contribute to. 241 | 242 | It is important that community involved in SIG-Windows agree upon common 243 | goals and features, so they find SWDT usable for their own tasks. 244 | Otherwise, the project will become deprecated sooner than it is released. 245 | -------------------------------------------------------------------------------- /experiments/swdt/.gitignore: -------------------------------------------------------------------------------- 1 | bin/ -------------------------------------------------------------------------------- /experiments/swdt/LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | 3 | Version 2.0, January 2004 4 | 5 | http://www.apache.org/licenses/ 6 | 7 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 8 | 9 | 1. Definitions. 10 | 11 | "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. 16 | 17 | "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. 18 | 19 | "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. 20 | 21 | "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. 22 | 23 | "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). 24 | 25 | "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. 26 | 27 | "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." 28 | 29 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 30 | 31 | 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 32 | 33 | 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 34 | 35 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: 36 | 37 | You must give any other recipients of the Work or Derivative Works a copy of this License; and 38 | You must cause any modified files to carry prominent notices stating that You changed the files; and 39 | You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 40 | If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. 41 | 42 | You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 43 | 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 44 | 45 | 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 46 | 47 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 48 | 49 | 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 50 | 51 | 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. 52 | 53 | END OF TERMS AND CONDITIONS 54 | -------------------------------------------------------------------------------- /experiments/swdt/Makefile: -------------------------------------------------------------------------------- 1 | 2 | # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) 3 | ifeq (,$(shell go env GOBIN)) 4 | GOBIN=$(shell go env GOPATH)/bin 5 | else 6 | GOBIN=$(shell go env GOBIN) 7 | endif 8 | 9 | # CONTAINER_TOOL defines the container tool to be used for building images. 10 | # Be aware that the target commands are only tested with Docker which is 11 | # scaffolded by default. However, you might want to replace it to use other 12 | # tools. (i.e. podman) 13 | CONTAINER_TOOL ?= docker 14 | 15 | # Setting SHELL to bash allows bash commands to be executed by recipes. 16 | # Options are set to exit when a recipe line exits non-zero or a piped command fails. 17 | SHELL = /usr/bin/env bash -o pipefail 18 | .SHELLFLAGS = -ec 19 | 20 | .PHONY: all 21 | all: build 22 | 23 | ## Location to install dependencies to 24 | LOCALBIN ?= $(shell pwd)/bin 25 | $(LOCALBIN): 26 | mkdir -p $(LOCALBIN) 27 | 28 | ## Tool Versions 29 | KUSTOMIZE_VERSION ?= v5.0.1 30 | CONTROLLER_TOOLS_VERSION ?= v0.12.0 31 | 32 | CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen 33 | 34 | ##@ General 35 | 36 | .PHONY: help 37 | help: ## Display this help. 38 | @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) 39 | 40 | .PHONY: controller-gen 41 | controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. If wrong version is installed, it will be overwritten. 42 | $(CONTROLLER_GEN): $(LOCALBIN) 43 | test -s $(LOCALBIN)/controller-gen && $(LOCALBIN)/controller-gen --version | grep -q $(CONTROLLER_TOOLS_VERSION) || \ 44 | GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) 45 | 46 | .PHONY: generate 47 | generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. 48 | $(CONTROLLER_GEN) object:headerFile="./hack/boilerplate.go.txt" paths="./..." 49 | 50 | .PHONY: test 51 | test: 52 | go test ./... -v 2 53 | 54 | .PHONY: build 55 | build: 56 | go build -o swdt . 57 | -------------------------------------------------------------------------------- /experiments/swdt/README.md: -------------------------------------------------------------------------------- 1 | # Design document for the SWDT CLI 2 | 3 | This document is a reference for decisions regarding multi-OS support architecture. It is developed with Cobra/spf13 to conform to a stable namespace and different connection libraries to support remote command execution. 4 | 5 | Follow the [docs](../../docs) for more information. 6 | 7 | ## Subcommands and namespace 8 | 9 | These are the current supported subcommands for the program: 10 | 11 | * `swdt setup` 12 | * Initialize the node setup, enabling RDP, installing Choco and packages, etc. 13 | * `swdt kubernetes` 14 | * Deploy Kubernetes binaries from the HTTP server indicated in the configuration. 15 | * `swdt readiness` 16 | * Run the [windows operational readiness](https://github.com/kubernetes-sigs/windows-operational-readiness) project in the local cluster 17 | 18 | ## Configuration 19 | 20 | The configuration API follows the GVK (GroupVersionKind) model from Kubernetes, using api-machinery for marshaling and unmarshalling, as well as defaulting values and validating its content. The goal of reusing this API is to enable sharing of the data structure not only with the CLI, but also with controllers and other projects in a well-known and agreed-upon format. 21 | 22 | The project contains a Makefile with targets to generate the required API management and boilerplate functions 23 | 24 | ```mermaid 25 | flowchart TD 26 | 27 | U(Admin) -->|writes CR| A(Kind: Config) 28 | A(Kind: Config) -->|parsed by| B(CLI / decoder) 29 | A(Kind: Config) -->|parsed by| C(Controller) 30 | 31 | B(CLI / decoder) -->|used by| D(business logic / pkg) 32 | C(Controller) -->|used by| D(business logic / pkg) 33 | ``` 34 | 35 | ### Fields and schema 36 | 37 | These are the supported fields for configuring access node credentials, setting up options, and providing Kubernetes binaries information. 38 | 39 | * Credentials: support for SSH and WinRM settings 40 | * Setup: Define options for initial node bootstrap 41 | * Kubernetes: Auxiliary Kubernetes binaries installation 42 | 43 | Following a configuration sample: 44 | 45 | ``` 46 | apiVersion: windows.k8s.io/v1alpha1 47 | kind: Node 48 | metadata: 49 | name: sample 50 | spec: 51 | credentials: 52 | username: Administrator 53 | hostname: 192.168.122.220:22 54 | publicKey: 55 | setup: 56 | enableRDP: true 57 | chocoPackages: 58 | - vim 59 | - grep 60 | kubernetes: 61 | provisioners: 62 | - name: containerd: 63 | version: 1.7.11 64 | sourceURL: http://xyz/containerd.exe 65 | destination: c:\Program Files\containerd\bin\containerd.exe 66 | overwrite: true 67 | - name: kubelet 68 | version: 1.29.0 69 | sourceURL: http://xyz/kubelet.exe 70 | destination: c:\k\kubelet.exe 71 | overwrite: true 72 | ``` 73 | 74 | ## Connections 75 | 76 | Currently, the project SSH for running commands remotely on the node. The common fields required are username and hostname. To proceed, ssh object content should be filled out with the proper connections parameters. 77 | 78 | ## Testing 79 | 80 | See [experimental early guide for testers](samples/mloskot/README.windows.md) 81 | dedicated to try SWDT CLI on Windows host. 82 | -------------------------------------------------------------------------------- /experiments/swdt/apis/config/v1alpha1/config_types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | ) 22 | 23 | type ProvisionerSpec struct { 24 | // Name of the service to be deployed 25 | Name string `json:"name,omitempty"` 26 | 27 | // Version is the binary version to be deployed 28 | Version string `json:"version,omitempty"` 29 | 30 | // SourceURL set the HTTP server to be downloaded from 31 | SourceURL string `json:"sourceURL,omitempty"` 32 | 33 | // Destination set the Windows patch to upload the file 34 | Destination string `json:"destination,omitempty"` 35 | 36 | // Overwrite delete the old file if exists first. 37 | Overwrite bool `json:"overwrite,omitempty"` 38 | } 39 | 40 | type KubernetesSpec struct { 41 | // Provisioners list the objects to be deployed 42 | Provisioners []ProvisionerSpec `json:"provisioners"` 43 | } 44 | 45 | type CredentialsSpec struct { 46 | // Username set the Windows user 47 | Username string `json:"username,omitempty"` 48 | 49 | // Hostname set the Windows node endpoint 50 | Hostname string `json:"hostname,omitempty"` 51 | 52 | // Password is the SSH password for this user 53 | Password string `json:"password,omitempty"` 54 | 55 | // PrivateKey is the SSH private path for this user 56 | PrivateKey string `json:"privateKey,omitempty"` 57 | } 58 | 59 | type SetupSpec struct { 60 | // EnableRDP set up the remote desktop service and enable firewall for it. 61 | EnableRDP *bool `json:"enableRDP"` 62 | 63 | // ChocoPackages provides a list of packages automatically installed in the node. 64 | ChocoPackages *[]string `json:"chocoPackages,omitempty"` 65 | } 66 | 67 | // NodeSpec defines the desired state of Node 68 | type NodeSpec struct { 69 | Cred CredentialsSpec `json:"credentials,omitempty"` 70 | Setup SetupSpec `json:"setup,omitempty"` 71 | Kubernetes KubernetesSpec `json:"kubernetes,omitempty"` 72 | } 73 | 74 | // NodeStatus -- tbd 75 | type NodeStatus struct { 76 | } 77 | 78 | //+kubebuilder:object:root=true 79 | //+kubebuilder:subresource:status 80 | //+k8s:defaulter-gen=true 81 | 82 | // Node is the Schema for the configs API 83 | type Node struct { 84 | metav1.TypeMeta `json:",inline"` 85 | metav1.ObjectMeta `json:"metadata,omitempty"` 86 | 87 | Spec NodeSpec `json:"spec,omitempty"` 88 | Status NodeStatus `json:"status,omitempty"` 89 | } 90 | 91 | //+kubebuilder:object:root=true 92 | 93 | // NodeList contains a list of Node 94 | type NodeList struct { 95 | metav1.TypeMeta `json:",inline"` 96 | metav1.ListMeta `json:"metadata,omitempty"` 97 | Items []Node `json:"items"` 98 | } 99 | 100 | func init() { 101 | SchemeBuilder.Register(&Node{}, &NodeList{}) 102 | } 103 | -------------------------------------------------------------------------------- /experiments/swdt/apis/config/v1alpha1/defaults.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | var defaultTrue = true 20 | 21 | // todo(knabben): this is not the best approach and a workaround for the CLI. 22 | 23 | // Defaults must be called to fill out the empty values 24 | func (c *NodeSpec) Defaults() { 25 | if c.Setup.EnableRDP == nil { 26 | c.Setup.EnableRDP = &defaultTrue 27 | } 28 | if c.Setup.ChocoPackages == nil { 29 | c.Setup.ChocoPackages = &[]string{} 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /experiments/swdt/apis/config/v1alpha1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package v1alpha1 contains API Schema definitions for the windows v1alpha1 API group 18 | // +kubebuilder:object:generate=true 19 | // +groupName=windows.sigs.k8s.io 20 | package v1alpha1 21 | 22 | import ( 23 | "k8s.io/apimachinery/pkg/runtime/schema" 24 | "sigs.k8s.io/controller-runtime/pkg/scheme" 25 | ) 26 | 27 | var ( 28 | // GroupVersion is group version used to register these objects 29 | GroupVersion = schema.GroupVersion{Group: "windows.k8s.io", Version: "v1alpha1"} 30 | 31 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 32 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 33 | 34 | // AddToScheme adds the types in this group-version to the given scheme. 35 | AddToScheme = SchemeBuilder.AddToScheme 36 | ) 37 | -------------------------------------------------------------------------------- /experiments/swdt/apis/config/v1alpha1/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | /* 5 | Copyright 2024 The Kubernetes Authors. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | 20 | // Code generated by controller-gen. DO NOT EDIT. 21 | 22 | package v1alpha1 23 | 24 | import ( 25 | runtime "k8s.io/apimachinery/pkg/runtime" 26 | ) 27 | 28 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 29 | func (in *CredentialsSpec) DeepCopyInto(out *CredentialsSpec) { 30 | *out = *in 31 | } 32 | 33 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CredentialsSpec. 34 | func (in *CredentialsSpec) DeepCopy() *CredentialsSpec { 35 | if in == nil { 36 | return nil 37 | } 38 | out := new(CredentialsSpec) 39 | in.DeepCopyInto(out) 40 | return out 41 | } 42 | 43 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 44 | func (in *KubernetesSpec) DeepCopyInto(out *KubernetesSpec) { 45 | *out = *in 46 | if in.Provisioners != nil { 47 | in, out := &in.Provisioners, &out.Provisioners 48 | *out = make([]ProvisionerSpec, len(*in)) 49 | copy(*out, *in) 50 | } 51 | } 52 | 53 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesSpec. 54 | func (in *KubernetesSpec) DeepCopy() *KubernetesSpec { 55 | if in == nil { 56 | return nil 57 | } 58 | out := new(KubernetesSpec) 59 | in.DeepCopyInto(out) 60 | return out 61 | } 62 | 63 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 64 | func (in *Node) DeepCopyInto(out *Node) { 65 | *out = *in 66 | out.TypeMeta = in.TypeMeta 67 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 68 | in.Spec.DeepCopyInto(&out.Spec) 69 | out.Status = in.Status 70 | } 71 | 72 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Node. 73 | func (in *Node) DeepCopy() *Node { 74 | if in == nil { 75 | return nil 76 | } 77 | out := new(Node) 78 | in.DeepCopyInto(out) 79 | return out 80 | } 81 | 82 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 83 | func (in *Node) DeepCopyObject() runtime.Object { 84 | if c := in.DeepCopy(); c != nil { 85 | return c 86 | } 87 | return nil 88 | } 89 | 90 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 91 | func (in *NodeList) DeepCopyInto(out *NodeList) { 92 | *out = *in 93 | out.TypeMeta = in.TypeMeta 94 | in.ListMeta.DeepCopyInto(&out.ListMeta) 95 | if in.Items != nil { 96 | in, out := &in.Items, &out.Items 97 | *out = make([]Node, len(*in)) 98 | for i := range *in { 99 | (*in)[i].DeepCopyInto(&(*out)[i]) 100 | } 101 | } 102 | } 103 | 104 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeList. 105 | func (in *NodeList) DeepCopy() *NodeList { 106 | if in == nil { 107 | return nil 108 | } 109 | out := new(NodeList) 110 | in.DeepCopyInto(out) 111 | return out 112 | } 113 | 114 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 115 | func (in *NodeList) DeepCopyObject() runtime.Object { 116 | if c := in.DeepCopy(); c != nil { 117 | return c 118 | } 119 | return nil 120 | } 121 | 122 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 123 | func (in *NodeSpec) DeepCopyInto(out *NodeSpec) { 124 | *out = *in 125 | out.Cred = in.Cred 126 | in.Setup.DeepCopyInto(&out.Setup) 127 | in.Kubernetes.DeepCopyInto(&out.Kubernetes) 128 | } 129 | 130 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeSpec. 131 | func (in *NodeSpec) DeepCopy() *NodeSpec { 132 | if in == nil { 133 | return nil 134 | } 135 | out := new(NodeSpec) 136 | in.DeepCopyInto(out) 137 | return out 138 | } 139 | 140 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 141 | func (in *NodeStatus) DeepCopyInto(out *NodeStatus) { 142 | *out = *in 143 | } 144 | 145 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeStatus. 146 | func (in *NodeStatus) DeepCopy() *NodeStatus { 147 | if in == nil { 148 | return nil 149 | } 150 | out := new(NodeStatus) 151 | in.DeepCopyInto(out) 152 | return out 153 | } 154 | 155 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 156 | func (in *ProvisionerSpec) DeepCopyInto(out *ProvisionerSpec) { 157 | *out = *in 158 | } 159 | 160 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProvisionerSpec. 161 | func (in *ProvisionerSpec) DeepCopy() *ProvisionerSpec { 162 | if in == nil { 163 | return nil 164 | } 165 | out := new(ProvisionerSpec) 166 | in.DeepCopyInto(out) 167 | return out 168 | } 169 | 170 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 171 | func (in *SetupSpec) DeepCopyInto(out *SetupSpec) { 172 | *out = *in 173 | if in.EnableRDP != nil { 174 | in, out := &in.EnableRDP, &out.EnableRDP 175 | *out = new(bool) 176 | **out = **in 177 | } 178 | if in.ChocoPackages != nil { 179 | in, out := &in.ChocoPackages, &out.ChocoPackages 180 | *out = new([]string) 181 | if **in != nil { 182 | in, out := *in, *out 183 | *out = make([]string, len(*in)) 184 | copy(*out, *in) 185 | } 186 | } 187 | } 188 | 189 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SetupSpec. 190 | func (in *SetupSpec) DeepCopy() *SetupSpec { 191 | if in == nil { 192 | return nil 193 | } 194 | out := new(SetupSpec) 195 | in.DeepCopyInto(out) 196 | return out 197 | } 198 | -------------------------------------------------------------------------------- /experiments/swdt/apis/config/v1alpha1/zz_generated.defaults.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | /* 5 | Copyright 2024 The Kubernetes Authors. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | // Code generated by defaulter-gen. DO NOT EDIT. 20 | 21 | package v1alpha1 22 | 23 | import ( 24 | runtime "k8s.io/apimachinery/pkg/runtime" 25 | ) 26 | 27 | // RegisterDefaults adds defaulters functions to the given scheme. 28 | // Public to allow building arbitrary schemes. 29 | // All generated defaulters are covering - they call all nested defaulters. 30 | func RegisterDefaults(scheme *runtime.Scheme) error { 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /experiments/swdt/cmd/kubernetes.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cmd 18 | 19 | import ( 20 | "swdt/apis/config/v1alpha1" 21 | "swdt/pkg/pwsh/executor" 22 | "swdt/pkg/pwsh/kubernetes" 23 | 24 | "github.com/spf13/cobra" 25 | ) 26 | 27 | // setupCmd represents the setup command 28 | var kubernetesCmd = &cobra.Command{ 29 | Use: "kubernetes", 30 | Short: "Provision Kubernetes binaries into a running node", 31 | Long: `Provision Kubernetes binaries into a running node`, 32 | RunE: RunKubernetes, 33 | } 34 | 35 | func RunKubernetes(cmd *cobra.Command, args []string) error { 36 | var ( 37 | err error 38 | nodeConfig *v1alpha1.Node 39 | ) 40 | if nodeConfig, err = loadConfiguration(cmd); err != nil { 41 | return err 42 | } 43 | 44 | runner, err := executor.NewRunner(nodeConfig, &kubernetes.KubernetesRunner{}) 45 | if err != nil { 46 | return err 47 | } 48 | defer runner.CloseConnection() // nolint 49 | 50 | return runner.Inner.InstallProvisioners(nodeConfig.Spec.Kubernetes.Provisioners) 51 | } 52 | -------------------------------------------------------------------------------- /experiments/swdt/cmd/root.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cmd 18 | 19 | import ( 20 | "swdt/apis/config/v1alpha1" 21 | "swdt/pkg/config" 22 | 23 | "github.com/spf13/cobra" 24 | 25 | "k8s.io/component-base/featuregate" 26 | logsapi "k8s.io/component-base/logs/api/v1" 27 | ) 28 | 29 | var featureGate = featuregate.NewFeatureGate() 30 | 31 | // NewRootCommand creates the `swdt` command and its nested children. 32 | func NewRootCommand() *cobra.Command { 33 | logscfg := logsapi.NewLoggingConfiguration() 34 | if err := logsapi.AddFeatureGates(featureGate); err != nil { 35 | panic(err) 36 | } 37 | if err := logsapi.ValidateAndApply(logscfg, featureGate); err != nil { 38 | panic(err) 39 | } 40 | 41 | cmd := &cobra.Command{ 42 | Use: "swdt", 43 | Short: "SIG Windows Development Tools", 44 | Long: `Auxiliary program for Windows nodes installation and initial setup. 45 | Check the subcommands.`, 46 | } 47 | 48 | featureGate.AddFlag(cmd.Flags()) 49 | logsapi.AddFlags(logscfg, cmd.Flags()) 50 | 51 | cmd.PersistentFlags().StringP("config", "c", "samples/config.yaml", "Configuration file path.") 52 | 53 | cmd.AddCommand(setupCmd) 54 | cmd.AddCommand(kubernetesCmd) 55 | 56 | return cmd 57 | } 58 | 59 | // loadConfiguration marshal the YAML configuration in an internal struct 60 | func loadConfiguration(cmd *cobra.Command) (*v1alpha1.Node, error) { 61 | return config.LoadConfigNodeFromFile(cmd.Flag("config").Value.String()) 62 | } 63 | -------------------------------------------------------------------------------- /experiments/swdt/cmd/setup.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cmd 18 | 19 | import ( 20 | "swdt/apis/config/v1alpha1" 21 | "swdt/pkg/pwsh/executor" 22 | "swdt/pkg/pwsh/setup" 23 | 24 | "github.com/spf13/cobra" 25 | ) 26 | 27 | // setupCmd represents the setup command 28 | var setupCmd = &cobra.Command{ 29 | Use: "setup", 30 | Short: "Bootstrap the node via basic unit setup", 31 | Long: `Bootstrap the node via basic unit setup`, 32 | RunE: RunSetup, 33 | } 34 | 35 | func RunSetup(cmd *cobra.Command, args []string) error { 36 | var ( 37 | err error 38 | nodeConfig *v1alpha1.Node 39 | ) 40 | 41 | if nodeConfig, err = loadConfiguration(cmd); err != nil { 42 | return err 43 | } 44 | 45 | runner, err := executor.NewRunner(nodeConfig, &setup.SetupRunner{}) 46 | if err != nil { 47 | return err 48 | } 49 | defer runner.CloseConnection() // nolint 50 | 51 | // Install choco binary 52 | if err = runner.Inner.InstallChoco(); err != nil { 53 | return err 54 | } 55 | 56 | // Install Choco packages from the input list 57 | if err = runner.Inner.InstallChocoPackages(*nodeConfig.Spec.Setup.ChocoPackages); err != nil { 58 | return err 59 | } 60 | 61 | // Enable RDP if the option is true 62 | return runner.Inner.EnableRDP(*nodeConfig.Spec.Setup.EnableRDP) 63 | } 64 | -------------------------------------------------------------------------------- /experiments/swdt/go.mod: -------------------------------------------------------------------------------- 1 | module swdt 2 | 3 | go 1.21.1 4 | 5 | require ( 6 | github.com/bramvdbogaerde/go-scp v1.2.1 7 | github.com/fatih/color v1.16.0 8 | github.com/spf13/cobra v1.8.0 9 | github.com/stretchr/testify v1.8.4 10 | golang.org/x/crypto v0.14.0 11 | golang.org/x/term v0.13.0 12 | k8s.io/apimachinery v0.29.0 13 | k8s.io/component-base v0.28.3 14 | k8s.io/klog/v2 v2.110.1 15 | sigs.k8s.io/controller-runtime v0.16.3 16 | ) 17 | 18 | require ( 19 | github.com/beorn7/perks v1.0.1 // indirect 20 | github.com/blang/semver/v4 v4.0.0 // indirect 21 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 22 | github.com/davecgh/go-spew v1.1.1 // indirect 23 | github.com/go-logr/logr v1.3.0 // indirect 24 | github.com/gogo/protobuf v1.3.2 // indirect 25 | github.com/golang/protobuf v1.5.3 // indirect 26 | github.com/google/go-cmp v0.6.0 // indirect 27 | github.com/google/gofuzz v1.2.0 // indirect 28 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 29 | github.com/json-iterator/go v1.1.12 // indirect 30 | github.com/kr/text v0.2.0 // indirect 31 | github.com/mattn/go-colorable v0.1.13 // indirect 32 | github.com/mattn/go-isatty v0.0.20 // indirect 33 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 34 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 35 | github.com/modern-go/reflect2 v1.0.2 // indirect 36 | github.com/pmezard/go-difflib v1.0.0 // indirect 37 | github.com/prometheus/client_golang v1.16.0 // indirect 38 | github.com/prometheus/client_model v0.4.0 // indirect 39 | github.com/prometheus/common v0.44.0 // indirect 40 | github.com/prometheus/procfs v0.10.1 // indirect 41 | github.com/spf13/pflag v1.0.5 // indirect 42 | golang.org/x/net v0.17.0 // indirect 43 | golang.org/x/sys v0.14.0 // indirect 44 | golang.org/x/text v0.13.0 // indirect 45 | google.golang.org/protobuf v1.31.0 // indirect 46 | gopkg.in/inf.v0 v0.9.1 // indirect 47 | gopkg.in/yaml.v2 v2.4.0 // indirect 48 | gopkg.in/yaml.v3 v3.0.1 // indirect 49 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect 50 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 51 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect 52 | sigs.k8s.io/yaml v1.3.0 // indirect 53 | ) 54 | -------------------------------------------------------------------------------- /experiments/swdt/go.sum: -------------------------------------------------------------------------------- 1 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 2 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 3 | github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= 4 | github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= 5 | github.com/bramvdbogaerde/go-scp v1.2.1 h1:BKTqrqXiQYovrDlfuVFaEGz0r4Ou6EED8L7jCXw6Buw= 6 | github.com/bramvdbogaerde/go-scp v1.2.1/go.mod h1:s4ZldBoRAOgUg8IrRP2Urmq5qqd2yPXQTPshACY8vQ0= 7 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 8 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 9 | github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 10 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 11 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 13 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 | github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= 15 | github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= 16 | github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= 17 | github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 18 | github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= 19 | github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= 20 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= 21 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= 22 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 23 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 24 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 25 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 26 | github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= 27 | github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 28 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 29 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 30 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 31 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 32 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 33 | github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= 34 | github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 35 | github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= 36 | github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= 37 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 38 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 39 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 40 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 41 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 42 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 43 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 44 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 45 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 46 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 47 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 48 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 49 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 50 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 51 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 52 | github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= 53 | github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= 54 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 55 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 56 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 57 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 58 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 59 | github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= 60 | github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= 61 | github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= 62 | github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= 63 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 64 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 65 | github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= 66 | github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= 67 | github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= 68 | github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= 69 | github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= 70 | github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= 71 | github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= 72 | github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= 73 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 74 | github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= 75 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 76 | github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= 77 | github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= 78 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 79 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 80 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 81 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 82 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= 83 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 84 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 85 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 86 | go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= 87 | go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= 88 | go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= 89 | go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= 90 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 91 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 92 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 93 | golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= 94 | golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= 95 | golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= 96 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 97 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 98 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 99 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 100 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 101 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 102 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 103 | golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= 104 | golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= 105 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 106 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 107 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 108 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 109 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 110 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 111 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 112 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 113 | golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 114 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 115 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 116 | golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= 117 | golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 118 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 119 | golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= 120 | golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= 121 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 122 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 123 | golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= 124 | golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= 125 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 126 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 127 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 128 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 129 | golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= 130 | golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= 131 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 132 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 133 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 134 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 135 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 136 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 137 | google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= 138 | google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 139 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 140 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 141 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 142 | gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= 143 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 144 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 145 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 146 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 147 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 148 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 149 | k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM= 150 | k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc= 151 | k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o= 152 | k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis= 153 | k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= 154 | k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= 155 | k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= 156 | k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= 157 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= 158 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= 159 | sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= 160 | sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= 161 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= 162 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= 163 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= 164 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= 165 | sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= 166 | sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= 167 | -------------------------------------------------------------------------------- /experiments/swdt/hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ -------------------------------------------------------------------------------- /experiments/swdt/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "os" 21 | "swdt/cmd" 22 | 23 | "k8s.io/apimachinery/pkg/util/runtime" 24 | "k8s.io/component-base/cli" 25 | "k8s.io/component-base/featuregate" 26 | logsapi "k8s.io/component-base/logs/api/v1" 27 | ) 28 | 29 | var featureGate = featuregate.NewFeatureGate() 30 | 31 | func main() { 32 | runtime.Must(logsapi.AddFeatureGates(featureGate)) 33 | command := cmd.NewRootCommand() 34 | code := cli.Run(command) 35 | os.Exit(code) 36 | } 37 | -------------------------------------------------------------------------------- /experiments/swdt/packer/.gitignore: -------------------------------------------------------------------------------- 1 | output/ 2 | kvm/isos/ 3 | -------------------------------------------------------------------------------- /experiments/swdt/packer/Makefile: -------------------------------------------------------------------------------- 1 | start: 2 | packer init kvm 3 | PACKER_LOG=1 packer build kvm 4 | -------------------------------------------------------------------------------- /experiments/swdt/packer/README.md: -------------------------------------------------------------------------------- 1 | ## Packer VM image builder 2 | 3 | This folder hosts the plain boot and automatic installation scripts 4 | using packer, the final outcome is the qemu artifact ready to be used 5 | as a VM for swdt with SSH enabled. 6 | 7 | Pre-requisites: 8 | 9 | * Hashicorp Packer >=1.10.2 10 | 11 | 2 ISOs are required, save them on isos folder: 12 | 13 | * **window.iso** - [Windows 2022 Server](https://www.microsoft.com/en-us/evalcenter/evaluate-windows-server-2022) 14 | * **virtio-win.iso** - [Windows Virtio Drivers](https://github.com/virtio-win/virtio-win-pkg-scripts/blob/master/README.md) 15 | 16 | ### Running 17 | 18 | ```shell 19 | make start 20 | ``` 21 | 22 | Behind the scenes it will call Packer in the kvm build 23 | 24 | ```shell 25 | packer init kvm 26 | PACKER_LOG=1 packer build kvm 27 | ``` 28 | 29 | ### Export 30 | 31 | The folder `output` will contain the `win2k22` QEMU QCOW Image. 32 | 33 | -------------------------------------------------------------------------------- /experiments/swdt/packer/kvm/floppy/autounattend.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | 8 | 9 | a:\ 10 | 11 | 12 | E:\STORAGE\SERVER_2008\AMD64 13 | 14 | 15 | E:\NETWORK\SERVER_2008\AMD64 16 | 17 | 18 | 19 | 22 | 23 | 24 | 0 25 | true 26 | 27 | 28 | 1 29 | 350 30 | Primary 31 | 32 | 33 | 2 34 | true 35 | Primary 36 | 37 | 38 | 39 | 40 | NTFS 41 | 42 | 1 43 | 1 44 | 0x27 45 | 46 | 47 | 2 48 | 2 49 | C 50 | 51 | NTFS 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 0 60 | 2 61 | 62 | 63 | 64 | /IMAGE/INDEX 65 | 3 66 | 67 | 68 | OnError 69 | 70 | 71 | 72 | true 73 | Administrator 74 | Organization 75 | 76 | Never 77 | 78 | 79 | true 80 | 81 | 84 | 85 | en-US 86 | 87 | 0409:00000409 88 | en-US 89 | en-US 90 | en-US 91 | en-US 92 | 93 | 94 | 95 | 98 | false 99 | 100 | 101 | 102 | 105 | 1 106 | true 107 | 108 | 109 | 110 | 113 | 0409:00000409 114 | en-US 115 | en-US 116 | en-US 117 | en-US 118 | 119 | 122 | true 123 | 124 | 127 | 0 128 | 129 | 132 | win2k22 133 | 134 | 135 | 136 | 139 | 140 | 141 | S3cr3t0! 142 | true</PlainText> 143 | </Password> 144 | <Enabled>true</Enabled> 145 | <Username>Administrator</Username> 146 | </AutoLogon> 147 | <FirstLogonCommands> 148 | <SynchronousCommand wcm:action="add"> 149 | <Order>1</Order> 150 | <Description>Set Execution Policy 64 Bit</Description> 151 | <CommandLine>cmd.exe /c powershell -Command "Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Force"</CommandLine> 152 | <RequiresUserInput>true</RequiresUserInput> 153 | </SynchronousCommand> 154 | <SynchronousCommand wcm:action="add"> 155 | <Order>2</Order> 156 | <Description>Set Execution Policy 32 Bit</Description> 157 | <CommandLine>%SystemDrive%\Windows\SysWOW64\cmd.exe /c powershell -Command "Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Force"</CommandLine> 158 | <RequiresUserInput>true</RequiresUserInput> 159 | </SynchronousCommand> 160 | <SynchronousCommand wcm:action="add"> 161 | <CommandLine>%SystemRoot%\System32\reg.exe ADD HKCU\SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\Advanced\ /v HideFileExt /t REG_DWORD /d 0 /f</CommandLine> 162 | <Order>3</Order> 163 | <Description>Show file extensions in Explorer</Description> 164 | </SynchronousCommand> 165 | <SynchronousCommand wcm:action="add"> 166 | <CommandLine>%SystemRoot%\System32\reg.exe ADD HKCU\Console /v QuickEdit /t REG_DWORD /d 1 /f</CommandLine> 167 | <Order>4</Order> 168 | <Description>Enable QuickEdit mode</Description> 169 | </SynchronousCommand> 170 | <SynchronousCommand wcm:action="add"> 171 | <CommandLine>%SystemRoot%\System32\reg.exe ADD HKCU\SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\Advanced\ /v Start_ShowRun /t REG_DWORD /d 1 /f</CommandLine> 172 | <Order>5</Order> 173 | <Description>Show Run command in Start Menu</Description> 174 | </SynchronousCommand> 175 | <SynchronousCommand wcm:action="add"> 176 | <CommandLine>%SystemRoot%\System32\reg.exe ADD HKCU\SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\Advanced\ /v StartMenuAdminTools /t REG_DWORD /d 1 /f</CommandLine> 177 | <Order>6</Order> 178 | <Description>Show Administrative Tools in Start Menu</Description> 179 | </SynchronousCommand> 180 | <SynchronousCommand wcm:action="add"> 181 | <CommandLine>%SystemRoot%\System32\reg.exe ADD HKLM\SYSTEM\CurrentControlSet\Control\Power\ /v HibernateFileSizePercent /t REG_DWORD /d 0 /f</CommandLine> 182 | <Order>7</Order> 183 | <Description>Zero Hibernation File</Description> 184 | </SynchronousCommand> 185 | <SynchronousCommand wcm:action="add"> 186 | <CommandLine>%SystemRoot%\System32\reg.exe ADD HKLM\SYSTEM\CurrentControlSet\Control\Power\ /v HibernateEnabled /t REG_DWORD /d 0 /f</CommandLine> 187 | <Order>8</Order> 188 | <Description>Disable Hibernation Mode</Description> 189 | </SynchronousCommand> 190 | <SynchronousCommand wcm:action="add"> 191 | <CommandLine>cmd.exe /c wmic useraccount where "name='Administrator'" set PasswordExpires=FALSE</CommandLine> 192 | <Order>9</Order> 193 | <Description>Disable password expiration for Administrator user</Description> 194 | </SynchronousCommand> 195 | <SynchronousCommand wcm:action="add"> 196 | <CommandLine>cmd.exe /c %SystemDrive%\Windows\System32\WindowsPowerShell\v1.0\powershell.exe -Command "Set-SConfig -AutoLaunch $false"</CommandLine> 197 | <Description>Disable SCConfig</Description> 198 | <Order>10</Order> 199 | </SynchronousCommand> 200 | <SynchronousCommand wcm:action="add"> 201 | <CommandLine>cmd.exe /c powershell -File "a:\openssh.ps1"</CommandLine> 202 | <Description>Enable SSH</Description> 203 | <Order>11</Order> 204 | </SynchronousCommand> 205 | </FirstLogonCommands> 206 | <OOBE> 207 | <HideEULAPage>true</HideEULAPage> 208 | <HideLocalAccountScreen>true</HideLocalAccountScreen> 209 | <HideOEMRegistrationScreen>true</HideOEMRegistrationScreen> 210 | <HideOnlineAccountScreens>true</HideOnlineAccountScreens> 211 | <HideWirelessSetupInOOBE>true</HideWirelessSetupInOOBE> 212 | <NetworkLocation>Work</NetworkLocation> 213 | <ProtectYourPC>1</ProtectYourPC> 214 | <SkipMachineOOBE>true</SkipMachineOOBE> 215 | <SkipUserOOBE>true</SkipUserOOBE> 216 | </OOBE> 217 | <RegisteredOrganization>Organization</RegisteredOrganization> 218 | <RegisteredOwner>Owner</RegisteredOwner> 219 | <DisableAutoDaylightTimeSet>false</DisableAutoDaylightTimeSet> 220 | <TimeZone>Pacific Standard Time</TimeZone> 221 | <UserAccounts> 222 | <AdministratorPassword> 223 | <Value>S3cr3t0!</Value> 224 | <PlainText>true</PlainText> 225 | </AdministratorPassword> 226 | <LocalAccounts> 227 | <LocalAccount wcm:action="add"> 228 | <Description>Administrator</Description> 229 | <DisplayName>Administrator</DisplayName> 230 | <Group>Administrators</Group> 231 | <Name>Administrator</Name> 232 | </LocalAccount> 233 | </LocalAccounts> 234 | </UserAccounts> 235 | </component> 236 | </settings> 237 | </unattend> 238 | -------------------------------------------------------------------------------- /experiments/swdt/packer/kvm/floppy/openssh.ps1: -------------------------------------------------------------------------------- 1 | echo "starting openssh" >> c:\temp\openssh.log 2 | Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0 3 | Set-Service -Name sshd -StartupType Automatic 4 | Start-Service sshd 5 | -------------------------------------------------------------------------------- /experiments/swdt/packer/kvm/win2022.pkr.hcl: -------------------------------------------------------------------------------- 1 | packer { 2 | required_plugins { 3 | qemu = { 4 | version = "~> 1" 5 | source = "github.com/hashicorp/qemu" 6 | } 7 | } 8 | } 9 | 10 | source "qemu" "windows" { 11 | vm_name = "win2k22" 12 | format = "qcow2" 13 | accelerator = "kvm" 14 | 15 | iso_url = "kvm/isos/windows.iso" 16 | iso_checksum = "sha256:3e4fa6d8507b554856fc9ca6079cc402df11a8b79344871669f0251535255325" 17 | 18 | cpus = 4 19 | memory = 4096 20 | 21 | efi_boot = false 22 | disk_size = "15G" 23 | disk_interface = "virtio" 24 | 25 | floppy_files = ["kvm/floppy/autounattend.xml", "kvm/floppy/openssh.ps1"] 26 | qemuargs = [["-cdrom", "./kvm/isos/virtio-win.iso"]] 27 | 28 | output_directory = "output" 29 | 30 | communicator = "ssh" 31 | ssh_username = "Administrator" 32 | ssh_password = "S3cr3t0!" 33 | ssh_timeout = "1h" 34 | 35 | boot_wait = "10s" 36 | shutdown_command = "shutdown /s /t 30 /f" 37 | shutdown_timeout = "15m" 38 | } 39 | 40 | build { 41 | name = "win2022" 42 | sources = ["source.qemu.windows"] 43 | } 44 | 45 | -------------------------------------------------------------------------------- /experiments/swdt/pkg/config/node.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | 23 | "swdt/apis/config/v1alpha1" 24 | 25 | "k8s.io/apimachinery/pkg/runtime" 26 | "k8s.io/apimachinery/pkg/runtime/serializer" 27 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 28 | klog "k8s.io/klog/v2" 29 | ) 30 | 31 | var ( 32 | scheme = runtime.NewScheme() 33 | codecs = serializer.NewCodecFactory(scheme, serializer.EnableStrict) 34 | ) 35 | 36 | func init() { 37 | utilruntime.Must(v1alpha1.AddToScheme(scheme)) 38 | } 39 | 40 | // LoadConfigNodeFromFile LoadConfigFromFile returns the marshalled Node configuration object 41 | func LoadConfigNodeFromFile(file string) (*v1alpha1.Node, error) { 42 | klog.V(2).Infof("Loading node configuration from '%s'", file) 43 | 44 | data, err := os.ReadFile(file) 45 | if err != nil { 46 | return nil, err 47 | } 48 | return loadConfigNode(data) 49 | } 50 | 51 | // loadConfig decode the input read YAML into a configuration object 52 | func loadConfigNode(data []byte) (*v1alpha1.Node, error) { 53 | var deserializer = codecs.UniversalDeserializer() 54 | configObj, gvk, err := deserializer.Decode(data, nil, nil) 55 | if err != nil { 56 | return nil, err 57 | } 58 | config, ok := configObj.(*v1alpha1.Node) 59 | if !ok { 60 | return nil, fmt.Errorf("got unexpected config type: %v", gvk) 61 | } 62 | config.Spec.Defaults() 63 | return config, nil 64 | } 65 | -------------------------------------------------------------------------------- /experiments/swdt/pkg/config/node_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/stretchr/testify/assert" 23 | ) 24 | 25 | const ( 26 | SAMPLE_FILE = "../../samples/config.yaml" 27 | SAMPLE_DEFAULT = `apiVersion: windows.k8s.io/v1alpha1 28 | kind: Node 29 | metadata: 30 | name: sample 31 | spec:` 32 | ) 33 | 34 | func TestLoadConfigNodeDefaults(t *testing.T) { 35 | config, err := loadConfigNode([]byte(SAMPLE_DEFAULT)) 36 | assert.Nil(t, err) 37 | 38 | assert.True(t, *config.Spec.Setup.EnableRDP) 39 | assert.Len(t, *config.Spec.Setup.ChocoPackages, 0) 40 | } 41 | 42 | func TestLoadConfigNode(t *testing.T) { 43 | config, err := LoadConfigNodeFromFile(SAMPLE_FILE) 44 | assert.Nil(t, err) 45 | 46 | assert.True(t, *config.Spec.Setup.EnableRDP) 47 | assert.Equal(t, len(*config.Spec.Setup.ChocoPackages), 2) 48 | 49 | provisioners := config.Spec.Kubernetes.Provisioners 50 | assert.Len(t, provisioners, 2) 51 | 52 | for _, d := range provisioners { 53 | assert.GreaterOrEqual(t, len(d.SourceURL), 2) 54 | assert.GreaterOrEqual(t, len(d.Destination), 4) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /experiments/swdt/pkg/connections/interface.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package connections 18 | 19 | import ( 20 | "swdt/apis/config/v1alpha1" 21 | ) 22 | 23 | type Connection interface { 24 | // Connect creates the initial connection objects 25 | Connect() error 26 | 27 | // Run execute the command via transport method 28 | Run(args string) (string, error) 29 | 30 | // Copy files from and to the node 31 | Copy(local, remote, perm string) error 32 | 33 | // Close the used connection and sessions 34 | Close() error 35 | } 36 | 37 | func NewConnection(credentials v1alpha1.CredentialsSpec) Connection { 38 | return &SSHConnection{credentials: &credentials} 39 | } 40 | -------------------------------------------------------------------------------- /experiments/swdt/pkg/connections/ssh.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package connections 18 | 19 | import ( 20 | "bytes" 21 | "context" 22 | "errors" 23 | "fmt" 24 | "io" 25 | "os" 26 | "path" 27 | "regexp" 28 | "swdt/apis/config/v1alpha1" 29 | "sync" 30 | "time" 31 | 32 | scp "github.com/bramvdbogaerde/go-scp" 33 | "golang.org/x/crypto/ssh" 34 | 35 | klog "k8s.io/klog/v2" 36 | ) 37 | 38 | const ( 39 | TCP_TYPE = "tcp" 40 | SCP_BINARY = "C:\\Windows\\System32\\OpenSSH\\scp.exe" 41 | timeout time.Duration = 30 * time.Second 42 | ) 43 | 44 | type SSHConnection struct { 45 | credentials *v1alpha1.CredentialsSpec 46 | client *ssh.Client 47 | } 48 | 49 | // fetchAuthMethod fetches all available authentication methods 50 | func (c *SSHConnection) fetchAuthMethod() (authMethod []ssh.AuthMethod, err error) { 51 | var ( 52 | file *os.File 53 | privateKey = c.credentials.PrivateKey 54 | password = c.credentials.Password 55 | content []byte 56 | signer ssh.Signer 57 | ) 58 | if privateKey != "" { 59 | klog.V(2).Infof("SSH authenticating with private key '%s'\n", privateKey) 60 | file, err = os.Open(privateKey) 61 | if err != nil { 62 | return 63 | } 64 | content, err = io.ReadAll(file) 65 | if err != nil { 66 | return 67 | } 68 | signer, err = ssh.ParsePrivateKey(content) 69 | if err != nil { 70 | return 71 | } 72 | authMethod = append(authMethod, ssh.PublicKeys(signer)) 73 | } 74 | if password != "" { 75 | klog.V(2).Info("SSH authenticating with password") 76 | authMethod = append(authMethod, ssh.Password(password)) 77 | } 78 | return 79 | } 80 | 81 | // Connect creates the client connection object 82 | func (c *SSHConnection) Connect() error { 83 | authMethod, err := c.fetchAuthMethod() 84 | if err != nil { 85 | return err 86 | } 87 | 88 | klog.V(2).Infof("SSH connecting to '%s' as '%s'\n", c.credentials.Hostname, c.credentials.Username) 89 | client, err := ssh.Dial(TCP_TYPE, c.credentials.Hostname, &ssh.ClientConfig{ 90 | User: c.credentials.Username, 91 | Auth: authMethod, 92 | HostKeyCallback: ssh.InsecureIgnoreHostKey(), 93 | }) 94 | if err != nil { 95 | return fmt.Errorf("failed to dial: %s", err) 96 | } 97 | c.client = client 98 | return nil 99 | } 100 | 101 | // Run a powershell command passed in the argument 102 | func (c *SSHConnection) Run(args string) (string, error) { 103 | if c.client == nil { 104 | return "", fmt.Errorf("client is empty, call Connect() first") 105 | } 106 | 107 | var ( 108 | err error 109 | session *ssh.Session 110 | b bytes.Buffer 111 | ) 112 | if session, err = c.client.NewSession(); err != nil { 113 | return "", err 114 | } 115 | defer session.Close() 116 | session.Stdout = &b 117 | 118 | // Multiline PowerShell commands over SSH trip over newlines - only first one is executed 119 | args = regexp.MustCompile(`\r?\n`).ReplaceAllLiteralString(args, ";") 120 | cmd := fmt.Sprintf("powershell -nologo -noprofile -c { %s }", args) 121 | klog.V(2).Infof("SSH executing PowerShell command: %s\n", cmd) 122 | if err := session.Run(cmd); err != nil { 123 | return "", err 124 | } 125 | 126 | return b.String(), nil 127 | } 128 | 129 | // Copy a file from local to remote setting the permissions 130 | func (c *SSHConnection) Copy(local, remote, perm string) error { 131 | klog.V(2).Infof("SSH copying local '%s' to remote '%s'\n", local, remote) 132 | file, err := os.Open(local) 133 | if err != nil { 134 | return err 135 | } 136 | var contents []byte 137 | contents, err = io.ReadAll(file) 138 | if err != nil { 139 | return fmt.Errorf("failed to read all data from reader: %w", err) 140 | } 141 | return c.CopyPassThru(bytes.NewReader(contents), remote, perm, int64(len(contents))) 142 | } 143 | 144 | // CopyPassThru is an auxiliary function for Copy 145 | func (c *SSHConnection) CopyPassThru(reader io.Reader, remote string, permissions string, size int64) error { 146 | var ( 147 | ctx = context.Background() 148 | filename = path.Base(remote) 149 | ) 150 | 151 | session, err := c.client.NewSession() 152 | if err != nil { 153 | return err 154 | } 155 | defer session.Close() 156 | 157 | stdout, err := session.StdoutPipe() 158 | if err != nil { 159 | return err 160 | } 161 | writer, err := session.StdinPipe() 162 | if err != nil { 163 | return err 164 | } 165 | defer writer.Close() 166 | 167 | wg := sync.WaitGroup{} 168 | wg.Add(2) 169 | 170 | errCh := make(chan error, 2) 171 | 172 | go func() { 173 | defer wg.Done() 174 | defer writer.Close() 175 | 176 | _, err := fmt.Fprintln(writer, "C"+permissions, size, filename) 177 | if err != nil { 178 | errCh <- err 179 | return 180 | } 181 | 182 | if err = checkResponse(stdout); err != nil { 183 | errCh <- err 184 | return 185 | } 186 | _, err = io.Copy(writer, reader) 187 | if err != nil { 188 | errCh <- err 189 | return 190 | } 191 | 192 | _, err = fmt.Fprint(writer, "\x00") 193 | if err != nil { 194 | errCh <- err 195 | return 196 | } 197 | if err = checkResponse(stdout); err != nil { 198 | errCh <- err 199 | return 200 | } 201 | }() 202 | 203 | go func() { 204 | defer wg.Done() 205 | err := session.Start(fmt.Sprintf("%s -qt %q", SCP_BINARY, remote)) 206 | if err != nil { 207 | errCh <- err 208 | return 209 | } 210 | }() 211 | 212 | if timeout > 0 { 213 | var cancel context.CancelFunc 214 | ctx, cancel = context.WithTimeout(ctx, timeout) 215 | defer cancel() 216 | } 217 | 218 | if err := wait(&wg, ctx); err != nil { 219 | return err 220 | } 221 | 222 | close(errCh) 223 | for err := range errCh { 224 | if err != nil { 225 | return err 226 | } 227 | } 228 | return nil 229 | } 230 | 231 | // wait waits for the waitgroup for the specified max timeout. 232 | // Returns true if waiting timed out. 233 | func wait(wg *sync.WaitGroup, ctx context.Context) error { 234 | c := make(chan struct{}) 235 | go func() { 236 | defer close(c) 237 | wg.Wait() 238 | }() 239 | 240 | select { 241 | case <-c: 242 | return nil 243 | 244 | case <-ctx.Done(): 245 | return ctx.Err() 246 | } 247 | } 248 | 249 | // Close finishes the connection 250 | func (c *SSHConnection) Close() error { 251 | if c.client == nil { 252 | return nil 253 | } 254 | return c.client.Close() 255 | } 256 | 257 | func checkResponse(r io.Reader) error { 258 | response, err := scp.ParseResponse(r) 259 | if err != nil { 260 | return err 261 | } 262 | if response.IsFailure() { 263 | return errors.New(response.GetMessage()) 264 | } 265 | return nil 266 | } 267 | -------------------------------------------------------------------------------- /experiments/swdt/pkg/connections/ssh_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package connections 18 | 19 | import ( 20 | "swdt/pkg/connections/tests" 21 | "testing" 22 | 23 | "github.com/stretchr/testify/assert" 24 | "swdt/apis/config/v1alpha1" 25 | ) 26 | 27 | func TestRunWithoutConnect(t *testing.T) { 28 | credentials := v1alpha1.CredentialsSpec{} 29 | conn := NewConnection(credentials) 30 | assert.NotEqual(t, conn, nil) 31 | out, err := conn.Run("ls") 32 | assert.NotNil(t, err) 33 | assert.Equal(t, out, "") 34 | } 35 | 36 | func TestConnect(t *testing.T) { 37 | var ( 38 | out string 39 | err error 40 | expected = "Running kubelet Kubelet" 41 | cmd = "get-service -name kubelet" 42 | ) 43 | 44 | // start a fake SSH server 45 | hostname := tests.GetHostname("2023") 46 | tests.NewServer(hostname, expected) 47 | 48 | credentials := v1alpha1.CredentialsSpec{ 49 | Hostname: hostname, 50 | Username: tests.Username, 51 | Password: tests.FakePassword, 52 | } 53 | 54 | conn := NewConnection(credentials) 55 | assert.NotEqual(t, conn, nil) 56 | err = conn.Connect() 57 | assert.Nil(t, err) 58 | out, err = conn.Run(cmd) 59 | assert.Nil(t, err) 60 | assert.Equal(t, out, expected) 61 | } 62 | -------------------------------------------------------------------------------- /experiments/swdt/pkg/connections/tests/server.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "fmt" 5 | "golang.org/x/crypto/ssh" 6 | "golang.org/x/term" 7 | "log" 8 | "net" 9 | ) 10 | 11 | var ( 12 | Username = "Administrator" 13 | FakePassword = "fakepassword" 14 | 15 | privateKey = []byte(` 16 | -----BEGIN OPENSSH PRIVATE KEY----- 17 | b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn 18 | NhAAAAAwEAAQAAAYEAxQa3P/xehGWMg9tJrCLV1YmY5MgBTTcK3SEVhcdwqBtr6wVkSwuO 19 | TICKOTFt0ToC71HBSbDHxenTCxg8jVugTSF22ZGbJFojSKOB12FNeEDhNkxEMlbp+3TIYJ 20 | jrr7TEoKRTXHNVKfP9QfoxOMqhFhKLdMu3dE2nP3hTiFhGXydpQFb9fv7261TTn1y+Z6jZ 21 | vnBWY8oSdC+k4XqTiLpleiwOTNU8Li5E9xgcEzVS6MIHNdFZBbaiHqK8BXui2Vt8yeWc+b 22 | OgX/f+bbg245huAxiICjKDOVmVcyrJ2+5ASGx5bALkCVysmrJ1B5bDgkWJb0Ef+700nszk 23 | cnYVcFuxOVXI6pNQUElYCOVzH1NyF9o/emmXwndZz8RbjS1C551tTW95UqJguWkbYENNUd 24 | MDwUok+uC80gu+RPdRRHhsDEn+OJtHawsVShVSjnM1eqN24HC5k4rUF3bADvzcVbWckOP9 25 | el4qVNy+S/HuH6UWIMrj5uQ7fgaUfc2wYTTRfbDhAAAFiEIGaARCBmgEAAAAB3NzaC1yc2 26 | EAAAGBAMUGtz/8XoRljIPbSawi1dWJmOTIAU03Ct0hFYXHcKgba+sFZEsLjkyAijkxbdE6 27 | Au9RwUmwx8Xp0wsYPI1boE0hdtmRmyRaI0ijgddhTXhA4TZMRDJW6ft0yGCY66+0xKCkU1 28 | xzVSnz/UH6MTjKoRYSi3TLt3RNpz94U4hYRl8naUBW/X7+9utU059cvmeo2b5wVmPKEnQv 29 | pOF6k4i6ZXosDkzVPC4uRPcYHBM1UujCBzXRWQW2oh6ivAV7otlbfMnlnPmzoF/3/m24Nu 30 | OYbgMYiAoygzlZlXMqydvuQEhseWwC5AlcrJqydQeWw4JFiW9BH/u9NJ7M5HJ2FXBbsTlV 31 | yOqTUFBJWAjlcx9TchfaP3ppl8J3Wc/EW40tQuedbU1veVKiYLlpG2BDTVHTA8FKJPrgvN 32 | ILvkT3UUR4bAxJ/jibR2sLFUoVUo5zNXqjduBwuZOK1Bd2wA783FW1nJDj/XpeKlTcvkvx 33 | 7h+lFiDK4+bkO34GlH3NsGE00X2w4QAAAAMBAAEAAAF/DKBpjgg2ZnW7k5eyGP4ChjTTP5 34 | YxvykP4SwFnRUy+xMGz4EA9G5BKFX0hcXNK+Nz3LJ4mKhjpSNfCw76knSUyVyjqT3Tm3jL 35 | WhRgddUeid5ekIRCupcnV54cWVRzhkcncsQVM4+QnaetS1UlYmZZ/Hgjx9BmaWWwmjiz4c 36 | EGgYKdFCp/BGyCloJRLZ1b9nizu6inYK3KkPecsXaRjemkJzg7kmD4Al2kvdElu3VnYtNM 37 | cv5/ngYeTahQNGm//f4G3GZhMkfU8tT8OqJ8imqcXmvHL0CApQ9jK9BjZ9Ez3TN3EFljw3 38 | OSQIK5XnFyrF7pKfVNy8W8eqSJvZNFwIFdX5ZWsNuC7EHZGXieEKijl6VGeeyJgPJwxwMM 39 | c5+A8GFa6cj6UCf2KC6go3FBu249PsipGtzJYWu2qV2qGz3L6sVQWB0KocUphYXIPU6a/D 40 | W3cY9od0pdb2LC/gAPM/IBLnTVcPrOxHd/iZU/7i4OFclrPi9adNE3+z2WVLRpKwEAAADB 41 | ALjKbznpaarpFajb2UKObyzt2w4dc+gZimInSnu6vbDOpeEBKgkVM0JJqzD1WgOnoRXnWO 42 | PbyQ9sAbzHcbYJL3ByG0xwaeDw1do9/WNF2nlm2J6fqK4Kea1Vwq2PhQ4agcO9uh0nFJZM 43 | QC/WToyUzioiTwu3neDLDzV32egscRrOIpLdIHZH9MA0SBC6I9SfqTQBOrosIEVvBKR1Hg 44 | 1s6mWBwZ8kAdBJgzL11vXhoJ1W1+LPAk/dzaiU/L/5t/iMrgAAAMEAzoY8ZfYGc/y5Na9t 45 | zQtMZLH8BbZZhcUJ9fN0RbZy/L01a0OMUgG1HMhTrET8SBxjqvoFZ+y3COdEG4eE2sE3e/ 46 | 6eCaZo4fTqoghN9xyKq6EgH78AKTDGKdUBy+ezSwD3N8DH6VPSrVoPsEtUMHMXhvIQxUMM 47 | i3qGB0eyTB6IXR2+muep52Y3teK0bOSmjiHJ0whNPX+B7Nxc8YzQOWCKRea8zr0BUbdxEL 48 | FP6+yfrSv2VdMajhzRbxmwM2742qfhAAAAwQD0OfojCbeBEziQXyTzAb8Oi0OqS2hD6d02 49 | yDRqifw0Ud7VUlpYzDv/UWQW4Y7L3L3Z042XAzAH8BY27lyqn/f5lDmOYjCWgumK189CIy 50 | Pewr7aufywcOT99fdWPff4JMs6BEOgwbxctj+8h/41XmswuCWMimmuHp/LUclWd695dbBp 51 | r4wISfe/JpBKGYMMpYmAEYDlO06leJi05rWO2WQvUOKxdz2JWL7H4e15kqf0Yemwmpkf9q 52 | KaT3SUfkvAKQEAAAAOYWtuYWJiZW5AaG9ydXMBAgMEBQ== 53 | -----END OPENSSH PRIVATE KEY-----`) 54 | ) 55 | 56 | func NewServer(hostname, expected string) { 57 | var err error 58 | config := &ssh.ServerConfig{PasswordCallback: passwordCallback} 59 | if err = parsePrivateKey(config, privateKey); err != nil { 60 | log.Fatal(err) 61 | } 62 | 63 | listener, err := net.Listen("tcp", hostname) 64 | if err != nil { 65 | log.Fatal("failed on listener: ", err) 66 | } 67 | 68 | go acceptConnection(listener, config, expected) 69 | } 70 | 71 | func acceptConnection(listener net.Listener, config *ssh.ServerConfig, result string) { 72 | for { 73 | conn, err := listener.Accept() 74 | if err != nil { 75 | log.Fatal("failed to accept conn: ", err) 76 | } 77 | _, channels, _, err := ssh.NewServerConn(conn, config) 78 | if err != nil { 79 | log.Fatal("failed to handshake: ", err) 80 | } 81 | 82 | for channel := range channels { 83 | channel, requests, err := channel.Accept() // accept channel 84 | if err != nil { 85 | log.Fatalf("error accepting channel: %v", err) 86 | } 87 | 88 | go handleRequest(requests, channel) 89 | 90 | t := term.NewTerminal(channel, "") 91 | _, err = t.Write([]byte(result)) 92 | if err != nil { 93 | log.Fatalf("error writing channel: %v", err) 94 | } 95 | t.ReadLine() // nolint 96 | channel.Close() // nolint 97 | } 98 | } 99 | } 100 | 101 | func handleRequest(in <-chan *ssh.Request, channel ssh.Channel) { 102 | for req := range in { 103 | switch req.Type { 104 | case "exec": 105 | channel.SendRequest("exit-status", false, []byte{0, 0, 0, 0}) // nolint 106 | } 107 | req.Reply(req.Type == "exec", nil) // nolint 108 | } 109 | } 110 | 111 | func passwordCallback(meta ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) { 112 | if meta.User() == Username && string(pass) == FakePassword { 113 | return nil, nil 114 | } 115 | return nil, fmt.Errorf("invalid password") 116 | } 117 | 118 | func GetHostname(port string) string { 119 | if port == "" { 120 | port = "2022" 121 | } 122 | return fmt.Sprintf("%s:%s", "127.0.0.1", port) 123 | } 124 | 125 | func parsePrivateKey(config *ssh.ServerConfig, key []byte) (err error) { 126 | var signer ssh.Signer 127 | signer, err = ssh.ParsePrivateKey(key) 128 | if err != nil { 129 | return err 130 | } 131 | config.AddHostKey(signer) 132 | return nil 133 | } 134 | -------------------------------------------------------------------------------- /experiments/swdt/pkg/pwsh/executor/runner.go: -------------------------------------------------------------------------------- 1 | package executor 2 | 3 | import ( 4 | "swdt/apis/config/v1alpha1" 5 | "swdt/pkg/connections" 6 | "swdt/pkg/pwsh/kubernetes" 7 | "swdt/pkg/pwsh/setup" 8 | ) 9 | 10 | type RunnerInterface interface { 11 | *setup.SetupRunner | *kubernetes.KubernetesRunner 12 | SetConnection(conn *connections.Connection) 13 | } 14 | 15 | type Runner[R RunnerInterface] struct { 16 | Inner R 17 | Conn connections.Connection 18 | } 19 | 20 | // SetConnection forward the object to the inner runner 21 | func (r *Runner[R]) SetConnection(conn *connections.Connection) { 22 | r.Conn = *conn 23 | r.Inner.SetConnection(conn) 24 | } 25 | 26 | // CloseConnection finished the remote connection from runner 27 | func (r *Runner[R]) CloseConnection() error { 28 | return r.Conn.Close() 29 | } 30 | 31 | // NewRunner returns a new encapsulated Runner to be reused on specialized commands. 32 | func NewRunner[R RunnerInterface](nodeConfig *v1alpha1.Node, run R) (*Runner[R], error) { 33 | conn := connections.NewConnection(nodeConfig.Spec.Cred) 34 | if err := conn.Connect(); err != nil { 35 | return nil, err 36 | } 37 | runner := Runner[R]{Inner: run} 38 | runner.SetConnection(&conn) 39 | return &runner, nil 40 | } 41 | -------------------------------------------------------------------------------- /experiments/swdt/pkg/pwsh/executor/runner_test.go: -------------------------------------------------------------------------------- 1 | package executor 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "swdt/apis/config/v1alpha1" 6 | "swdt/pkg/connections/tests" 7 | "swdt/pkg/pwsh/kubernetes" 8 | "swdt/pkg/pwsh/setup" 9 | "testing" 10 | ) 11 | 12 | func TestMultipleExecutors(t *testing.T) { 13 | hostname := tests.GetHostname("2024") 14 | tests.NewServer(hostname, "fake runner") 15 | 16 | kubeRunner, err := NewRunner(createNodeConfig(hostname), &kubernetes.KubernetesRunner{}) 17 | assert.Nil(t, err) 18 | assert.NotNil(t, kubeRunner.Inner) 19 | assert.IsType(t, kubernetes.KubernetesRunner{}, *kubeRunner.Inner) 20 | err = kubeRunner.CloseConnection() 21 | assert.Nil(t, err) 22 | 23 | setupRunner, err := NewRunner(createNodeConfig(hostname), &setup.SetupRunner{}) 24 | assert.Nil(t, err) 25 | assert.NotNil(t, setupRunner.Inner) 26 | assert.IsType(t, setup.SetupRunner{}, *setupRunner.Inner) 27 | err = setupRunner.CloseConnection() 28 | assert.Nil(t, err) 29 | } 30 | 31 | func createNodeConfig(hostname string) *v1alpha1.Node { 32 | credentials := v1alpha1.CredentialsSpec{ 33 | Hostname: hostname, 34 | Username: tests.Username, 35 | Password: tests.FakePassword, 36 | } 37 | return &v1alpha1.Node{Spec: v1alpha1.NodeSpec{Cred: credentials}} 38 | } 39 | -------------------------------------------------------------------------------- /experiments/swdt/pkg/pwsh/kubernetes/provisioners.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "fmt" 5 | "github.com/fatih/color" 6 | klog "k8s.io/klog/v2" 7 | "swdt/apis/config/v1alpha1" 8 | "swdt/pkg/connections" 9 | ) 10 | 11 | var ( 12 | resc = color.New(color.FgHiGreen).Add(color.Bold) 13 | permission = "0755" 14 | ) 15 | 16 | type KubernetesRunner struct { 17 | conn connections.Connection 18 | run func(args string) (string, error) 19 | copy func(local, remote, perm string) error 20 | } 21 | 22 | func (r *KubernetesRunner) SetConnection(conn *connections.Connection) { 23 | r.conn = *conn 24 | r.run = r.conn.Run 25 | r.copy = r.conn.Copy 26 | } 27 | 28 | func (r *KubernetesRunner) InstallProvisioners(provisioners []v1alpha1.ProvisionerSpec) error { 29 | for _, provisioner := range provisioners { 30 | source, destination := provisioner.SourceURL, provisioner.Destination 31 | name := provisioner.Name 32 | klog.Info(resc.Sprintf("Service %s binary replacement, trying to stop service...", name)) 33 | _, err := r.run(fmt.Sprintf("Stop-Service -name %s -Force", name)) 34 | if err != nil { 35 | klog.Error(err) 36 | continue 37 | } 38 | klog.Infof("Service stopped. Copying file %s to remote %s...", source, destination) 39 | if err = r.copy(source, destination, permission); err != nil { 40 | klog.Error(err) 41 | continue 42 | } 43 | klog.Infof("starting service %s again...", name) 44 | _, err = r.run(fmt.Sprintf("Start-Service -name %s", name)) 45 | if err != nil { 46 | klog.Error(err) 47 | continue 48 | } 49 | klog.Info(resc.Sprintf("Service started.\n")) 50 | } 51 | return nil 52 | } 53 | -------------------------------------------------------------------------------- /experiments/swdt/pkg/pwsh/kubernetes/provisioners_test.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "swdt/apis/config/v1alpha1" 7 | "testing" 8 | ) 9 | 10 | var ( 11 | calls = []string{} 12 | ) 13 | 14 | func validateRun(args string) (string, error) { 15 | calls = append(calls, args) 16 | return "cmd stdout", nil 17 | } 18 | 19 | func validateCopy(l, r, p string) error { 20 | return nil 21 | } 22 | 23 | func TestInstallProvisioners(t *testing.T) { 24 | r := KubernetesRunner{run: validateRun, copy: validateCopy} 25 | serviceName := "containerd" 26 | provisioners := []v1alpha1.ProvisionerSpec{{Name: serviceName}} 27 | assert.Nil(t, r.InstallProvisioners(provisioners)) 28 | assert.Len(t, calls, 2) 29 | assert.Equal(t, calls[0], fmt.Sprintf("Stop-Service -name %s -Force", serviceName)) 30 | assert.Equal(t, calls[1], fmt.Sprintf("Start-Service -name %s", serviceName)) 31 | } 32 | -------------------------------------------------------------------------------- /experiments/swdt/pkg/pwsh/setup/setup.go: -------------------------------------------------------------------------------- 1 | package setup 2 | 3 | import ( 4 | "fmt" 5 | "github.com/fatih/color" 6 | klog "k8s.io/klog/v2" 7 | "swdt/pkg/connections" 8 | ) 9 | 10 | var ( 11 | mainc = color.New(color.FgHiBlack).Add(color.Underline) 12 | resc = color.New(color.FgHiGreen).Add(color.Bold) 13 | ) 14 | 15 | const ( 16 | CHOCO_PATH = "C:\\ProgramData\\chocolatey\\bin\\choco.exe" 17 | CHOCO_INSTALL = "install --accept-licenses --yes" 18 | ) 19 | 20 | type SetupRunner struct { 21 | conn connections.Connection 22 | run func(args string) (string, error) 23 | copy func(local, remote, perm string) error 24 | } 25 | 26 | func (r *SetupRunner) SetConnection(conn *connections.Connection) { 27 | r.conn = *conn 28 | r.run = r.conn.Run 29 | r.copy = r.conn.Copy 30 | } 31 | 32 | // InstallChoco proceed to install choco in the default ProgramData folder. 33 | func (r *SetupRunner) InstallChoco() error { 34 | klog.Info(mainc.Sprint("Installing Choco with PowerShell")) 35 | 36 | if r.ChocoExists() { 37 | klog.Info(resc.Sprintf("Choco already exists, skipping installation...")) 38 | return nil 39 | } 40 | 41 | // Proceed to install choco package manager. 42 | output, err := r.run(`Set-ExecutionPolicy Bypass -Scope Process -Force; 43 | [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; 44 | iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))`) 45 | if err != nil { 46 | return err 47 | } 48 | klog.Info(resc.Sprintf("Installed Choco with: %s", output)) 49 | 50 | return nil 51 | } 52 | 53 | // InstallChocoPackages iterate on a list of packages and execute the installation. 54 | func (r *SetupRunner) InstallChocoPackages(packages []string) error { 55 | if !r.ChocoExists() { 56 | return fmt.Errorf("choco not installed. Skipping package installation") 57 | } 58 | 59 | klog.Info(mainc.Sprint("Installing Choco packages.")) 60 | for _, pkg := range packages { 61 | output, err := r.run(fmt.Sprintf("%s %s %s", CHOCO_PATH, CHOCO_INSTALL, pkg)) 62 | if err != nil { 63 | return err 64 | } 65 | klog.Info(resc.Sprintf("Installed package %s: %s", pkg, output)) 66 | } 67 | return nil 68 | } 69 | 70 | // ChocoExists check if choco is already installed in the system. 71 | // todo(knabben) - fix the error granularity and find the correct stderr 72 | func (r *SetupRunner) ChocoExists() bool { 73 | _, err := r.run(fmt.Sprintf("%s --version", CHOCO_PATH)) 74 | return err == nil 75 | } 76 | 77 | // EnableRDP allow RDP to be accessed in Windows property and Firewall rule 78 | func (r *SetupRunner) EnableRDP(enable bool) error { 79 | if !enable { 80 | klog.Warning("Remote Desktop field is disabled. Check the configuration to enable it.") 81 | return nil 82 | } 83 | 84 | klog.Info(resc.Sprintf("Enabling RDP.")) 85 | output, err := r.run(`Set-ItemProperty -Path "HKLM:\System\CurrentControlSet\Control\Terminal Server" -name "fDenyTSConnections" -value 0; 86 | Enable-NetFirewallRule -DisplayGroup "Remote Desktop"`) 87 | if err != nil { 88 | return err 89 | } 90 | klog.Info(resc.Sprintf(output)) 91 | return nil 92 | } 93 | -------------------------------------------------------------------------------- /experiments/swdt/pkg/pwsh/setup/setup_test.go: -------------------------------------------------------------------------------- 1 | package setup 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "swdt/apis/config/v1alpha1" 7 | "testing" 8 | ) 9 | 10 | var ( 11 | calls = []string{} 12 | chocoCheck = fmt.Sprintf("%s --version", CHOCO_PATH) 13 | ) 14 | 15 | func validateRun(args string) (string, error) { 16 | calls = append(calls, args) 17 | return "cmd stdout", nil 18 | } 19 | 20 | func TestChocoExist(t *testing.T) { 21 | calls = []string{} 22 | expectedCalls := 1 23 | 24 | r := SetupRunner{run: validateRun} 25 | assert.True(t, r.ChocoExists()) 26 | assert.Len(t, calls, expectedCalls) 27 | assert.Equal(t, calls[0], chocoCheck) 28 | } 29 | 30 | func TestInstallChocoPackages(t *testing.T) { 31 | calls = []string{} 32 | expectedCalls := 3 33 | pkgs := []string{"vim", "grep"} 34 | 35 | r := SetupRunner{run: validateRun} 36 | config := v1alpha1.SetupSpec{ChocoPackages: &pkgs} 37 | err := r.InstallChocoPackages(*config.ChocoPackages) 38 | assert.Nil(t, err) 39 | 40 | assert.Len(t, calls, expectedCalls) 41 | assert.Equal(t, calls[0], chocoCheck) 42 | for i := 0; i < expectedCalls-1; i++ { 43 | assert.Equal(t, calls[i+1], fmt.Sprintf("%s %s %s", CHOCO_PATH, CHOCO_INSTALL, pkgs[i])) 44 | } 45 | } 46 | 47 | func TestEnableRDP(t *testing.T) { 48 | var defaultTrue = true 49 | r := SetupRunner{run: validateRun} 50 | config := v1alpha1.SetupSpec{EnableRDP: &defaultTrue} 51 | 52 | err := r.EnableRDP(*config.EnableRDP) 53 | assert.Nil(t, err) 54 | } 55 | -------------------------------------------------------------------------------- /experiments/swdt/samples/config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: windows.k8s.io/v1alpha1 2 | kind: Node 3 | metadata: 4 | name: sample 5 | spec: 6 | credentials: 7 | username: "Administrator" 8 | hostname: "192.168.122.220:22" 9 | privateKey: "/home/<user>/.ssh/id_rsa" 10 | setup: 11 | enableRDP: true 12 | chocoPackages: 13 | - vim 14 | - grep 15 | kubernetes: 16 | provisioners: 17 | - name: containerd 18 | version: 1.7.11 19 | sourceURL: "/home/<user>/go/src/github.com/containerd/containerd/bin/containerd" 20 | destination: "C:\\Program Files\\containerd\\containerd.exe" 21 | - name: kubelet 22 | version: 1.29.0 23 | sourceURL: "/home/<user>/go/src/k8s.io/kubernetes/_output/local/bin/windows/amd64/kubelet.exe" 24 | destination: "C:\\k\\kubelet.exe" -------------------------------------------------------------------------------- /experiments/swdt/samples/mloskot/.gitignore: -------------------------------------------------------------------------------- 1 | *id_rsa* 2 | vm*/ 3 | linkworker/ 4 | winworker/ -------------------------------------------------------------------------------- /experiments/swdt/samples/mloskot/README.windows.md: -------------------------------------------------------------------------------- 1 | # Testing SWDT CLI for Windows node on Windows host 2 | 3 | This is a very early guide with step-by-step instructions for those who 4 | want to try and test the SWDT CLI on Windows host targeting Windows VM 5 | using PowerShell in order to set it up and join as Kubernetes cluster node, 6 | that is, as SWDT CLI is being completed with new features. 7 | 8 | > *IMPORTANT*: 9 | > Run the presented PowerShell commands one by one in order as their are presented. 10 | > If any command fails for you, please, report it. 11 | 12 | ## Prerequisites 13 | 14 | - Windows host 15 | - PowerShell 7 > Run as Administrator 16 | - Downloaded [Windows Server 2022 VHD](https://www.microsoft.com/en-us/evalcenter/download-windows-server-2022) 17 | 18 | ## 1. Preparation 19 | 20 | Check PowerShell on Windows host runs as Administrator: 21 | 22 | ```powershell 23 | (New-Object Security.Principal.WindowsPrincipal $([Security.Principal.WindowsIdentity]::GetCurrent())).IsInRole([Security.Principal.WindowsBuiltinRole]::Administrator) 24 | ``` 25 | 26 | Enable Hyper-V on Windows host: 27 | 28 | ```powershell 29 | Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -All 30 | ``` 31 | 32 | ## 2. Generate SSH key 33 | 34 | Generate SSH key to be deployed to Windows VM for convenient password-less SSH communication: 35 | 36 | ```powershell 37 | ssh-keygen -f .\experiments\swdt\samples\mloskot\ssh.id_rsa 38 | ``` 39 | 40 | and fix the private key file permissions: 41 | 42 | ```powershell 43 | New-Variable -Name sshKey -Value ".\experiments\swdt\samples\mloskot\ssh.id_rsa" 44 | icacls $sshKey /c /t /Inheritance:d 45 | icacls $sshKey /c /t /Grant ${env:UserName}:F 46 | takeown /F $sshKey 47 | icacls $sshKey /c /t /Grant:r ${env:UserName}:F 48 | icacls $sshKey /c /t /Remove:g Administrator "Authenticated Users" BUILTIN\Administrators BUILTIN Everyone System Users 49 | Remove-Variable -Name sshKey 50 | ``` 51 | 52 | > *IMPORTANT:* The location of the SSH private key, relative to the project repository root folder, 53 | is already present in the [winworker.yaml](winworker.yaml) configuration file. 54 | 55 | ## 3. Create Hyper-V NAT network 56 | 57 | Run the following PowerShell commands on Windows host as Administrator. 58 | 59 | Windows currently [allows to set up only one NAT network per host](https://learn.microsoft.com/en-us/virtualization/hyper-v-on-windows/user-guide/setup-nat-network), 60 | hence a generic non-SWDT specific name is picked below: 61 | 62 | ```powershell 63 | New-VMSwitch -SwitchName 'ClusterNatSwitch' -SwitchType Internal -Notes 'Virtual Switch with NAT used for networking between nodes of hybrid Kubernets cluster, with Internet access.' 64 | ``` 65 | 66 | ```powershell 67 | New-NetIPAddress -IPAddress 192.168.10.1 -PrefixLength 24 -InterfaceAlias 'vEthernet (ClusterNatSwitch)' 68 | ``` 69 | 70 | ```powershell 71 | New-NetNAT -Name 'ClusterNatNetwork' -InternalIPInterfaceAddressPrefix 192.168.10.0/24 72 | ``` 73 | 74 | ```powershell 75 | Add-Content -Path "$($Env:WinDir)\system32\Drivers\etc\hosts" -Value '192.168.10.1 gateway.cluster gateway # ClusterNatSwitch IP' 76 | Add-Content -Path "$($Env:WinDir)\system32\Drivers\etc\hosts" -Value '192.168.10.2 master.cluster master # Kubernetes Linux node (control-plane)' 77 | Add-Content -Path "$($Env:WinDir)\system32\Drivers\etc\hosts" -Value '192.168.10.3 winworker.cluster winworker # Kubernetes Windows node' 78 | ``` 79 | 80 | ## 4. Create Windows VM 81 | 82 | The VHD requires VM generation 1, does not boot for VM generation 2. 83 | Using the official Windows Server VHD to avoid walk through the manual 84 | process of Windows installation from ISO. It will require to complete 85 | initial configuration interactively (i.e. language and keyboard selection, 86 | setting password for Administrator user - use `K8s@windows` as reasonable default). 87 | 88 | ```powershell 89 | $vmName = 'winworker' 90 | ``` 91 | 92 | ```powershell 93 | $vmConfigPath = New-Item -Path ".\experiments\swdt\samples\mloskot" -Name $vmName -ItemType Directory -Force; 94 | $vmVhdPath = Join-Path -Path $vmConfigPath -ChildPath 'os.vhdx'; 95 | Convert-VHD -Path "$($Env:UserProfile)\Downloads\20348.169.amd64fre.fe_release_svc_refresh.210806-2348_server_serverdatacentereval_en-us.vhd" -DestinationPath $vmVhdPath; 96 | ``` 97 | 98 | ```powershell 99 | New-VM -Name $vmName -Generation 1 -Switch 'ClusterNatSwitch' -Path $vmConfigPath; 100 | Add-VMHardDiskDrive -VMName $vmName -Path $vmVhdPath -ControllerType IDE -ControllerNumber 0 -ControllerLocation 1; 101 | Set-VMBios -VMName $vmName -StartupOrder @("IDE", "Floppy", "LegacyNetworkAdapter", "CD") 102 | Set-VMMemory -VMName $vmName -DynamicMemoryEnabled $true -MinimumBytes 2GB -StartupBytes 4GB -MaximumBytes 8GB; 103 | Set-VMProcessor -VMName $vmName -Count 2; 104 | Set-VMProcessor -VMName $vmName -ExposeVirtualizationExtensions $true; 105 | Get-VMNetworkAdapter -VMName $vmName | Connect-VMNetworkAdapter -SwitchName 'ClusterNatSwitch'; 106 | Get-VMNetworkAdapter -VMName $vmName | Set-VMNetworkAdapter -MacAddressSpoofing On; 107 | Start-VM -Name $vmName; 108 | vmconnect.exe $env:ComputerName $vmName; 109 | ``` 110 | 111 | The VM should start and Windows boot displaying `Hi there` screen where the initial configuration needs to be completed interactively: 112 | 113 | 1. Select region, language and keyboard layout. 114 | 2. Accept the licence terms. 115 | 3. Set the `Administrator` password, here are reasonable defaults: 116 | 117 | ```powershell 118 | $vmAdminUsername = "Administrator"; 119 | $vmAdminPassword = "K8s@windows"; 120 | ``` 121 | 122 | ## 5. Configure Windows VM 123 | 124 | The following PowerShell Direct commands are executed directly on the Windows VM. 125 | 126 | *TODO(mloskot):* Run those commands from within dedicated SWDT CLI command. 127 | 128 | ```powershell 129 | $vmAdminUsername = "Administrator"; 130 | $vmAdminPassword = "K8s@windows"; 131 | $vmAdminPasswordSecure = New-Object -TypeName System.Security.SecureString; 132 | $vmAdminPassword.ToCharArray() | ForEach-Object { $vmAdminPasswordSecure.AppendChar($_) }; 133 | $vmAdminCredential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $vmAdminUsername, $vmAdminPasswordSecure; 134 | ``` 135 | 136 | ```powershell 137 | Invoke-Command -VMName $vmName -Credential $vmAdminCredential -ScriptBlock { 138 | Rename-Computer -NewName 'winworker'; # cannot access variable from outside script block 139 | New-NetIPAddress -IPAddress 192.168.10.3 -PrefixLength 24 -InterfaceAlias "Ethernet" -DefaultGateway 192.168.10.1; 140 | Set-DnsClientServerAddress -ServerAddresses 1.1.1.1,8.8.8.8 -InterfaceAlias "Ethernet"; 141 | } 142 | ``` 143 | 144 | > *NOTE:* The Windows VM IP address is already present in the [winworker.yaml](winworker.yaml) configuration file. 145 | 146 | [Configure Windows Firewall](https://learn.microsoft.com/en-us/windows/security/operating-system-security/network-security/windows-firewall/configure-with-command-line?tabs=powershell) 147 | to allow all traffic from Linux nodes, including ICMP - Lazy way: 148 | 149 | ```powershell 150 | Invoke-Command -VMName $vmName -Credential $vmAdminCredential -ScriptBlock { 151 | Set-NetFirewallProfile -Profile Domain,Public,Private -Enabled False; 152 | } 153 | ``` 154 | 155 | Restart VM: 156 | 157 | ```powershell 158 | Invoke-Command -VMName $vmName -Credential $vmAdminCredential -ScriptBlock { 159 | Restart-Computer -Force; 160 | } 161 | ``` 162 | 163 | Since, currently, SWDT CLI executes commands on remote host via SSH, 164 | it is a good idea to [set up SSH on Windows VM](https://learn.microsoft.com/en-us/windows-server/administration/openssh/openssh_install_firstuse?tabs=powershell), 165 | with key based authentication in the next steps: 166 | 167 | ```powershell 168 | Invoke-Command -VMName $vmName -Credential $vmAdminCredential -ScriptBlock { 169 | $ProgressPreference = 'SilentlyContinue'; 170 | Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0; 171 | Set-Service -Name sshd -StartupType 'Automatic'; 172 | Start-Service sshd; 173 | } 174 | ``` 175 | 176 | Fix broken configuration of SSH on Windows: 177 | 178 | - <https://stackoverflow.com/a/77705199/151641> 179 | - <https://github.com/PowerShell/Win32-OpenSSH/issues/1942#issuecomment-1868015179> 180 | 181 | ```powershell 182 | Invoke-Command -VMName $vmName -Credential $vmAdminCredential -ScriptBlock { 183 | $content = Get-Content -Path $env:ProgramData\ssh\sshd_config; 184 | $content = $content -replace '.*Match Group administrators.*', ''; 185 | $content = $content -replace '.*AuthorizedKeysFile.*__PROGRAMDATA__.*', ''; 186 | Set-Content -Path $env:ProgramData\ssh\sshd_config -Value $content; 187 | } 188 | ``` 189 | 190 | Use SSH to [deploy the public key](https://learn.microsoft.com/en-us/windows-server/administration/openssh/openssh_keymanagement#deploying-the-public-key) to Windows VM: 191 | 192 | ```powershell 193 | ssh-keygen -R winworker; 194 | $publicKey = Get-Content -Path '.\experiments\swdt\samples\mloskot\ssh.id_rsa.pub'; 195 | $remoteCmd = "powershell New-Item -Force -ItemType Directory -Path C:\Users\Administrator\.ssh; Add-Content -Force -Path C:\Users\Administrator\.ssh\authorized_keys -Value '$publicKey'; icacls.exe ""C:\Users\Administrator\.ssh\authorized_keys "" /inheritance:r /grant ""Administrators:F"" /grant ""SYSTEM:F""; Restart-Service sshd;"; 196 | ssh Administrator@winworker $remoteCmd 197 | ``` 198 | 199 | Test SSH authentication using the private key - no password prompt is expected: 200 | 201 | ```powershell 202 | ssh -i '.\experiments\swdt\samples\mloskot\ssh.id_rsa' Administrator@winworker 203 | ``` 204 | 205 | ## 6. Set up Windows node 206 | 207 | ```powershell 208 | .\swdt.ps1 setup --config .\experiments\swdt\samples\mloskot\winworker.yaml 209 | ``` 210 | 211 | Assuming successful completion of the command above the Windows node 212 | should now be provision with Chocolatey and some general purpose utilities. 213 | 214 | Try it: 215 | 216 | ```powershell 217 | ssh -i '.\experiments\swdt\samples\mloskot\ssh.id_rsa' Administrator@winworker "C:\ProgramData\chocolatey\bin\choco.exe --version" 218 | ``` 219 | 220 | *TODO(mloskot):* Keep this guide up to date as new SWDT CLI features are implemented 221 | -------------------------------------------------------------------------------- /experiments/swdt/samples/mloskot/ssh.id_rsa: -------------------------------------------------------------------------------- 1 | -----BEGIN OPENSSH PRIVATE KEY----- 2 | b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn 3 | NhAAAAAwEAAQAAAYEA2/pytRDF8CEA/S2scwB4Wb9SDmcE6DnJSwbmKtR5/skRJq2uDZ0U 4 | dpPpBJR5a0DTWtMCgYSCHJHNbKM3780d4Mb+0NJsBkmDwFZwUyKfMm7j1mjbn3cmiv7SK3 5 | x3ktT0Z97FthV9tg1+hJ5R9gZssiGQafBX4O5C7/4eHYSwkva9jrEaGkS1mSkgXH/9w6ke 6 | f5IlVHJEHEsArPRseE325Ftva8Oym7hKq1iaa4XLINR7Ba9QxHDkq8xkROpjoQfu0bBc6J 7 | wA5yzFHj1ke7wTO3f1gjlfgr1+G514aRNf+QBkRq2uS0h8tlRfBun3qERzKKX0wkcpGNuC 8 | XxYVfgt8LM/ODMzrJ9syjeXTGAmoDXPkoDi+bfdM1r+7XiMG96Jyg2GaBlSYM3sDrmeA6n 9 | NcjUoA5T/eletlS93vFU5MHfUqyU+RwykNXgtfki63drT7EYlCyGHaUQIRFygLPE0vtQT5 10 | h6L+SqtNRzJJ4rBFSZAyBn+DaYDw9N1hbdH5vFAbAAAFiEyZYFZMmWBWAAAAB3NzaC1yc2 11 | EAAAGBANv6crUQxfAhAP0trHMAeFm/Ug5nBOg5yUsG5irUef7JESatrg2dFHaT6QSUeWtA 12 | 01rTAoGEghyRzWyjN+/NHeDG/tDSbAZJg8BWcFMinzJu49Zo2593Jor+0it8d5LU9Gfexb 13 | YVfbYNfoSeUfYGbLIhkGnwV+DuQu/+Hh2EsJL2vY6xGhpEtZkpIFx//cOpHn+SJVRyRBxL 14 | AKz0bHhN9uRbb2vDspu4SqtYmmuFyyDUewWvUMRw5KvMZETqY6EH7tGwXOicAOcsxR49ZH 15 | u8Ezt39YI5X4K9fhudeGkTX/kAZEatrktIfLZUXwbp96hEcyil9MJHKRjbgl8WFX4LfCzP 16 | zgzM6yfbMo3l0xgJqA1z5KA4vm33TNa/u14jBveicoNhmgZUmDN7A65ngOpzXI1KAOU/3p 17 | XrZUvd7xVOTB31KslPkcMpDV4LX5Iut3a0+xGJQshh2lECERcoCzxNL7UE+Yei/kqrTUcy 18 | SeKwRUmQMgZ/g2mA8PTdYW3R+bxQGwAAAAMBAAEAAAGBANszGIR231al/I9MqO+7wzFjJQ 19 | g2/bvOgVfi2w5IxfvuLcoF/U+RfCegweyPauJGKZXyeCfJe8GuRC4CAqZRFXm7sYUyePRW 20 | 4xOQzsL41/vwZdZsjJS9rUmKHLG6iIhm7dhnM0p65da4YZ8TCAcJZdym93mpRlhb85Wj+i 21 | xd4XnApyj+L02HtBGX2qhx3dPPvRVP8yTP7DDpSBYQRdonjmME/eU7j8n9/WH6m32TR9NC 22 | CAilJu1otWS/+hZ2HasD88wlJwItlMm4ZSivhOHXbS0STP5pyX/EAxkErKdUvg5Ykkl0zz 23 | iXHa8EX5ly0/nB6RFjfVSPphT1NEWmGx8Bv8kegmn/ev3sNZZ4aVdT4/eEJsbvseYAk4FP 24 | B7/457Ils+CSqUozwjiS+vuUcYw/e89h7LYByGHzsWEo7Z4d7o8nfDuKoOerpW94hTlMwo 25 | hiQdxZh4y6l3FlbeHtPBdsJdZEOW7QBDo7CZHTuvCsp9wBGVxgOtADmiO/L63rZbi0MQAA 26 | AMEAp4XgoacLsOq8qBCkzNYIeUrthGC0RWS+S29+u4d2WaTlMvd+YGOM473RsJ33e+5FUn 27 | gQW5qoAuALLCe4xDeA2eEMadT3XSvWwzJn3OPDh8dwnH1Th9xTKVuTzu+f0VqGBJsdynhr 28 | GJYzTQmef29arijIDjyI1X1jWfMexuif1qKfw0vJ8yDtY/zg01/PPeW3+jOr7BlOVlpg7i 29 | Tcdt5pX0Cw41/3Z7pT5O+sYz1c9hJiNzR4kqR4hREkwJVQqqYYAAAAwQDzcXDwF39cPpNn 30 | P5lkpSj0pjzVhsH6XfDApk6x9riQkAbWEQ6squE18S0HWcbL1XN8nQfOj/EsurXLYN7uIv 31 | Tf17lt2x1vfK2N9ANyEaYv5XrXv4tsJTzbjrX+NcmWXwpDc+Y4kNAmsMtfbwBcA8/2ERC6 32 | WFXpySx87Qn/zY7cK2HsgJ5V/4BGPJYTwGJVLNTH2fcGvg4BmyMzLT+1rqkihP6Ku+tC3C 33 | cNLQYIxMcFOgKi1hxidoC62x27G/z76lMAAADBAOdTKhU70/8HxeqjuD65XtmCnDlPt2Ek 34 | pFWrnN8ERAOpzaLS1dwozdSrMFg/yeLt+WwYrHF0s0HP1EjPQEmSMJneLLYs3weVzF6qSE 35 | sXxjxX0zuJje4q2pgiGdpnGav9cBa5/EZLfhW3EnlFPq1MXckfOuBizIKcrfnQRd7K0m3O 36 | dbG8ORS3MMtn4/N7+Jrl351OXGNjnDgE9TU5/YgVm8Q1xlRZiEVaaxhMiWVEoyv6PxBUnl 37 | W0ARRd+QY5RJIaGQAAAAttYXRldUBHSU5FUwECAwQFBg== 38 | -----END OPENSSH PRIVATE KEY----- 39 | -------------------------------------------------------------------------------- /experiments/swdt/samples/mloskot/ssh.id_rsa.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDb+nK1EMXwIQD9LaxzAHhZv1IOZwToOclLBuYq1Hn+yREmra4NnRR2k+kElHlrQNNa0wKBhIIckc1sozfvzR3gxv7Q0mwGSYPAVnBTIp8ybuPWaNufdyaK/tIrfHeS1PRn3sW2FX22DX6EnlH2BmyyIZBp8Ffg7kLv/h4dhLCS9r2OsRoaRLWZKSBcf/3DqR5/kiVUckQcSwCs9Gx4TfbkW29rw7KbuEqrWJprhcsg1HsFr1DEcOSrzGRE6mOhB+7RsFzonADnLMUePWR7vBM7d/WCOV+CvX4bnXhpE1/5AGRGra5LSHy2VF8G6feoRHMopfTCRykY24JfFhV+C3wsz84MzOsn2zKN5dMYCagNc+SgOL5t90zWv7teIwb3onKDYZoGVJgzewOuZ4Dqc1yNSgDlP96V62VL3e8VTkwd9SrJT5HDKQ1eC1+SLrd2tPsRiULIYdpRAhEXKAs8TS+1BPmHov5Kq01HMknisEVJkDIGf4NpgPD03WFt0fm8UBs= mateu@GINES 2 | -------------------------------------------------------------------------------- /experiments/swdt/samples/mloskot/winworker.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: windows.k8s.io/v1alpha1 2 | kind: Node 3 | metadata: 4 | name: winworker 5 | spec: 6 | credentials: 7 | hostname: "192.168.10.3:22" 8 | username: "Administrator" 9 | privateKey: ".\\experiments\\swdt\\samples\\mloskot\\ssh.id_rsa" 10 | setup: 11 | enableRDP: true 12 | chocoPackages: 13 | - vim 14 | - grep 15 | -------------------------------------------------------------------------------- /fetch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | : ' 3 | Copyright 2021 The Kubernetes Authors. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ' 17 | set -e 18 | 19 | VARIABLES_FILE="sync/shared/variables.yaml" 20 | BUILD_FROM_SOURCE=0 21 | [[ 22 | $(awk '/build_from_source/ {print $2}' ${VARIABLES_FILE} | sed -e 's/^"//' -e 's/"$//' | head -1) =~ "true" 23 | ]] && BUILD_FROM_SOURCE=1 24 | 25 | # kubernetes version can be passed as param, otherwise it will be read from variables_file 26 | 27 | variables_file="sync/shared/variables.yaml" 28 | version=${1:-`awk '/kubernetes_version/ {print \$2}' ${variables_file} | sed -e 's/^"//' -e 's/"$//' | head -1`} 29 | kubernetes_latest_file=`curl -f https://storage.googleapis.com/k8s-release-dev/ci/latest-${version}.txt` 30 | 31 | kubernetes_sha=$(echo ${kubernetes_latest_file} | cut -d "+" -f 2) 32 | kubernetes_tag=$(echo ${kubernetes_latest_file} | cut -d "+" -f 1) 33 | 34 | if [ ! -z ${kubernetes_sha} ]; then 35 | echo "Using Kubernetes version ${kubernetes_tag}-${kubernetes_sha} from upstream" 36 | fi 37 | 38 | if [[ $BUILD_FROM_SOURCE -eq 1 ]]; then 39 | if [[ -d "kubernetes" ]] ; then 40 | echo "kubernetes/ exists, not cloning..." 41 | pushd kubernetes 42 | git checkout $kubernetes_sha -f 43 | popd 44 | else 45 | git clone https://github.com/kubernetes/kubernetes.git 46 | pushd kubernetes 47 | git checkout $kubernetes_sha 48 | popd 49 | fi 50 | fi 51 | 52 | -------------------------------------------------------------------------------- /forked/StartKubelet.ps1: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The Kubernetes Authors. 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # From https://github.com/kubernetes-sigs/sig-windows-tools/blob/master/kubeadm/scripts/PrepareNode.ps1 16 | $FileContent = Get-Content -Path "/var/lib/kubelet/kubeadm-flags.env" 17 | $kubeAdmArgs = $FileContent.TrimStart('KUBELET_KUBEADM_ARGS=').Trim('"') 18 | 19 | $args = "--cert-dir=$env:SYSTEMDRIVE/var/lib/kubelet/pki", 20 | "--config=$env:SYSTEMDRIVE/var/lib/kubelet/config.yaml", 21 | "--bootstrap-kubeconfig=$env:SYSTEMDRIVE/etc/kubernetes/bootstrap-kubelet.conf", 22 | "--kubeconfig=$env:SYSTEMDRIVE/etc/kubernetes/kubelet.conf", 23 | "--hostname-override=$(hostname)", 24 | "--pod-infra-container-image=`"{{ pause_image }}`"", 25 | "--enable-debugging-handlers", 26 | "--cgroups-per-qos=false", 27 | "--enforce-node-allocatable=`"`"", 28 | "--resolv-conf=`"`"" 29 | 30 | $kubeletCommandLine = "kubelet.exe " + ($args -join " ") + " $kubeAdmArgs" 31 | Invoke-Expression $kubeletCommandLine 32 | -------------------------------------------------------------------------------- /forked/config.ps1: -------------------------------------------------------------------------------- 1 | $baseDir = "$PSScriptRoot" 2 | ipmo $baseDir\libs\calico\calico.psm1 -Force 3 | 4 | Write-Host "Setting environment variables if not set..." 5 | 6 | ## Cluster configuration: 7 | 8 | # KUBE_NETWORK should be set to a regular expression that matches the HNS network(s) used for pods. 9 | # The default, "Calico.*", is correct for Calico CNI. 10 | Set-EnvVarIfNotSet -var "KUBE_NETWORK" -defaultValue "Calico.*" 11 | 12 | # Set this to one of the following values: 13 | # - "vxlan" for Calico VXLAN networking 14 | # - "windows-bgp" for Calico BGP networking using the Windows BGP router. 15 | # - "none" to disable the Calico CNI plugin (so that you can use another plugin). 16 | Set-EnvVarIfNotSet -var "CALICO_NETWORKING_BACKEND" -defaultValue "vxlan" 17 | 18 | # Set to match your Kubernetes service CIDR. 19 | Set-EnvVarIfNotSet -var "K8S_SERVICE_CIDR" -defaultValue "10.96.0.0/12" 20 | Set-EnvVarIfNotSet -var "DNS_NAME_SERVERS" -defaultValue "10.96.0.10" 21 | Set-EnvVarIfNotSet -var "DNS_SEARCH" -defaultValue "svc.cluster.local" 22 | 23 | ## Datastore configuration: 24 | 25 | # Set this to "kubernetes" to use the kubernetes datastore, or "etcdv3" for etcd. 26 | Set-EnvVarIfNotSet -var "CALICO_DATASTORE_TYPE" -defaultValue "kubernetes" 27 | 28 | # Set KUBECONFIG to the path of your kubeconfig file. 29 | Set-EnvVarIfNotSet -var "KUBECONFIG" -defaultValue "C:/etc/kubernetes/kubelet.conf" 30 | 31 | Set-EnvVarIfNotSet -var "CNI_BIN_DIR" -defaultValue "C:/opt/cni/bin" 32 | Set-EnvVarIfNotSet -var "CNI_CONF_DIR" -defaultValue "C:/etc/cni/net.d" 33 | 34 | Set-EnvVarIfNotSet -var "CNI_CONF_FILENAME" -defaultValue "10-calico.conf" 35 | # IPAM type to use with Calico's CNI plugin. One of "calico-ipam" or "host-local". 36 | Set-EnvVarIfNotSet -var "CNI_IPAM_TYPE" -defaultValue "calico-ipam" 37 | 38 | ## VXLAN-specific configuration. 39 | 40 | # The VXLAN VNI / VSID. Must match the VXLANVNI felix configuration parameter used 41 | # for Linux nodes. 42 | Set-EnvVarIfNotSet -var "VXLAN_VNI" -defaultValue 4096 43 | # Prefix used when generating MAC addresses for virtual NICs. 44 | $env:VXLAN_MAC_PREFIX = "0E-2A" 45 | Set-EnvVarIfNotSet -var "VXLAN_MAC_PREFIX" -defaultValue "0E-2A" 46 | # Network Adapter used on VXLAN, leave blank for primary NIC. 47 | Set-EnvVarIfNotSet -var "VXLAN_ADAPTER" -defaultValue "" 48 | 49 | ## Node configuration. 50 | 51 | # The NODENAME variable should be set to match the Kubernetes Node name of this host. 52 | # The default uses this node's hostname (which is the same as kubelet). 53 | # 54 | # Note: on AWS, kubelet is often configured to use the internal domain name of the host rather than 55 | # the simple hostname, for example "ip-172-16-101-135.us-west-2.compute.internal". 56 | Set-EnvVarIfNotSet -var "NODENAME" -defaultValue $(hostname).ToLower() 57 | # Similarly, CALICO_K8S_NODE_REF should be set to the Kubernetes Node name. When using etcd, 58 | # the Calico kube-controllers pod will clean up Calico node objects if the corresponding Kubernetes Node is 59 | # cleaned up. 60 | Set-EnvVarIfNotSet -var "CALICO_K8S_NODE_REF" -defaultValue $env:NODENAME 61 | 62 | # The time out to wait for a valid IP of an interface to be assigned before initialising Calico 63 | # after a reboot. 64 | Set-EnvVarIfNotSet -var "STARTUP_VALID_IP_TIMEOUT" -defaultValue 90 65 | 66 | # The IP of the node; the default will auto-detect a usable IP in most cases. 67 | Set-EnvVarIfNotSet -var "IP" -defaultValue "autodetect" 68 | 69 | ## Logging. 70 | 71 | Set-EnvVarIfNotSet -var "CALICO_LOG_DIR" -defaultValue "$PSScriptRoot\logs" 72 | 73 | # Felix logs to screen at info level by default. Uncomment this line to override the log 74 | # level. Alternatively, (if this is commented out) the log level can be controlled via 75 | # the FelixConfiguration resource in the datastore. 76 | # $env:FELIX_LOGSEVERITYSCREEN = "info" 77 | # Disable logging to file by default since the service wrapper will redirect our log to file. 78 | Set-EnvVarIfNotSet -var "FELIX_LOGSEVERITYFILE" -defaultValue "none" 79 | # Disable syslog logging, which is not supported on Windows. 80 | Set-EnvVarIfNotSet -var "FELIX_LOGSEVERITYSYS" -defaultValue "none" 81 | # confd logs to screen at info level by default. Uncomment this line to override the log 82 | # level. 83 | #Set-EnvVarIfNotSet -var "BGP_LOGSEVERITYSCREEN" -defaultValue "debug" 84 | -------------------------------------------------------------------------------- /forked/node-service.ps1: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-2021 Tigera, Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http:#www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # This script is run from the main Calico folder. 16 | . .\config.ps1 17 | 18 | ipmo .\libs\calico\calico.psm1 19 | ipmo .\libs\hns\hns.psm1 20 | 21 | $lastBootTime = Get-LastBootTime 22 | $Stored = Get-StoredLastBootTime 23 | Write-Host "StoredLastBootTime $Stored, CurrentLastBootTime $lastBootTime" 24 | 25 | $timeout = $env:STARTUP_VALID_IP_TIMEOUT 26 | $vxlanAdapter = $env:VXLAN_ADAPTER 27 | 28 | # Autoconfigure the IPAM block mode. 29 | if ($env:CNI_IPAM_TYPE -EQ "host-local") { 30 | $env:USE_POD_CIDR = "true" 31 | } else { 32 | $env:USE_POD_CIDR = "false" 33 | } 34 | 35 | $platform = Get-PlatformType 36 | 37 | if ($env:CALICO_NETWORKING_BACKEND -EQ "windows-bgp" -OR $env:CALICO_NETWORKING_BACKEND -EQ "vxlan") 38 | { 39 | Write-Host "Calico $env:CALICO_NETWORKING_BACKEND networking enabled." 40 | 41 | # Check if the node has been rebooted. If so, the HNS networks will be in unknown state so we need to 42 | # clean them up and recreate them. 43 | $prevLastBootTime = Get-StoredLastBootTime 44 | if ($prevLastBootTime -NE $lastBootTime) 45 | { 46 | if ((Get-HNSNetwork | ? Type -NE nat)) 47 | { 48 | Write-Host "First time Calico has run since boot up, cleaning out any old network state." 49 | Get-HNSNetwork | ? Type -NE nat | Remove-HNSNetwork 50 | do 51 | { 52 | Write-Host "Waiting for network deletion to complete." 53 | Start-Sleep 1 54 | } while ((Get-HNSNetwork | ? Type -NE nat)) 55 | } 56 | 57 | # After deletion of all hns networks, wait for an interface to have an IP that is not a 169.254.0.0/16 (or 127.0.0.0/8) address, 58 | # before creation of External network. 59 | $isValidIP = $false 60 | $IPRegEx1='(^127\.0\.0\.)' 61 | $IPRegEx2='(^169\.254\.)' 62 | while(!($isValidIP) -AND ($timeout -gt 0)) 63 | { 64 | $IPAddress = (Get-NetIPAddress -AddressFamily IPv4).IPAddress 65 | Write-Host "`nTimeout Remaining: $timeout sec" 66 | Write-Host "List of IP Address before initialising Calico: $IPAddress" 67 | Foreach ($ip in $IPAddress) 68 | { 69 | if (($ip -NotMatch $IPRegEx1) -AND ($ip -NotMatch $IPRegEx2)) 70 | { 71 | $isValidIP = $true 72 | Write-Host "`nFound valid IP: $ip" 73 | break 74 | } 75 | } 76 | if (!($isValidIP)) 77 | { 78 | Start-Sleep -s 5 79 | $timeout = $timeout - 5 80 | } 81 | } 82 | } 83 | 84 | # Create a bridge to trigger a vSwitch creation. Do this only once 85 | Write-Host "`nStart creating vSwitch. Note: Connection may get lost for RDP, please reconnect...`n" 86 | while (!(Get-HnsNetwork | ? Name -EQ "External")) 87 | { 88 | if ($env:CALICO_NETWORKING_BACKEND -EQ "vxlan") { 89 | # FIXME Firewall rule port? 90 | New-NetFirewallRule -Name OverlayTraffic4789UDP -Description "Overlay network traffic UDP" -Action Allow -LocalPort 4789 -Enabled True -DisplayName "Overlay Traffic 4789 UDP" -Protocol UDP -ErrorAction SilentlyContinue 91 | $result = New-HNSNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; }) -AdapterName $vxlanAdapter -Verbose 92 | } 93 | else 94 | { 95 | $result = New-HNSNetwork -Type L2Bridge -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -Verbose 96 | } 97 | if ($result.Error -OR (!$result.Success)) { 98 | Write-Host "Failed to create network, retrying..." 99 | Start-Sleep 1 100 | } else { 101 | break 102 | } 103 | } 104 | 105 | # Wait for the management IP to show up and then give an extra grace period for 106 | # the networking stack to settle down. 107 | $mgmtIP = Wait-ForManagementIP "External" 108 | Write-Host "Management IP detected on vSwitch: $mgmtIP." 109 | Start-Sleep 10 110 | 111 | if (($platform -EQ "ec2") -or ($platform -EQ "gce")) { 112 | Set-MetaDataServerRoute -mgmtIP $mgmtIP 113 | } 114 | 115 | if ($env:CALICO_NETWORKING_BACKEND -EQ "windows-bgp") { 116 | Write-Host "Restarting BGP service to pick up any interface renumbering..." 117 | Restart-Service RemoteAccess 118 | } 119 | } 120 | 121 | $env:CALICO_NODENAME_FILE = ".\nodename" 122 | 123 | # We use this setting as a trigger for the other scripts to proceed. 124 | Set-StoredLastBootTime $lastBootTime 125 | $Stored = Get-StoredLastBootTime 126 | Write-Host "Stored new lastBootTime $Stored" 127 | 128 | # Run the startup script whenever kubelet (re)starts. This makes sure that we refresh our Node annotations if 129 | # kubelet recreates the Node resource. 130 | $kubeletPid = -1 131 | while ($True) 132 | { 133 | try 134 | { 135 | # Run calico-node.exe if kubelet starts/restarts 136 | $currentKubeletPid = (Get-Process -Name kubelet -ErrorAction Stop).id 137 | if ($currentKubeletPid -NE $kubeletPid) 138 | { 139 | Write-Host "Kubelet has (re)started, (re)initialising the node..." 140 | $kubeletPid = $currentKubeletPid 141 | while ($true) 142 | { 143 | .\calico-node.exe -startup 144 | if ($LastExitCode -EQ 0) 145 | { 146 | Write-Host "Calico node initialisation succeeded; monitoring kubelet for restarts..." 147 | break 148 | } 149 | 150 | Write-Host "Calico node initialisation failed, will retry..." 151 | Start-Sleep 1 152 | } 153 | } 154 | } 155 | catch 156 | { 157 | Write-Host "Kubelet not running, waiting for Kubelet to start..." 158 | $kubeletPid = -1 159 | } 160 | Start-Sleep 10 161 | } 162 | -------------------------------------------------------------------------------- /swdt.ps1: -------------------------------------------------------------------------------- 1 | Set-StrictMode -Version Latest 2 | 3 | $SWDT_CLI_ROOT = './experiments/swdt' 4 | 5 | Write-Host "[swdt.ps1] $args" 6 | 7 | if (-not (Test-Path -Path 'go.work' -PathType Leaf)) { 8 | & go work init $SWDT_CLI_ROOT 9 | } 10 | 11 | $startTime = (Get-Date) 12 | if ($args.Length -gt 0 -and $args[0] -eq 'test') { 13 | & go test -v $SWDT_CLI_ROOT/... 14 | } 15 | else { 16 | & go run -buildvcs=true $SWDT_CLI_ROOT/main.go $args | Out-Default 17 | } 18 | $runTime = (Get-Date) - $startTime 19 | 20 | Write-Host ('[swdt.ps1] Run in {0:00}:{1:00}:{2:00} and exited with {3} code' ` 21 | -f $runTime.Hours, $runTime.Minutes, $runTime.Seconds, $global:LASTEXITCODE) 22 | -------------------------------------------------------------------------------- /swdt.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | SWDT_CLI_ROOT="./experiments/swdt" 5 | 6 | echo "[swdt.sh] $*" 7 | 8 | if [[ ! -f ./go.work ]]; then 9 | go work init "$SWDT_CLI_ROOT" 10 | fi 11 | 12 | start_time=$(date +%s) 13 | if [[ "$1" == "test" ]]; then 14 | go test -v "$SWDT_CLI_ROOT/..." 15 | else 16 | go run -buildvcs=true "$SWDT_CLI_ROOT/main.go" "$@" 17 | fi 18 | stop_time=$(date +%s) 19 | 20 | printf "[swdt.sh] Run in %d seconds and exited with $? code\n" $((stop_time-start_time)) 21 | -------------------------------------------------------------------------------- /sync/linux/antrea-0.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export KUBECONFIG=/home/vagrant/.kube/config 3 | 4 | kubectl taint nodes --all node-role.kubernetes.io/master- 5 | kubectl apply -f https://github.com/antrea-io/antrea/releases/download/v1.8.0/antrea.yml 6 | 7 | sleep 20 8 | -------------------------------------------------------------------------------- /sync/linux/calico-0.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "running calico installer now with pod_cidr $1" 3 | 4 | if [[ "$1" == "" || "$2" == "" ]]; then 5 | cat << EOF 6 | Missing args. 7 | You need to send pod_cidr and calico version i.e. 8 | ./calico-0.sh.sh 100.244.0.0/16 3.25.0 9 | Normally these are in your variables.yml, and piped in by Vagrant. 10 | So, check that you didn't break the Vagrantfile :) 11 | EOF 12 | exit 1 13 | fi 14 | 15 | pod_cidr=${1} 16 | calico_version=${2} 17 | 18 | sleep 2 19 | export KUBECONFIG=/home/vagrant/.kube/config 20 | KUBECONFIG=/home/vagrant/.kube/config 21 | 22 | 23 | kubectl taint nodes --all node-role.kubernetes.io/control-plane- 24 | kubectl taint nodes --all node-role.kubernetes.io/master- 25 | 26 | 27 | kubectl create ns calico-system 28 | kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v${calico_version}/manifests/tigera-operator.yaml 29 | 30 | wget https://raw.githubusercontent.com/projectcalico/calico/v${calico_version}/manifests/custom-resources.yaml -O tigera-custom-resource.yaml 31 | sed -i "s|cidr: 192.168.0.0/16|cidr: ${pod_cidr}|g" tigera-custom-resource.yaml 32 | kubectl create -f tigera-custom-resource.yaml 33 | kubectl patch installation default --type=merge -p '{"spec": {"calicoNetwork": {"bgp": "Disabled"}}}' 34 | 35 | echo "waiting 20s for calico pods..." 36 | sleep 20 37 | 38 | wget https://raw.githubusercontent.com/projectcalico/calico/v${calico_version}/manifests/calico-windows-vxlan.yaml -O calico-windows.yaml 39 | k8s_service_host=$(kubectl get endpoints kubernetes -n default -o jsonpath='{.subsets[0].addresses[0].ip}') 40 | k8s_service_port=$(kubectl get endpoints kubernetes -n default -o jsonpath='{.subsets[0].ports[0].port}') 41 | sed -i "s|KUBERNETES_SERVICE_HOST: \"\"|KUBERNETES_SERVICE_HOST: \"$k8s_service_host\"|g" calico-windows.yaml 42 | sed -i "s|KUBERNETES_SERVICE_PORT: \"\"|KUBERNETES_SERVICE_PORT: \"$k8s_service_port\"|g" calico-windows.yaml 43 | kubectl create -f calico-windows.yaml 44 | 45 | curl -L https://github.com/projectcalico/calico/releases/download/v${calico_version}/calicoctl-linux-amd64 -o calicoctl 46 | chmod +x ./calicoctl 47 | ./calicoctl ipam configure --strictaffinity=true 48 | 49 | kubectl get pods -n calico-system -------------------------------------------------------------------------------- /sync/linux/controlplane.sh: -------------------------------------------------------------------------------- 1 | : ' 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | ' 16 | 17 | set -e 18 | 19 | echo "ARGS: $1 $2 $3" 20 | if [[ "$1" == "" || "$2" == "" || "$3" == "" ]]; then 21 | cat << EOF 22 | Missing args. 23 | You need to send kubernetes_version, k8s_kubelet_nodeip and pod_cidr i.e. 24 | ./controlplane.sh 1.21 10.20.30.10 100.244.0.0/16 25 | Normally these are in your variables.yml, and piped in by Vagrant. 26 | So, check that you didn't break the Vagrantfile :) 27 | BTW the only reason this error message is fancy is because friedrich said we should be curteous to people who want to 28 | copy paste this code from the internet and reuse it. 29 | EOF 30 | exit 1 31 | fi 32 | 33 | kubernetes_version=${1} 34 | k8s_kubelet_node_ip=${2} 35 | pod_cidr=${3} 36 | k8s_linux_apiserver="stable-${kubernetes_version}" 37 | 38 | if [[ "$3" == "" ]]; then 39 | pod_cidr="100.244.0.0/16" 40 | fi 41 | 42 | echo "Using $kubernetes_version as the Kubernetes version" 43 | 44 | # Installing packages 45 | 46 | # Add GPG keys and repository for Kubernetes 47 | echo "Setting up internet connectivity to /etc/resolv.conf" 48 | cat <<EOF | sudo tee /etc/resolv.conf 49 | nameserver 8.8.8.8 50 | nameserver 1.1.1.1 51 | EOF 52 | 53 | echo "now curling to add keys..." 54 | curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - 55 | cat << EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list 56 | deb https://apt.kubernetes.io/ kubernetes-xenial main 57 | EOF 58 | 59 | echo "SWDT: Running apt get update -y" 60 | sudo apt-get update -y 61 | # Disable swap 62 | sudo swapoff -a 63 | sudo sed -i '/swap/d' /etc/fstab 64 | 65 | cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf 66 | overlay 67 | br_netfilter 68 | EOF 69 | 70 | echo "SWDT: Running modprobes" 71 | sudo modprobe overlay 72 | sudo modprobe br_netfilter 73 | 74 | # Setup required sysctl params, these persist across reboots. 75 | cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf 76 | net.bridge.bridge-nf-call-iptables = 1 77 | net.ipv4.ip_forward = 1 78 | net.bridge.bridge-nf-call-ip6tables = 1 79 | EOF 80 | 81 | # Apply sysctl params without reboot 82 | sudo sysctl --system 83 | 84 | # Install Kubernetes binaries, using latest available, and 85 | # overwritting the binaries later. 86 | 87 | echo "SWDT installing kubelet, kubeadm, kubectl will overwrite them later as needeed..." 88 | sudo apt-get install -y kubelet kubeadm kubectl 89 | 90 | sudo apt-mark hold kubelet kubeadm kubectl 91 | 92 | # Install containerd - latest version 93 | echo "Configuring Containerd" 94 | sudo apt-get install \ 95 | ca-certificates \ 96 | gnupg \ 97 | lsb-release -y 98 | 99 | sudo mkdir -p /etc/apt/keyrings 100 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg 101 | 102 | echo \ 103 | "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ 104 | $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null 105 | 106 | sudo apt-get update 107 | 108 | sudo apt-get install containerd.io 109 | 110 | ## Test if binaries folder exists 111 | #if $overwrite_linux_bins ; then 112 | for BIN in kubeadm kubectl kubelet 113 | do 114 | file="/var/sync/linux/bin/$BIN" 115 | if [ -f $file ]; then 116 | echo "copying $file to node path.." 117 | sudo cp $file /usr/bin/ -f 118 | sudo chmod +x /usr/bin/$BIN 119 | fi 120 | done 121 | 122 | # Configuring and starting containerd 123 | sudo mkdir -p /etc/containerd 124 | containerd config default | sudo tee /etc/containerd/config.toml 125 | sudo systemctl restart containerd 126 | sudo crictl config --set runtime-endpoint=unix:///run/containerd/containerd.sock 127 | 128 | cat << EOF > /var/sync/shared/kubeadm.yaml 129 | apiVersion: kubeadm.k8s.io/v1beta3 130 | kind: InitConfiguration 131 | localAPIEndpoint: 132 | advertiseAddress: $k8s_kubelet_node_ip 133 | nodeRegistration: 134 | kubeletExtraArgs: 135 | node-ip: $k8s_kubelet_node_ip 136 | cgroup-driver: cgroupfs 137 | --- 138 | apiVersion: kubeadm.k8s.io/v1beta3 139 | kind: ClusterConfiguration 140 | kubernetesVersion: $k8s_linux_apiserver 141 | networking: 142 | podSubnet: "${pod_cidr}" 143 | EOF 144 | 145 | # Ignore kubelet mismatch in the copy process 146 | sudo kubeadm init --config=/var/sync/shared/kubeadm.yaml --v=6 --ignore-preflight-errors=KubeletVersion 147 | 148 | #to start the cluster with the current user: 149 | mkdir -p $HOME/.kube 150 | sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config 151 | sudo chown $(id -u):$(id -g) $HOME/.kube/config 152 | 153 | rm -f /var/sync/shared/config 154 | cp $HOME/.kube/config /var/sync/shared/config 155 | 156 | ######## MAKE THE JOIN FILE FOR WINDOWS ########## 157 | ######## MAKE THE JOIN FILE FOR WINDOWS ########## 158 | ######## MAKE THE JOIN FILE FOR WINDOWS ########## 159 | rm -f /var/sync/shared/kubejoin.ps1 160 | 161 | cat << EOF > /var/sync/shared/kubejoin.ps1 162 | if(!(Test-Path ("C:\Program Files\containerd\crictl.exe"))) { 163 | mv "C:\Users\vagrant\crictl.exe" "C:\Program Files\containerd\" 164 | } 165 | stop-service -name kubelet 166 | cp C:\sync\windows\bin\* c:\k 167 | 168 | \$env:path += ";C:\Program Files\containerd" 169 | [Environment]::SetEnvironmentVariable("Path", \$env:Path, [System.EnvironmentVariableTarget]::Machine) 170 | EOF 171 | 172 | kubeadm token create --print-join-command >> /var/sync/shared/kubejoin.ps1 173 | 174 | sed -i 's#--token#--cri-socket "npipe:////./pipe/containerd-containerd" --token#g' /var/sync/shared/kubejoin.ps1 175 | 176 | cat << EOF > kube-proxy-and-antrea.yaml 177 | apiVersion: v1 178 | kind: ServiceAccount 179 | metadata: 180 | labels: 181 | app: kube-proxy 182 | name: kube-proxy-windows 183 | namespace: kube-system 184 | --- 185 | apiVersion: rbac.authorization.k8s.io/v1 186 | kind: ClusterRoleBinding 187 | metadata: 188 | name: node:kube-proxy 189 | roleRef: 190 | apiGroup: rbac.authorization.k8s.io 191 | kind: ClusterRole 192 | name: system:node-proxier 193 | subjects: 194 | - kind: Group 195 | name: system:node 196 | apiGroup: rbac.authorization.k8s.io 197 | - kind: Group 198 | name: system:nodes 199 | apiGroup: rbac.authorization.k8s.io 200 | - kind: ServiceAccount 201 | name: kube-proxy-windows 202 | namespace: kube-system 203 | --- 204 | apiVersion: rbac.authorization.k8s.io/v1 205 | kind: ClusterRoleBinding 206 | metadata: 207 | name: node:god2 208 | namespace: kube-system 209 | subjects: 210 | - kind: User 211 | name: system:node:winw1 212 | apiGroup: rbac.authorization.k8s.io 213 | roleRef: 214 | kind: ClusterRole 215 | name: cluster-admin 216 | apiGroup: rbac.authorization.k8s.io 217 | --- 218 | apiVersion: rbac.authorization.k8s.io/v1 219 | kind: ClusterRoleBinding 220 | metadata: 221 | name: node:god3 222 | namespace: kube-system 223 | subjects: 224 | - kind: Group 225 | name: system:nodes 226 | apiGroup: rbac.authorization.k8s.io 227 | roleRef: 228 | kind: ClusterRole 229 | name: cluster-admin 230 | apiGroup: rbac.authorization.k8s.io 231 | --- 232 | apiVersion: rbac.authorization.k8s.io/v1 233 | kind: ClusterRoleBinding 234 | metadata: 235 | name: node:god4 236 | namespace: kube-system 237 | subjects: 238 | - kind: User 239 | name: system:serviceaccount:kube-system:kube-proxy-windows 240 | apiGroup: rbac.authorization.k8s.io 241 | roleRef: 242 | kind: ClusterRole 243 | name: cluster-admin 244 | apiGroup: rbac.authorization.k8s.io 245 | EOF 246 | 247 | kubectl create -f kube-proxy-and-antrea.yaml 248 | 249 | echo "Testing controlplane nodes!" 250 | 251 | kubectl get pods -A 252 | -------------------------------------------------------------------------------- /sync/linux/e2e.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | : ' 3 | Copyright 2021 The Kubernetes Authors. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ' 17 | set -e 18 | 19 | plugin=./wine2e.yaml 20 | downloadURL=https://github.com/vmware-tanzu/sonobuoy/releases/download/v0.54.0/sonobuoy_0.54.0_linux_amd64.tar.gz 21 | wget $downloadURL 22 | tar -xzf ./sonobuoy_0.54.0_linux_amd64.tar.gz 23 | chmod +x ./sonobuoy 24 | rm ./sonobuoy_*.tar.gz 25 | 26 | # Notes: 27 | # - sonobuoy will use the same kubectl that kubectl uses on the control plane. Assumed this will be run from there. 28 | # - Hardcoding the kubernetes version since, as of this testing, we were on an alpha release that didn't have an upstream k8s conformance image 29 | # - Choosing a single, tiny, Windows test which just executes quickly. Can expand as desired. 30 | # - Using the 'main' image of sonobuoy; can be removed once we know it works with the latest tag 31 | ./sonobuoy run --wait -p $plugin \ 32 | --config ./sonobuoyconfig.json \ 33 | --aggregator-node-selector kubernetes.io/os:linux \ 34 | --kubernetes-version=v1.22.0 \ 35 | --level=trace 36 | 37 | ./sonobuoy retrieve -f out.tar.gz 38 | ./sonobuoy results out.tar.gz 39 | numfailed=$(./sonobuoy results out.tar.gz | grep Failed | cut -d':' -f2 | awk '{$1=$1};1') 40 | if [ "$numfailed" != "0" ]; then 41 | echo "Failed a non-zero number of tests; exit 1" 42 | exit 1 43 | fi 44 | exit 0 -------------------------------------------------------------------------------- /sync/linux/smoke-test.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: nginx-deployment 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: nginx 9 | replicas: 1 10 | template: 11 | metadata: 12 | labels: 13 | app: nginx 14 | spec: 15 | tolerations: 16 | - effect: NoSchedule 17 | key: os 18 | operator: Equal 19 | value: windows 20 | - effect: NoSchedule 21 | key: os 22 | operator: Equal 23 | value: Windows 24 | containers: 25 | - name: nginx 26 | image: nginx:1.7.9 27 | securityContext: 28 | privileged: true 29 | nodeSelector: 30 | kubernetes.io/os: linux 31 | --- 32 | apiVersion: v1 33 | kind: Service 34 | metadata: 35 | name: nginx 36 | labels: 37 | app: nginx 38 | spec: 39 | ports: 40 | - port: 80 41 | targetPort: 80 42 | selector: 43 | app: nginx 44 | --- 45 | apiVersion: apps/v1 46 | kind: Deployment 47 | metadata: 48 | name: whoami-windows 49 | spec: 50 | replicas: 1 51 | selector: 52 | matchLabels: 53 | app: whoami-windows 54 | template: 55 | metadata: 56 | labels: 57 | app: whoami-windows 58 | spec: 59 | containers: 60 | - image: stefanscherer/whoami:windows-amd64-2.0.1 61 | name: whoami-windows 62 | nodeSelector: 63 | kubernetes.io/os: windows 64 | --- 65 | apiVersion: v1 66 | kind: Service 67 | metadata: 68 | name: whoami-windows 69 | labels: 70 | app: whoami-windows 71 | spec: 72 | ports: 73 | - port: 80 74 | targetPort: 8080 75 | selector: 76 | app: whoami-windows 77 | --- 78 | apiVersion: v1 79 | kind: Pod 80 | metadata: 81 | labels: 82 | run: netshoot 83 | name: netshoot 84 | spec: 85 | containers: 86 | - command: 87 | - sleep 88 | - "360000" 89 | image: nicolaka/netshoot:latest 90 | name: netshoot 91 | resources: {} 92 | nodeSelector: 93 | kubernetes.io/os: linux 94 | 95 | -------------------------------------------------------------------------------- /sync/linux/sonobuoyconfig.json: -------------------------------------------------------------------------------- 1 | {"Description":"DEFAULT","UUID":"","Version":"v0.54.0","ResultsDir":"/tmp/sonobuoy/results","Resources":null,"Filters":{"Namespaces":".*","LabelSelector":""},"Limits":{"PodLogs":{"Namespaces":"kube-system","SonobuoyNamespace":true,"FieldSelectors":[],"LabelSelector":"","Previous":false,"SinceSeconds":null,"SinceTime":null,"Timestamps":false,"TailLines":null,"LimitBytes":null,"LimitSize":"","LimitTime":""}},"QPS":30,"Burst":50,"Server":{"bindaddress":"0.0.0.0","bindport":8080,"advertiseaddress":"","timeoutseconds":21600},"Plugins":null,"PluginSearchPath":["./plugins.d","/etc/sonobuoy/plugins.d","~/sonobuoy/plugins.d"],"Namespace":"sonobuoy","WorkerImage":"sonobuoy/sonobuoy:main","ImagePullPolicy":"IfNotPresent","ImagePullSecrets":"","ProgressUpdatesPort":"8099","SecurityContextMode":"none"} 2 | -------------------------------------------------------------------------------- /sync/linux/wine2e.yaml: -------------------------------------------------------------------------------- 1 | config-map: 2 | image-repo-list-master: | 3 | gcAuthenticatedRegistry: e2eprivate 4 | gcEtcdRegistry: k8sprow.azurecr.io/kubernetes-e2e-test-images 5 | privateRegistry: e2eteam 6 | podSpec: 7 | containers: [] 8 | nodeSelector: 9 | kubernetes.io/os: linux 10 | restartPolicy: Never 11 | serviceAccountName: sonobuoy-serviceaccount 12 | tolerations: 13 | - effect: NoSchedule 14 | key: node-role.kubernetes.io/master 15 | operator: Exists 16 | - key: CriticalAddonsOnly 17 | operator: Exists 18 | - key: kubernetes.io/e2e-evict-taint-key 19 | operator: Exists 20 | sonobuoy-config: 21 | driver: Job 22 | plugin-name: wine2e 23 | result-format: junit 24 | source_url: https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/main/windows-e2e/win-e2e-image-repo-list-master.yaml 25 | description: Runs Kubernetes end-to-end tests on Windows clusters. 26 | spec: 27 | command: 28 | - /run_e2e.sh 29 | env: 30 | - name: RESULTS_DIR 31 | value: /tmp/sonobuoy/results 32 | - name: SONOBUOY_RESULTS_DIR 33 | value: /tmp/sonobuoy/results 34 | - name: E2E_FOCUS 35 | value: SecurityContext should ignore Linux Specific SecurityContext if set 36 | - name: E2E_SKIP 37 | value: \[LinuxOnly\]|\[Serial\]|GMSA|Guestbook.application.should.create.and.stop.a.working.application 38 | - name: E2E_PARALLEL 39 | value: "false" 40 | - name: E2E_USE_GO_RUNNER 41 | value: "true" 42 | - name: E2E_EXTRA_ARGS 43 | value: --progress-report-url=http://localhost:8099/progress --node-os-distro=windows 44 | - name: KUBE_TEST_REPO_LIST 45 | value: /tmp/sonobuoy/config/image-repo-list-master 46 | image: registry.k8s.io/conformance:$SONOBUOY_K8S_VERSION 47 | imagePullPolicy: Always 48 | name: e2e 49 | resources: {} 50 | volumeMounts: 51 | - mountPath: /tmp/sonobuoy/results 52 | name: results 53 | -------------------------------------------------------------------------------- /sync/shared/README: -------------------------------------------------------------------------------- 1 | stub 2 | -------------------------------------------------------------------------------- /sync/windows/0-antrea.ps1: -------------------------------------------------------------------------------- 1 | <# 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #> 16 | 17 | Set-PSDebug -Trace 1 18 | 19 | # Force Kubernetes folder 20 | mkdir -Force C:/k/ 21 | 22 | # Required configuration files symlink 23 | New-Item -ItemType HardLink -Target "C:\etc\kubernetes\kubelet.conf" -Path "C:\k\config" 24 | 25 | # Install NSSM: Workaround to privileged containers... 26 | mkdir C:/nssm/ -Force 27 | curl.exe -LO https://k8stestinfrabinaries.blob.core.windows.net/nssm-mirror/nssm-2.24.zip 28 | tar C C:/nssm/ -xvf ./nssm-2.24.zip --strip-components 2 */$arch/*.exe 29 | Remove-Item -Force ./nssm-2.24.zip 30 | 31 | # Install antrea: CNI Provider 32 | mkdir -Force C:/k/ 33 | mkdir -Force C:/k/antrea/ # scripts 34 | mkdir -Force C:/k/antrea/bin/ #executables 35 | mkdir -Force C:/k/antrea/etc/ # for antrea-agent.conf 36 | 37 | # Downloading from vcredist_x64.exe from microsoft.com is flaking inside the script, if this happens 38 | # need to do it manually. 39 | $antreaInstallationFiles = @{ 40 | "https://gist.githubusercontent.com/knabben/f478afc647152bf5c9702411296c604d/raw/fb093e42624411ba2e6933837d3664be34e09320/antrea-cni.conflist" = "C:/etc/cni/net.d/10-antrea.conflist" 41 | "https://raw.githubusercontent.com/antrea-io/antrea/main/hack/windows/Install-OVS.ps1" = "C:/k/antrea/Install-OVS.ps1" 42 | "https://raw.githubusercontent.com/antrea-io/antrea/main/hack/windows/Helper.psm1" = "C:/k/antrea/Helper.psm1" 43 | "https://github.com/antrea-io/antrea/releases/download/v1.8.0/antrea-agent-windows-x86_64.exe" = "C:/k/antrea/bin/antrea-agent.exe" 44 | "https://github.com/containernetworking/plugins/releases/download/v0.9.1/cni-plugins-windows-amd64-v0.9.1.tgz" = "C:/k/antrea/bin/cni-plugins-windows-amd64-v0.9.1.tgz" 45 | "https://gist.githubusercontent.com/knabben/5dec7c059916d3b487aeb2efd3a689b6/raw/90d50749fbce55fd80080d40585f14ce1e20b06a/antrea.yaml" = "C:/k/antrea/etc/antrea-agent.conf" 46 | # this is on jay's bucket because its otherwise a flakey download or not existent 47 | "https://storage.googleapis.com/jayunit100/Win64OpenSSL-3_0_3.exe" = "C:/ssl.exe" 48 | "https://storage.googleapis.com/jayunit100/vcd.exe" = "C:/vcd.exe" 49 | } 50 | 51 | foreach ($theURL in $antreaInstallationFiles.keys) { 52 | $outPath = $antreaInstallationFiles[$theURL] 53 | Write-Output("1 - checking $outPath ... ") 54 | if (!(Test-Path $outPath)) { 55 | Write-Output("2 - Acquiring ---> $theURL writing to $outPath") 56 | curl.exe -L $theURL -o $outPath 57 | # special logic for the host-local plugin... 58 | if ($theURL -eq "https://github.com/containernetworking/plugins/releases/download/v0.9.1/cni-plugins-windows-amd64-v0.9.1.tgz" ){ 59 | tar -xvzf C:/k/antrea/bin/cni-plugins-windows-amd64-v0.9.1.tgz 60 | cp ./host-local.exe "C:/opt/cni/bin/host-local.exe" 61 | } else { 62 | Write-Output("Nothing to do: $outPath exists in the right place already...") 63 | } 64 | Write-Output("$outPath ::: DETAILS ...") 65 | Get-ItemProperty $outPath 66 | ls $outPath 67 | Write-Output("$outPath ::: DONE VERIFYING") 68 | } 69 | if (!(Test-Path $outPath)) { 70 | Write-Error "That download totally failed $outPath is not created...." 71 | exit 123 72 | } 73 | } 74 | 75 | 76 | Write-Output("Now trying to execute VCD.exe") 77 | C:/vcd.exe /quiet /norestart 78 | 79 | Write-Output("Now trying to execute SSL.exe") 80 | C:/ssl.exe /silent /verysilent /sp- /suppressmsgboxes 81 | 82 | # Signing binaries 83 | Bcdedit.exe -set TESTSIGNING ON 84 | 85 | Start-Sleep -s 30 86 | Restart-Computer 87 | -------------------------------------------------------------------------------- /sync/windows/0-calico.ps1: -------------------------------------------------------------------------------- 1 | <# 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #> 16 | 17 | # Force Kubernetes folder 18 | mkdir -Force C:/k/ 19 | 20 | # Required configuration files symlink 21 | New-Item -ItemType HardLink -Target "C:\etc\kubernetes\kubelet.conf" -Path "C:\k\config" 22 | 23 | ## ------------------------------------------ 24 | Write-Output "Downloading Calico Artifacts" 25 | ## ------------------------------------------ 26 | 27 | # Cleanup old install 28 | rmdir C:\CalicoWindows 29 | 30 | Invoke-WebRequest https://projectcalico.docs.tigera.io/scripts/install-calico-windows.ps1 -OutFile c:\install-calico-windows.ps1 31 | C:\install-calico-windows.ps1 -DownloadOnly yes -CalicoBackend vxlan 32 | 33 | ## ------------------------------------------ 34 | Write-Output "Print installed files" 35 | ## ------------------------------------------ 36 | 37 | ls C:\CalicoWindows 38 | -------------------------------------------------------------------------------- /sync/windows/0-containerd.ps1: -------------------------------------------------------------------------------- 1 | <# 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #> 16 | 17 | Param( 18 | [parameter(HelpMessage="ContainerD Version")] 19 | [string] $calico_version="", 20 | [string] $containerd_version="" 21 | ) 22 | 23 | # download url doesn't use revision 24 | $calico_version = $calico_version | select-string "(?<majorminor>[0-9]+.[0-9]+)" | %{ $_.Matches[0].Value } 25 | 26 | ## ------------------------------------------ 27 | Write-Output "Stopping ContainerD & Kubelet" 28 | ## ------------------------------------------ 29 | 30 | Stop-Service containerd -Force 31 | 32 | 33 | ## ------------------------------------------ 34 | Write-Output "Downloading Calico using ContainerD - [calico: $calico_version] [containerd: $containerd_version]" 35 | ## ------------------------------------------ 36 | 37 | # download and extract binaries 38 | Invoke-WebRequest https://docs.tigera.io/calico/${calico_version}/scripts/Install-Containerd.ps1 -OutFile c:\Install-Containerd.ps1 39 | c:\Install-Containerd.ps1 -ContainerDVersion ${containerd_version} -CNIConfigPath "c:/etc/cni/net.d" -CNIBinPath "c:/opt/cni/bin" 40 | -------------------------------------------------------------------------------- /sync/windows/0-kubelet.ps1: -------------------------------------------------------------------------------- 1 | <# 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #> 16 | 17 | 18 | Start-Service -Name kubelet 19 | 20 | c:\CalicoWindows\install-calico.ps1 21 | 22 | c:\CalicoWindows\start-calico.ps1 23 | 24 | Write-Output "Get Calico Services" 25 | Get-Service *alico* 26 | 27 | #Write-Output("Creating Background Job for Kubelet!") 28 | #Start-Job -ScriptBlock { C:\k\StartKubelet.ps1 } 29 | 30 | #Write-Output("Background Jobs!") 31 | #Get-Job -------------------------------------------------------------------------------- /sync/windows/1-antrea.ps1: -------------------------------------------------------------------------------- 1 | <# 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #> 16 | Param( 17 | [parameter(HelpMessage="Windows Node IP")] 18 | [string] $windowsNodeIP = "10.20.30.11" 19 | ) 20 | $ErrorActionPreference = 'Stop' 21 | 22 | $folders = @('C:\k\antrea','C:\var\log\antrea','C:\k\antrea\bin', 'C:\var\log\kube-proxy', 'C:\opt\cni\bin', 'C:\etc\cni\net.d') 23 | foreach ($f in $folders) { 24 | New-Item -ItemType Directory -Force -Path $f 25 | } 26 | 27 | # ### Installing OVS 28 | 29 | # If you are doing this in production, you want to use the LocalFile option and 30 | # and you may want to run a signed OVS copy provided by a vendor 31 | C:/k/antrea/Install-OVS.ps1 32 | 33 | # Verify the OVs services are installed 34 | get-service ovsdb-server 35 | get-service ovs-vswitchd 36 | 37 | # Disable Windows Firewall 38 | Set-NetFirewallProfile -Profile Domain,Public,Private -Enabled False 39 | 40 | # ### Installing Antrea Agent 41 | 42 | # Add Windows Defender Options 43 | $avexceptions = @('c:\program files\containerd\ctr.exe', 'c:\program files\containerd\containerd.exe' ) 44 | foreach ($e in $avexceptions) { 45 | Add-MpPreference -ExclusionProcess $e 46 | } 47 | 48 | # Get HostIP and set in kubeadm-flags.env 49 | $file = 'C:\var\lib\kubelet\kubeadm-flags.env' 50 | $newstr ="--node-ip=$($windowsNodeIP)" # pass from args 51 | 52 | $raw = Get-Content -Path $file -TotalCount 1 53 | $raw = $raw -replace ".$" 54 | $new = "$($raw) $($newstr)`"" 55 | Set-Content $file $new 56 | $KubeConfigFile='C:\etc\kubernetes\kubelet.conf' 57 | 58 | # Setup kube-proxy config file 59 | $KubeProxyConfig="C:/k/antrea/etc/kube-proxy.conf" 60 | $KubeAPIServer=$(kubectl --kubeconfig=$KubeConfigFile config view -o jsonpath='{.clusters[0].cluster.server}') 61 | $KubeProxyTOKEN=$(kubectl --kubeconfig=$KubeConfigFile get secrets -n kube-system -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='antrea-agent')].data.token}") 62 | $KubeProxyTOKEN=$([System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($KubeProxyTOKEN))) 63 | 64 | # This writes out a kubeconfig file... i think ! 65 | kubectl config --kubeconfig=$KubeProxyConfig set-cluster kubernetes --server=$KubeAPIServer --insecure-skip-tls-verify 66 | 67 | # Now we set the defaults up... 68 | # Remember: Kube proxy needs to be happy for antrea to work, because 69 | # Antrea will attempt to access the APIServer through the kube proxy 70 | # Provisioned access point. 71 | kubectl config --kubeconfig=$KubeProxyConfig set-credentials kube-proxy-windows --token=$KubeProxyTOKEN 72 | kubectl config --kubeconfig=$KubeProxyConfig set-context kube-proxy-windows@kubernetes --cluster=kubernetes --user=kube-proxy-windows 73 | kubectl config --kubeconfig=$KubeProxyConfig use-context kube-proxy-windows@kubernetes 74 | if (!(Test-Path $KubeProxyConfig)) { 75 | Write-Output "$KubeProxyConfig is missing !!!" 76 | Write-Error "FATAL ERROR, CANNOT START ANTREA WITHOUT A VALID KUBE PROXY CONFIGURATION !!!" 77 | exit 5 78 | } 79 | 80 | # Wait for antrea-agent token to be ready 81 | $AntreaToken=$null 82 | $LoopCount=5 83 | do { 84 | $LoopCount=$LoopCount-1 85 | if ($LoopCount -eq 0) { 86 | break 87 | } 88 | Write-Output "Trying to set antrea token ~ via getting secrets from kubeconfig file" 89 | Write-Output $LoopCount 90 | sleep 120 91 | $AntreaToken=$(kubectl --kubeconfig=$KubeConfigFile get secrets -n kube-system -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='antrea-agent')].data.token}") 92 | } while ($null -eq $AntreaToken) 93 | 94 | # Install antrea-agent 95 | $antrea_helper="C:/k/antrea/Helper.psm1" 96 | if (!(Test-Path $antrea_helper)) { 97 | Write-Error "Couldnt find Helper.psm1 anywhere !!!" 98 | } 99 | Import-Module $antrea_helper 100 | 101 | & Install-AntreaAgent -KubernetesHome "c:/k" -KubeConfig "C:/etc/kubernetes/kubelet.conf" -AntreaVersion "v1.8.0" -AntreaHome "c:/k/antrea" 102 | New-KubeProxyServiceInterface 103 | 104 | # ### Installing Kube-Proxy 105 | 106 | $nssm = (Get-Command nssm).Source 107 | & $nssm set Kubelet start SERVICE_AUTO_START 108 | & nssm install kube-proxy "C:/k/kube-proxy.exe" "--proxy-mode=userspace --kubeconfig=$KubeProxyConfig --log-dir=c:/var/log/kube-proxy --logtostderr=false --alsologtostderr" 109 | 110 | & nssm install antrea-agent "C:/k/antrea/bin/antrea-agent.exe" "-v=3 --config=C:/k/antrea/etc/antrea-agent.conf --logtostderr=false --log_dir=c:/var/log/antrea --alsologtostderr --log_file_max_size=100 --log_file_max_num=4" 111 | & nssm set antrea-agent DependOnService kube-proxy ovs-vswitchd 112 | & nssm set antrea-agent Start SERVICE_DELAYED_AUTO_START 113 | 114 | # Start Services 115 | restart-service kubelet 116 | start-service kube-proxy 117 | Write-Output("...sleeping for a second before smoke testing...") 118 | sleep 5 119 | 120 | # Must happen *after* kube-proxy comes online.... so internal service endpoint is accessible to antrea-agent. 121 | Get-Service *kube* 122 | Get-Service *antrea* 123 | Get-Service *ovs* 124 | 125 | $antrea = Get-Service -Name "antrea-agent" 126 | $antrea_starts = 0 127 | while ($antrea.Status -ne 'Running') 128 | { 129 | Write-Output("... Trying to start antrea service... $antrea_starts") 130 | Start-Service "antrea-agent" 131 | $antrea_starts = $antrea_starts + 1 132 | $antrea.Refresh() 133 | } 134 | Write-Output("Done starting antrea... ") 135 | # Get-Service *kube* 136 | Get-Service *ovs* 137 | Get-Service *antrea* 138 | -------------------------------------------------------------------------------- /sync/windows/1-calico.ps1: -------------------------------------------------------------------------------- 1 | cd C:\CalicoWindows 2 | 3 | if (-not(Test-Path -Path $env:KUBECONFIG -PathType Leaf)) { 4 | Write-Output "Missing KUBECONFIG env var ! exiting $env:KUBECONFIG" 5 | exit 1 6 | } 7 | 8 | if (-not (Test-Path -Path ./install-calico.ps1 -PathType Leaf)) { 9 | Write-Output "WARNING WARNING WARNING : I DONT SEE THE INSTALL-CALICO.ps1 FILE !!!!!!!!!!" 10 | ls ./ 11 | exit 100 12 | } 13 | 14 | ## ------------------------------------------ 15 | Write-Output "Running install-calico.ps1 script" 16 | ## ------------------------------------------ 17 | 18 | cp C:/forked/config.ps1 .\config.ps1 19 | 20 | Get-ChildItem env: 21 | Get-Date 22 | c:\CalicoWindows\install-calico.ps1 23 | Write-Output "Done installing Calico!" 24 | 25 | ## ------------------------------------------ 26 | Write-Output "Starting calico felix" 27 | ## ------------------------------------------ 28 | Start-Service -Name CalicoFelix 29 | Start-Service -Name CalicoNode 30 | 31 | Write-Output "Checking for calico services ..." 32 | Get-Service *ico* 33 | 34 | ## ------------------------------------------ 35 | Write-Output "Starting Kube-proxy..." 36 | ## ------------------------------------------ 37 | C:\CalicoWindows\kubernetes\install-kube-services.ps1 38 | Start-Service -Name kube-proxy 39 | -------------------------------------------------------------------------------- /sync/windows/forked.ps1: -------------------------------------------------------------------------------- 1 | Param( 2 | [parameter(HelpMessage="Kubernetes Version")] 3 | [string] $kubernetesVersion="" 4 | ) 5 | 6 | # Force Kubernetes folder 7 | mkdir -Force C:/k/ 8 | 9 | # Copy a clean StartKubelet.ps1 configuration for 1.24+ 10 | If ([int]$kubernetesVersion.split(".",2)[1] -gt 23) { 11 | cp C:/forked/StartKubelet.ps1 c:\k\StartKubelet.ps1 12 | } 13 | -------------------------------------------------------------------------------- /sync/windows/ssh.ps1: -------------------------------------------------------------------------------- 1 | <# 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | #> 16 | 17 | $adminpath = "c:\ProgramData\ssh" 18 | $adminfile = "administrators_authorized_keys" 19 | 20 | $sshdService = Get-Service | ? Name -like 'sshd' 21 | if ($sshdService.Count -eq 0) 22 | { 23 | Write-Output "Installing OpenSSH" 24 | $isAvailable = Get-WindowsCapability -Online | ? Name -like 'OpenSSH*' 25 | 26 | if (!$isAvailable) { 27 | throw "OpenSSH is not available on this machine" 28 | } 29 | 30 | Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0 31 | } 32 | else 33 | { 34 | Write-Output "OpenSSH Server service detected - skipping online install..." 35 | } 36 | 37 | Start-Service sshd 38 | 39 | if (!(Test-Path "$adminpath")) { 40 | Write-Output "Created new file and text content added" 41 | New-Item -path $adminpath -name $adminfile -type "file" -value "" 42 | } 43 | 44 | Write-Output "$adminpath found." 45 | 46 | Write-Output "Setting required permissions..." 47 | icacls $adminpath\$adminfile /remove "NT AUTHORITY\Authenticated Users" 48 | icacls $adminpath\$adminfile /inheritance:r 49 | icacls $adminpath\$adminfile /grant SYSTEM:`(F`) 50 | icacls $adminpath\$adminfile /grant BUILTIN\Administrators:`(F`) 51 | 52 | # todo(knabben) - import vagrant pub key 53 | 54 | Write-Output "Restarting sshd service..." 55 | Restart-Service sshd 56 | # OPTIONAL but recommended: 57 | Set-Service -Name sshd -StartupType 'Automatic' 58 | # Dissabling all the firewalls 59 | Set-NetFirewallProfile -Profile Domain,Public,Private -Enabled False 60 | # Confirm the Firewall rule is configured. It should be created automatically by setup. 61 | $firewall = Get-NetFirewallRule -Name *ssh* 62 | 63 | if (!$firewall) { 64 | throw "OpenSSH is firewall is not configured properly" 65 | } 66 | Write-Output "OpenSSH installed and configured successfully" 67 | 68 | -------------------------------------------------------------------------------- /variables.yaml: -------------------------------------------------------------------------------- 1 | # Kubernetes version, pick major and minor, patch will be the latest released. 2 | kubernetes_version: "1.26" 3 | 4 | # if this options is used BINARIES will be downloaded instead of building from zero 5 | build_from_source: "false" 6 | 7 | # ip pool for pods 8 | pod_cidr: "100.244.0.0/16" 9 | 10 | # containerd version 11 | containerd_version: "1.6.15" 12 | calico_version: "3.25.0" 13 | 14 | ## Linux settings 15 | k8s_linux_kubelet_nodeip: "10.20.30.10" 16 | windows_node_ip: "10.20.30.11" 17 | windows_ram: 6048 18 | linux_ram: 4096 19 | linux_cpus: 2 20 | windows_cpus: 4 21 | 22 | ## CHOOSE YOUR CNI ($) 23 | # cni: "calico" || "antrea" 24 | cni: "calico" 25 | --------------------------------------------------------------------------------