├── .dockerignore ├── .editorconfig ├── .env ├── .eslintrc.js ├── .github └── workflows │ ├── docker-publish-release.yml │ └── docker-publish.yml ├── .gitignore ├── .nvmrc ├── .prettierignore ├── .prettierrc.js ├── .vscode ├── extensions.json └── settings.json ├── Dockerfile ├── Gruntfile.js ├── LICENSE ├── README.md ├── docker-compose.yml ├── entrypoint.sh ├── grafana-dashboard.json ├── helmfile ├── charts │ └── kconmon │ │ ├── Chart.yaml │ │ ├── templates │ │ ├── _helpers.tpl │ │ ├── agent │ │ │ ├── daemonset.yaml │ │ │ ├── service.yaml │ │ │ └── servicemonitor.yaml │ │ └── controller │ │ │ ├── clusterrole.yaml │ │ │ ├── clusterrolebinding.yaml │ │ │ ├── deployment.yaml │ │ │ ├── pdb.yaml │ │ │ ├── role.yaml │ │ │ ├── rolebinding.yaml │ │ │ ├── service.yaml │ │ │ └── serviceaccount.yaml │ │ └── values.yaml └── helmfile.yaml ├── images └── pretty.png ├── lib ├── apps │ ├── agent │ │ ├── controllers │ │ │ └── index.ts │ │ ├── index.ts │ │ ├── metrics.ts │ │ └── routes │ │ │ └── index.ts │ └── controller │ │ ├── controllers │ │ └── index.ts │ │ ├── index.ts │ │ └── routes │ │ └── index.ts ├── config │ └── index.ts ├── discovery │ ├── index.ts │ ├── kubernetes.ts │ └── service.ts ├── kubernetes │ ├── client.ts │ ├── models.ts │ └── monkeypatch │ │ └── client.ts ├── logger │ └── index.ts ├── tester │ └── index.ts ├── udp │ ├── client.ts │ ├── clientFactory.ts │ └── server.ts └── web-server │ └── index.ts ├── package.json ├── samples └── ping.ts ├── screenshots └── grafana.png ├── test ├── discovery │ ├── kubernetes.test.ts │ └── service.test.ts ├── helpers.ts ├── kubernetes │ └── client.test.ts ├── logger │ └── logger.test.ts ├── samples │ ├── agent.json │ ├── agents.json │ └── node.json ├── tester.test.ts ├── udp.test.ts └── webserver.test.ts └── tsconfig.json /.dockerignore: -------------------------------------------------------------------------------- 1 | .tscache 2 | built/ 3 | 4 | .env 5 | reports/ 6 | node_modules/ 7 | package-lock.json 8 | package-logs.json 9 | testresults/ 10 | stats/ 11 | docs/* 12 | .vagrant/ 13 | coverage/ 14 | 15 | *.tar.gz 16 | *.tgz 17 | *.swp 18 | *.log 19 | *.zat 20 | 21 | checkstyle-result.xml 22 | coverage.* 23 | junit.xml 24 | shippable/ 25 | 26 | kubernetes 27 | Dockerfile 28 | Dockerfile.worker 29 | docker-compose.yml 30 | .dockerignore 31 | .editorconfig 32 | .git 33 | .git-crypt 34 | .gitignore 35 | .idea 36 | samples/ 37 | 38 | helmfile 39 | smoke/ 40 | sniff/ 41 | sniff-namespace.sh 42 | README.md 43 | built/ 44 | images/ 45 | .vscode 46 | .tscache 47 | .github 48 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # This file is for unifying the coding style for different editors and IDEs 2 | # editorconfig.org 3 | root = true 4 | 5 | [*] 6 | end_of_line = lf 7 | charset = utf-8 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | indent_style = space 11 | indent_size = 2 12 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | SOME_ENV_VARIABLE=some-value 2 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | module.exports = { 4 | env: { 5 | es6: true, 6 | node: true 7 | }, 8 | parserOptions: { 9 | sourceType: 'module' 10 | }, 11 | extends: [ 12 | 'eslint:recommended', 13 | 'prettier', 14 | 'plugin:@typescript-eslint/recommended', 15 | 'plugin:mocha/recommended' 16 | ], 17 | parser: '@typescript-eslint/parser', 18 | plugins: ['@typescript-eslint/eslint-plugin', 'prettier', 'mocha'], 19 | rules: { 20 | '@typescript-eslint/no-namespace': 'off', 21 | '@typescript-eslint/no-empty-interface': 'off', 22 | '@typescript-eslint/no-var-requires': 'off', 23 | '@typescript-eslint/no-explicit-any': 'off', 24 | '@typescript-eslint/explicit-module-boundary-types': 'off', 25 | 'prettier/prettier': 'error', 26 | 'accessor-pairs': 'error', 27 | 'array-bracket-newline': 'off', 28 | 'array-bracket-spacing': 'error', 29 | 'array-callback-return': 'error', 30 | 'array-element-newline': ['warn', 'consistent'], 31 | 'arrow-body-style': 'off', 32 | 'arrow-parens': 'error', 33 | 'arrow-spacing': 'error', 34 | 'block-scoped-var': 'error', 35 | 'block-spacing': 'error', 36 | 'brace-style': ['off', '1tbs'], 37 | 'callback-return': 'off', 38 | camelcase: 'off', 39 | 'capitalized-comments': 'off', 40 | 'class-methods-use-this': 'off', 41 | 'comma-dangle': 'error', 42 | 'comma-spacing': [ 43 | 'error', 44 | { 45 | after: true, 46 | before: false 47 | } 48 | ], 49 | 'comma-style': 'error', 50 | complexity: 'error', 51 | 'computed-property-spacing': 'error', 52 | 'consistent-return': 'error', 53 | 'consistent-this': 'warn', 54 | curly: 'error', 55 | 'default-case': 'error', 56 | 'dot-location': ['error', 'property'], 57 | 'dot-notation': 'error', 58 | 'eol-last': 'error', 59 | eqeqeq: 'error', 60 | 'for-direction': 'error', 61 | 'func-call-spacing': 'error', 62 | 'func-name-matching': 'error', 63 | 'func-names': ['error', 'never'], 64 | 'func-style': [ 65 | 'error', 66 | 'declaration', 67 | { 68 | allowArrowFunctions: true 69 | } 70 | ], 71 | 'function-paren-newline': 'off', 72 | 'generator-star-spacing': 'error', 73 | 'getter-return': 'error', 74 | 'global-require': 'off', 75 | 'guard-for-in': 'error', 76 | 'handle-callback-err': 'error', 77 | 'id-blacklist': 'error', 78 | 'id-length': ['off', { exceptions: ['_'] }], 79 | 'id-match': 'error', 80 | 'implicit-arrow-linebreak': 'off', 81 | indent: 'off', 82 | 'indent-legacy': 'off', 83 | 'init-declarations': 'off', 84 | 'jsx-quotes': 'error', 85 | 'key-spacing': 'error', 86 | 'keyword-spacing': [ 87 | 'error', 88 | { 89 | after: true, 90 | before: true 91 | } 92 | ], 93 | 'line-comment-position': 'error', 94 | 'linebreak-style': ['error', 'unix'], 95 | 'lines-around-comment': 'off', 96 | 'lines-around-directive': 'off', 97 | 'lines-between-class-members': 'off', 98 | 'max-depth': 'error', 99 | 'max-len': 'off', 100 | 'max-lines': 'off', 101 | 'max-nested-callbacks': 'error', 102 | 'max-params': ['error', 10], 103 | 'max-statements': ['warn', 30], 104 | 'max-statements-per-line': 'error', 105 | 'multiline-comment-style': 'off', 106 | 'multiline-ternary': 'off', 107 | 'new-cap': 'off', 108 | 'new-parens': 'error', 109 | 'newline-after-var': ['off', 'always'], 110 | 'newline-before-return': 'off', 111 | 'newline-per-chained-call': 'off', 112 | 'no-alert': 'error', 113 | 'no-console': 'off', 114 | 'no-array-constructor': 'error', 115 | 'no-await-in-loop': 'off', 116 | 'no-bitwise': 'error', 117 | 'no-buffer-constructor': 'error', 118 | 'no-caller': 'error', 119 | 'no-catch-shadow': 'error', 120 | 'no-confusing-arrow': 'warn', 121 | 'no-continue': 'error', 122 | 'no-div-regex': 'error', 123 | 'no-duplicate-imports': 'error', 124 | 'no-else-return': 'error', 125 | 'no-empty-function': 'off', 126 | 'no-eq-null': 'error', 127 | 'no-eval': 'error', 128 | 'no-extend-native': 'error', 129 | 'no-extra-bind': 'error', 130 | 'no-extra-label': 'error', 131 | 'no-extra-parens': 'off', 132 | 'no-floating-decimal': 'error', 133 | 'no-implicit-coercion': 'error', 134 | 'no-implicit-globals': 'error', 135 | 'no-implied-eval': 'error', 136 | 'no-inline-comments': 'error', 137 | 'no-invalid-this': 'error', 138 | 'no-iterator': 'error', 139 | 'no-label-var': 'error', 140 | 'no-labels': 'error', 141 | 'no-lone-blocks': 'error', 142 | 'no-lonely-if': 'error', 143 | 'no-loop-func': 'error', 144 | 'no-magic-numbers': 'off', 145 | 'no-mixed-operators': 'error', 146 | 'no-mixed-requires': 'error', 147 | 'no-multi-assign': 'error', 148 | 'no-multi-spaces': 'off', 149 | 'no-multi-str': 'error', 150 | 'no-multiple-empty-lines': 'error', 151 | 'no-native-reassign': 'error', 152 | 'no-negated-condition': 'error', 153 | 'no-negated-in-lhs': 'error', 154 | 'no-nested-ternary': 'error', 155 | 'no-new': 'error', 156 | 'no-new-func': 'error', 157 | 'no-new-object': 'error', 158 | 'no-new-require': 'error', 159 | 'no-new-wrappers': 'error', 160 | 'no-octal-escape': 'error', 161 | 'no-param-reassign': 'error', 162 | 'no-path-concat': 'error', 163 | 'no-plusplus': 'error', 164 | 'no-process-env': 'off', 165 | 'no-process-exit': 'off', 166 | 'no-proto': 'error', 167 | 'no-prototype-builtins': 'error', 168 | 'no-restricted-globals': 'error', 169 | 'no-restricted-imports': 'error', 170 | 'no-restricted-modules': 'error', 171 | 'no-restricted-properties': 'error', 172 | 'no-restricted-syntax': 'error', 173 | 'no-return-assign': 'error', 174 | 'no-return-await': 'error', 175 | 'no-script-url': 'error', 176 | 'no-self-compare': 'error', 177 | 'no-sequences': 'error', 178 | 'no-shadow': 'error', 179 | 'no-shadow-restricted-names': 'error', 180 | 'no-spaced-func': 'error', 181 | 'no-sync': 'off', 182 | 'no-tabs': 'error', 183 | 'no-template-curly-in-string': 'error', 184 | 'no-ternary': 'off', 185 | 'no-throw-literal': 'error', 186 | 'no-trailing-spaces': 'error', 187 | 'no-undef': 'off', 188 | 'no-undef-init': 'error', 189 | 'no-undefined': 'warn', 190 | 'no-underscore-dangle': 'error', 191 | 'no-unmodified-loop-condition': 'error', 192 | 'no-unneeded-ternary': 'error', 193 | 'no-unused-expressions': 'error', 194 | 'no-unused-vars': 'error', 195 | '@typescript-eslint/no-unused-vars': 'error', 196 | 'no-use-before-define': 'error', 197 | 'no-useless-call': 'error', 198 | 'no-useless-computed-key': 'error', 199 | 'no-useless-concat': 'error', 200 | 'no-useless-constructor': 'error', 201 | 'no-useless-rename': 'error', 202 | 'no-useless-return': 'error', 203 | 'no-var': 'off', 204 | 'no-void': 'error', 205 | 'no-warning-comments': 'warn', 206 | 'no-whitespace-before-property': 'error', 207 | 'no-with': 'error', 208 | 'nonblock-statement-body-position': 'error', 209 | 'object-curly-newline': 'error', 210 | 'object-curly-spacing': ['error', 'always'], 211 | 'object-property-newline': ['warn', { allowAllPropertiesOnSameLine: true }], 212 | 'object-shorthand': 'error', 213 | 'one-var': 'off', 214 | 'one-var-declaration-per-line': 'error', 215 | 'operator-assignment': 'error', 216 | 'operator-linebreak': 'error', 217 | 'padded-blocks': 'off', 218 | 'padding-line-between-statements': 'error', 219 | 'prefer-arrow-callback': 'off', 220 | 'prefer-const': 'error', 221 | 'prefer-destructuring': 'off', 222 | 'prefer-numeric-literals': 'error', 223 | 'prefer-promise-reject-errors': 'error', 224 | 'prefer-reflect': 'off', 225 | 'prefer-rest-params': 'error', 226 | 'prefer-spread': 'error', 227 | 'prefer-template': 'error', 228 | 'quote-props': 'off', 229 | quotes: 'off', 230 | radix: 'error', 231 | 'require-await': 'off', 232 | 'require-jsdoc': [ 233 | 'off', 234 | { 235 | require: { 236 | FunctionDeclaration: false, 237 | MethodDefinition: true, 238 | ClassDeclaration: false, 239 | ArrowFunctionExpression: false, 240 | FunctionExpression: false 241 | } 242 | } 243 | ], 244 | 'rest-spread-spacing': 'error', 245 | semi: 'off', 246 | 'semi-spacing': 'error', 247 | 'semi-style': 'off', 248 | 'sort-imports': 'off', 249 | 'sort-keys': 'off', 250 | 'sort-vars': 'error', 251 | 'space-before-blocks': 'error', 252 | 'space-before-function-paren': 'off', 253 | 'space-in-parens': ['error', 'never'], 254 | 'space-infix-ops': 'error', 255 | 'space-unary-ops': 'error', 256 | 'spaced-comment': 'error', 257 | strict: 'error', 258 | 'switch-colon-spacing': 'error', 259 | 'symbol-description': 'error', 260 | 'template-curly-spacing': ['error', 'never'], 261 | 'template-tag-spacing': 'error', 262 | 'unicode-bom': ['error', 'never'], 263 | 'valid-jsdoc': 'error', 264 | 'vars-on-top': 'error', 265 | 'wrap-iife': 'error', 266 | 'wrap-regex': 'error', 267 | 'yield-star-spacing': 'error', 268 | yoda: 'error', 269 | 270 | // typescript-eslint rules 271 | '@typescript-eslint/interface-name-prefix': 'off', 272 | '@typescript-eslint/no-empty-function': 'off', 273 | 274 | // eslint-plugin-mocha rules 275 | 'mocha/handle-done-callback': 'error', 276 | 'mocha/max-top-level-suites': ['error', { limit: 5 }], 277 | 'mocha/no-async-describe': 'error', 278 | 'mocha/no-exclusive-tests': 'warn', 279 | 'mocha/no-global-tests': 'error', 280 | 'mocha/no-hooks': 'off', 281 | 'mocha/no-hooks-for-single-case': 'off', 282 | 'mocha/no-identical-title': 'error', 283 | 'mocha/no-mocha-arrows': 'off', 284 | 'mocha/no-nested-tests': 'error', 285 | 'mocha/no-pending-tests': 'warn', 286 | 'mocha/no-return-and-callback': 'error', 287 | 'mocha/no-return-from-async': 'off', 288 | 'mocha/no-setup-in-describe': 'off', 289 | 'mocha/no-sibling-hooks': 'error', 290 | 'mocha/no-skipped-tests': 'warn', 291 | 'mocha/no-synchronous-tests': 'off', 292 | 'mocha/no-top-level-hooks': 'off', 293 | 'mocha/prefer-arrow-callback': 'off', 294 | 'mocha/valid-suite-description': 'off', 295 | 'mocha/valid-test-description': 'off' 296 | } 297 | } 298 | -------------------------------------------------------------------------------- /.github/workflows/docker-publish-release.yml: -------------------------------------------------------------------------------- 1 | name: Publish Release 2 | 3 | on: 4 | create: 5 | tags: 6 | - v* 7 | 8 | env: 9 | DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} 10 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 11 | IMAGE: stono/kconmon 12 | 13 | jobs: 14 | config: 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - name: Set env 19 | run: echo ::set-env name=VERSION::${GITHUB_REF#refs/tags/v} 20 | 21 | - name: Echo env 22 | run: | 23 | if [ -z "$VERSION" ]; then 24 | echo "ERROR: Version was not set" 25 | exit 1 26 | fi 27 | echo Target version: $VERSION 28 | 29 | docker: 30 | needs: config 31 | runs-on: ubuntu-latest 32 | 33 | steps: 34 | - name: Checkout code 35 | uses: actions/checkout@v2 36 | 37 | - name: Set env 38 | run: echo ::set-env name=VERSION::${GITHUB_REF#refs/tags/v} 39 | 40 | - name: Pull image 41 | run: VERSION=latest docker-compose pull controller 42 | 43 | - name: Build image 44 | run: docker-compose build controller 45 | 46 | - name: Log into registry 47 | run: echo "${{ secrets.DOCKERHUB_TOKEN }}" | docker login -u stono --password-stdin 48 | 49 | - name: Push 0.0.0 image 50 | run: docker-compose push controller 51 | 52 | helm: 53 | needs: config 54 | runs-on: ubuntu-latest 55 | 56 | steps: 57 | - name: Checkout code 58 | uses: actions/checkout@v2 59 | 60 | - name: Set env 61 | run: echo ::set-env name=VERSION::${GITHUB_REF#refs/tags/v} 62 | 63 | - name: Package Helm 64 | run: | 65 | cd ./helmfile/charts/kconmon 66 | helm package --version=$VERSION . 67 | mv kconmon*.tgz kconmon-chart.tgz 68 | 69 | - name: Upload the helm chart 70 | uses: actions/upload-artifact@v1 71 | with: 72 | name: helm-chart 73 | path: ./helmfile/charts/kconmon/kconmon-chart.tgz 74 | 75 | release: 76 | needs: 77 | - docker 78 | - helm 79 | runs-on: ubuntu-latest 80 | 81 | steps: 82 | - name: Set output 83 | id: set_output 84 | run: echo ::set-output name=VERSION::${GITHUB_REF#refs/tags/v} 85 | 86 | - name: Create Release 87 | id: create_release 88 | uses: actions/create-release@v1 89 | with: 90 | tag_name: ${{ steps.set_output.outputs.VERSION }} 91 | release_name: ${{ steps.set_output.outputs.VERSION }} 92 | draft: false 93 | prerelease: false 94 | 95 | - name: Download helm chart 96 | uses: actions/download-artifact@v1 97 | with: 98 | name: helm-chart 99 | 100 | - name: Upload Release Asset 101 | id: upload-release-asset 102 | uses: actions/upload-release-asset@v1 103 | with: 104 | upload_url: ${{ steps.create_release.outputs.upload_url }} 105 | asset_path: ./helm-chart/kconmon-chart.tgz 106 | asset_name: helm-chart.tgz 107 | asset_content_type: application/gzip 108 | -------------------------------------------------------------------------------- /.github/workflows/docker-publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish Latest 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | paths-ignore: 8 | - README.md 9 | - helmfile/** 10 | - images/** 11 | - .vscode/** 12 | - samples/** 13 | 14 | env: 15 | DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} 16 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 17 | IMAGE: stono/kconmon 18 | VERSION: 0.0.0 19 | 20 | jobs: 21 | docker: 22 | runs-on: ubuntu-latest 23 | if: github.event_name == 'push' 24 | 25 | steps: 26 | - name: Checkout code 27 | uses: actions/checkout@v2 28 | 29 | - name: Pull image 30 | run: VERSION=latest docker-compose pull controller 31 | 32 | - name: Build image 33 | run: docker-compose build controller 34 | 35 | - name: Log into registry 36 | run: echo "${{ secrets.DOCKERHUB_TOKEN }}" | docker login -u stono --password-stdin 37 | 38 | - name: Push 0.0.0 image 39 | run: docker-compose push controller 40 | 41 | - name: Push latest image 42 | run: | 43 | docker tag $IMAGE:$VERSION $IMAGE:latest 44 | docker push $IMAGE:latest 45 | 46 | helm: 47 | runs-on: ubuntu-latest 48 | if: github.event_name == 'push' 49 | 50 | steps: 51 | - name: Checkout code 52 | uses: actions/checkout@v2 53 | 54 | - name: Package Helm 55 | run: | 56 | cd ./helmfile/charts/kconmon 57 | helm package --version=$VERSION . 58 | mv kconmon*.tgz kconmon-chart.tgz 59 | 60 | - name: Upload the helm chart 61 | uses: actions/upload-artifact@v1 62 | with: 63 | name: helm-chart 64 | path: ./helmfile/charts/kconmon/kconmon-chart.tgz 65 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | tscommand-* 2 | .DS_Store 3 | .tscache 4 | built 5 | 6 | reports/ 7 | node_modules/ 8 | testresults/ 9 | stats/ 10 | docs/* 11 | .vagrant/ 12 | coverage/ 13 | package-logs.json 14 | package-lock.json 15 | 16 | *.tar.gz 17 | *.tgz 18 | *.swp 19 | *.log 20 | *.zat 21 | 22 | checkstyle-result.xml 23 | coverage.* 24 | junit.xml 25 | scanLogs 26 | 27 | sniff/ 28 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 14.6 2 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | README.md 2 | -------------------------------------------------------------------------------- /.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | editorconfig: true, 3 | trailingComma: 'none', 4 | semi: false, 5 | singleQuote: true 6 | } 7 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "esbenp.prettier-vscode", 4 | "editorconfig.editorconfig", 5 | "dbaeumer.vscode-eslint", 6 | "henrynguyen5-vsc.vsc-nvm" 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.defaultFormatter": "esbenp.prettier-vscode", 3 | "editor.formatOnSave": true, 4 | "typescript.tsdk": "./node_modules/typescript/lib" 5 | } 6 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG NODEJS_VERSION 2 | FROM mhart/alpine-node:${NODEJS_VERSION} as build 3 | WORKDIR /app 4 | 5 | COPY package.json ./ 6 | RUN npm install 7 | COPY . ./ 8 | RUN ./node_modules/.bin/grunt 9 | RUN ./node_modules/.bin/grunt test 10 | RUN ./node_modules/.bin/grunt build 11 | 12 | FROM mhart/alpine-node:${NODEJS_VERSION} as deploy 13 | COPY entrypoint.sh /usr/local/bin/entrypoint.sh 14 | ENTRYPOINT [ "/usr/local/bin/entrypoint.sh" ] 15 | 16 | WORKDIR /app 17 | RUN addgroup nonroot && \ 18 | adduser -D nonroot -G nonroot && \ 19 | chown nonroot:nonroot /app 20 | 21 | USER nonroot 22 | RUN mkdir -p /home/nonroot/.npm 23 | VOLUME /home/nonroot/.npm 24 | COPY package.json ./ 25 | RUN npm install --production 26 | COPY --chown=nonroot:nonroot --from=build /app/built /app/ 27 | -------------------------------------------------------------------------------- /Gruntfile.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | require('grunt') 3 | require('mocha') 4 | 5 | const config = { 6 | targets: { 7 | test: ['test/**/*.test.ts'], 8 | ts: ['lib/**/*.ts', 'test/**/*.ts'] 9 | }, 10 | timeout: 10000, 11 | require: ['ts-node/register', 'tsconfig-paths/register', 'should'] 12 | } 13 | config.targets.all = config.targets.test.concat(config.targets.ts) 14 | 15 | const tsConfig = { 16 | default: { 17 | options: { 18 | fast: 'always', 19 | verbose: true 20 | }, 21 | tsconfig: './tsconfig.json' 22 | } 23 | } 24 | 25 | const mochaConfig = { 26 | unit: { 27 | options: { 28 | reporter: 'spec', 29 | timeout: config.timeout, 30 | require: config.require 31 | }, 32 | src: config.targets.test 33 | } 34 | } 35 | 36 | const eslintConfig = { 37 | options: { 38 | configFile: '.eslintrc.js' 39 | }, 40 | target: config.targets.ts 41 | } 42 | 43 | const execConfig = { 44 | clean: 'rm -rf ./built && rm -rf .tscache', 45 | start: './node_modules/.bin/ts-node index.ts' 46 | } 47 | 48 | const copyConfig = { 49 | config: { 50 | expand: true, 51 | cwd: '.', 52 | src: 'config.yml', 53 | dest: 'built/' 54 | }, 55 | public: { 56 | expand: true, 57 | cwd: '.', 58 | src: 'public/**/*', 59 | dest: 'built/' 60 | } 61 | } 62 | 63 | module.exports = function (grunt) { 64 | grunt.initConfig({ 65 | eslint: eslintConfig, 66 | ts: tsConfig, 67 | mochaTest: mochaConfig, 68 | exec: execConfig, 69 | copy: copyConfig, 70 | watch: { 71 | files: config.targets.all, 72 | tasks: ['default'] 73 | }, 74 | env: { 75 | default: { 76 | LOG_LEVEL: 'none' 77 | } 78 | } 79 | }) 80 | 81 | grunt.loadNpmTasks('grunt-mocha-test') 82 | grunt.loadNpmTasks('grunt-contrib-watch') 83 | grunt.loadNpmTasks('grunt-env') 84 | grunt.loadNpmTasks('grunt-ts') 85 | grunt.loadNpmTasks('grunt-exec') 86 | grunt.loadNpmTasks('grunt-contrib-copy') 87 | grunt.loadNpmTasks('grunt-eslint') 88 | 89 | // Default task. 90 | grunt.registerTask('lint', ['eslint']) 91 | grunt.registerTask('test', ['env:default', 'mochaTest:unit']) 92 | grunt.registerTask('build', ['exec:clean', 'ts', 'copy']) 93 | grunt.registerTask('default', ['lint', 'test']) 94 | } 95 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2020 Karl Stoney 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kmoncon - Monitoring connectivity between your kubernetes nodes 2 | 3 | A Kubernetes node connectivity tool that preforms frequent tests (tcp, udp and dns), and exposes [Prometheus](https://prometheus.io) metrics that are enriched with the node name, and the locality information (such as zone), enabling you to correlate issues between availability zones or nodes. 4 | 5 | The idea is this information supplements any other L7 monitoring you use, such as [Istio](https://istio.io/latest/docs/concepts/observability) observability or [Kube State Metrics](https://github.com/kubernetes/kube-state-metrics), to help you get to the root cause of a problem faster. 6 | 7 | It's really performant, considering the number of tests it is doing, on my clusters of 75 nodes, the agents have a mere 60m CPU/40mb RAM resource request. 8 | 9 | Once you've got it up and going, you can plot some pretty dashboards like this: 10 | 11 | ![grafana](screenshots/grafana.png) 12 | 13 | PS. I've included a sample dashboard [here](grafana-dashboard.json) to get you going 14 | 15 | **Known Issues**: 16 | 17 | - It's super, mega pre-alpha, the product of a weekends experimentation - so don't expect it to be perfect. I plan to improve it but wanted to get something out there to people who wanted it. 18 | - It's written in [nodejs](https://nodejs.org/en) which means the docker image is 130mb. That's not huge, but it isn't golang small either. 19 | - If you've got nodes coming up and down frequently, eventual consistency means that you might get some test failures as an agent is testing a node that's gone (but is yet to get an updated agent list). I plan to tackle this with push agent updates. 20 | 21 | ## Architecture 22 | 23 | The application consists of two components. 24 | 25 | ### Agent 26 | 27 | This agent runs a [Daemonset](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset) agent on [Kubernetes](https://kubernetes.io/) clusters, and requires minimal permissions to run. The agents purpose is to periodically run tests against the other agents, and expose the results as metrics. 28 | 29 | The agent also spawns with an initContainer, which sets some sysctl tcp optimisations. You can disable this behaviour in the [the helm values file](helmfile/charts/kconmon/values.yaml). 30 | 31 | ### Controller 32 | 33 | In order to discover other agents, and enrich the agent information with metadata about the node and availability zone, the controller constantly watches the kubernetes API and maintains the current state in memory. The agents connect to the controller when they start, to get their own metadata, and then every 5 seconds in order to get an up to date agent list. 34 | 35 | **NOTE**: Your cluster needs RBAC enabled as the controller uses in-cluster service-account authentication with the kubernetes master. 36 | 37 | ## Testing 38 | 39 | `kconmon` does a variety of different tests, and exposes the results as prometheus metrics enriched with the node and locality information. The interval is configurable in the [helm chart config](helmfile/charts/kconmon/values.yaml), and is subject to a 50-500ms jitter to spread the load. 40 | 41 | ### UDP Testing 42 | 43 | `kmoncon` agents by default will perform 5 x 4 byte UDP packet tests between every other agent, every 5 seconds. Each test waits for a response from the destination agent. The RTT timeout is 250ms, anything longer than that and we consider the packets lost in the abyss. The metrics output from UDP tests are: 44 | 45 | - `GAUGE kconmon_udp_duration_milliseconds`: The total RTT from sending the packet to receiving a response 46 | - `GAUGE kconmon_udp_duration_variance_milliseconds`: The variance between the slowest and the fastest packet 47 | - `GAUGE kconmon_udp_loss`: The percentage of requests from the batch that failed 48 | - `COUNTER kconmon_udp_results_total`: A Counter of test results, pass and fail 49 | 50 | ### TCP Testing 51 | 52 | `kmoncon` angets will perform a since HTTP GET request between every other agent, every 5 seconds. Each connection is terminated with `Connection: close` and [Nagle's Algorithm](https://en.wikipedia.org/wiki/Nagle%27s_algorithm) as disabled to ensure consistency across tests. 53 | 54 | The metrics output from TCP tests are: 55 | 56 | - `GAUGE kconmon_tcp_connect_milliseconds`: The duration from socket assignment to successful TCP connection of the last test run 57 | - `GAUGE kconmon_tcp_duration_milliseconds`: The total RTT of the request 58 | - `COUNTER kconmon_tcp_results_total`: A Counter of test results, pass and fail 59 | 60 | ### DNS Testing 61 | 62 | `kconmon` agents will perform DNS tests by defualt every 5 seconds. It's a good idea to have tests for a variety of different resolvers (eg kube-dns, public etc). 63 | 64 | The metrics output from DNS tests are: 65 | 66 | - `GAUGE kconmon_dns_duration_milliseconds`: The duration of the last test run 67 | - `COUNTER kconmon_dns_results_total`: A Counter of test results, pass and fail 68 | 69 | ## Prometheus Metrics 70 | 71 | The agents expose a metric endpoint on `:8080/metrics`, which you'll need to configure Prometheus to scrape. Here is an example scrape config: 72 | 73 | ``` 74 | - job_name: 'kconmon' 75 | honor_labels: true 76 | kubernetes_sd_configs: 77 | - role: pod 78 | namespaces: 79 | names: 80 | - kconmon 81 | relabel_configs: 82 | - source_labels: [__meta_kubernetes_pod_label_app, __meta_kubernetes_pod_label_component] 83 | action: keep 84 | regex: "(kconmon;agent)" 85 | - source_labels: [__address__] 86 | action: replace 87 | regex: ([^:]+)(?::\d+)? 88 | replacement: $1:8080 89 | target_label: __address__ 90 | metric_relabel_configs: 91 | - regex: "(instance|pod)" 92 | action: labeldrop 93 | - source_labels: [__name__] 94 | regex: "(kconmon_.*)" 95 | action: keep 96 | ``` 97 | 98 | Your other option if you're using the prometheus operator, is to install the helm chart with `--set prometheus.enableServiceMonitor=true`. This will create you a `Service` and a `ServiceMonitor`. 99 | 100 | ### Alerting 101 | 102 | You could configure some alerts too, like this one which fires when we have consistent TCP test failures between zones for 2 minutes: 103 | 104 | ``` 105 | groups: 106 | - name: kconmon.alerting-rules 107 | rules: 108 | - alert: TCPInterZoneTestFailure 109 | expr: | 110 | sum(increase(kconmon_tcp_results_total{result="fail"}[1m])) by (source_zone, destination_zone) > 0 111 | labels: 112 | for: 2m 113 | severity: warning 114 | source: '{{ "{{" }}$labels.source_zone{{ "}}" }}' 115 | annotations: 116 | instance: '{{ "{{" }}$labels.destination_zone{{ "}}" }}' 117 | description: >- 118 | TCP Test Failures detected between one or more zones 119 | summary: Inter Zone L7 Test Failure 120 | ``` 121 | 122 | ## Deployment 123 | 124 | The easiest way to install `kconmon` is with Helm. Head over to the [releases](https://github.com/Stono/kconmon/releases) page to download the latest chart. Check out the [values.yaml](helmfile/charts/kconmon/values.yaml) for all the available configuration options. 125 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.2' 2 | services: 3 | controller: 4 | image: stono/kconmon:${VERSION:-latest} 5 | build: 6 | context: '.' 7 | args: 8 | NODEJS_VERSION: 14.6.0 9 | command: 'controller' 10 | environment: 11 | PORT: '80' # Run on port 80 locally in docker-compose to replicate kubernetes service object port magenting 12 | hostname: 'controller' 13 | domainname: 'kconmon.svc.cluster.local' 14 | networks: 15 | fake-kubernetes: 16 | aliases: 17 | - 'controller.kconmon.svc.cluster.local' 18 | - 'controller.kconmon' 19 | - 'controller' 20 | ports: 21 | - '8080:80' 22 | volumes: 23 | - ~/.config:/home/nonroot/.config:ro 24 | - ~/.kube:/home/nonroot/.kube:ro 25 | 26 | agent: 27 | image: stono/kconmon:${VERSION:-latest} 28 | command: 'agent' 29 | environment: 30 | PORT: '80' # Run on port 80 locally in docker-compose to replicate kubernetes service object port magenting 31 | hostname: 'agent' 32 | domainname: 'kconmon.svc.cluster.local' 33 | networks: 34 | fake-kubernetes: 35 | aliases: 36 | - 'agent.kconmon.svc.cluster.local' 37 | - 'agent.kconmon' 38 | - 'agent' 39 | ports: 40 | - '8080:80' 41 | 42 | networks: 43 | fake-kubernetes: 44 | -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | if [ ! -z "$CONTAINER_RESOURCE_REQUEST_MEMORY" ]; then 5 | export MAX_OLD_SPACE=$(/usr/bin/node -pe 'Math.round(process.env.CONTAINER_RESOURCE_REQUEST_MEMORY / 1024 / 1024 / 100 * 75)') 6 | ADDITIONAL_ARGS="--max_old_space_size=$MAX_OLD_SPACE" 7 | fi 8 | 9 | if [ "$1" = "agent" ]; then 10 | TARGET_APP="/app/lib/apps/agent/index.js" 11 | elif [ "$1" = "controller" ]; then 12 | TARGET_APP="/app/lib/apps/controller/index.js" 13 | else 14 | echo "Unknown command: $1" 15 | exit 1 16 | fi 17 | 18 | # Pass through to the original node script 19 | exec /usr/bin/node $ADDITIONAL_ARGS $TARGET_APP 20 | -------------------------------------------------------------------------------- /helmfile/charts/kconmon/Chart.yaml: -------------------------------------------------------------------------------- 1 | name: kconmon 2 | version: 0.0.0 3 | description: A project for testing & monitor kubernetes nodes with TCP and UDP 4 | keywords: 5 | home: 6 | sources: 7 | - https://github.com/Stono/kconmon 8 | engine: gotpl 9 | deprecated: false 10 | -------------------------------------------------------------------------------- /helmfile/charts/kconmon/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "kconmon.app.name" -}} 5 | {{ .Chart.Name }} 6 | {{- end -}} 7 | 8 | {{- define "kconmon.app.labels.standard" -}} 9 | app: {{ include "kconmon.app.name" . }} 10 | heritage: {{ .Release.Service | quote }} 11 | release: {{ .Release.Name | quote }} 12 | chart: {{ .Chart.Name }} 13 | {{- end -}} 14 | -------------------------------------------------------------------------------- /helmfile/charts/kconmon/templates/agent/daemonset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: kconmon 5 | labels: 6 | {{ include "kconmon.app.labels.standard" . | indent 4 }} 7 | spec: 8 | selector: 9 | matchLabels: 10 | app: kconmon 11 | component: agent 12 | updateStrategy: 13 | type: RollingUpdate 14 | rollingUpdate: 15 | maxUnavailable: 100% 16 | template: 17 | metadata: 18 | annotations: 19 | sidecar.istio.io/inject: "false" 20 | labels: 21 | app: kconmon 22 | component: agent 23 | spec: 24 | terminationGracePeriodSeconds: 15 25 | dnsConfig: 26 | options: 27 | - name: ndots 28 | value: "1" 29 | searches: 30 | - {{ .Release.Namespace }}.svc.cluster.local 31 | - svc.cluster.local 32 | - cluster.local 33 | {{ if $.Values.config.nodeAntiAffinity }} 34 | affinity: 35 | nodeAffinity: 36 | requiredDuringSchedulingIgnoredDuringExecution: 37 | nodeSelectorTerms: 38 | {{- range $object := ($.Values.config.nodeAntiAffinity) }} 39 | - matchExpressions: 40 | - key: {{ $object.key }} 41 | operator: NotIn 42 | values: 43 | - "{{ $object.value }}" 44 | {{ end }} 45 | {{ end }} 46 | 47 | {{ if $.Values.enableTcpTweaks }} 48 | initContainers: 49 | - name: sysctls 50 | command: 51 | - /bin/sh 52 | - -c 53 | - | 54 | # TCP Connection Tweaks 55 | sysctl -w net.core.somaxconn=32768 56 | sysctl -w net.ipv4.tcp_fin_timeout=60 57 | sysctl -w net.ipv4.tcp_keepalive_time=30 58 | sysctl -w net.ipv4.tcp_keepalive_intvl=30 59 | sysctl -w net.ipv4.tcp_keepalive_probes=3 60 | sysctl -w net.ipv4.tcp_window_scaling=1 61 | sysctl -w net.ipv4.tcp_max_syn_backlog=3240000 62 | sysctl -w net.ipv4.tcp_timestamps=0 63 | sysctl -w net.ipv4.tcp_sack=0 64 | imagePullPolicy: IfNotPresent 65 | image: "{{ required "Please specify the docker image" .Values.docker.image }}:{{ .Values.docker.tag | default .Chart.Version }}" 66 | securityContext: 67 | privileged: true 68 | runAsUser: 0 69 | {{ end }} 70 | 71 | containers: 72 | - name: agent 73 | image: "{{ required "Please specify the docker image" .Values.docker.image }}:{{ .Values.docker.tag | default .Chart.Version }}" 74 | imagePullPolicy: IfNotPresent 75 | args: 76 | - agent 77 | env: 78 | - name: CONTAINER_RESOURCE_REQUEST_MEMORY 79 | valueFrom: 80 | resourceFieldRef: 81 | divisor: "0" 82 | resource: requests.memory 83 | - name: CONTAINER_RESOURCE_LIMIT_MEMORY 84 | valueFrom: 85 | resourceFieldRef: 86 | divisor: "0" 87 | resource: limits.memory 88 | - name: DEPLOYMENT_NAMESPACE 89 | valueFrom: 90 | fieldRef: 91 | fieldPath: metadata.namespace 92 | - name: CONFIG 93 | value: {{ $.Values.config | toJson | quote }} 94 | ports: 95 | - name: http-web 96 | containerPort: 8080 97 | protocol: TCP 98 | readinessProbe: 99 | initialDelaySeconds: 2 100 | periodSeconds: 10 101 | successThreshold: 1 102 | timeoutSeconds: 1 103 | failureThreshold: 1 104 | httpGet: 105 | path: /readiness 106 | port: 8080 107 | resources: 108 | requests: 109 | cpu: {{ required "Please set the agent cpu" $.Values.resources.agent.cpu }} 110 | memory: {{ $.Values.resources.agent.memory | default "45Mi" }} 111 | limits: 112 | memory: 128Mi 113 | tolerations: 114 | - effect: NoSchedule 115 | operator: Exists 116 | -------------------------------------------------------------------------------- /helmfile/charts/kconmon/templates/agent/service.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.prometheus.enableServiceMonitor }} 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | labels: 6 | {{ include "kconmon.app.labels.standard" . | indent 4 }} 7 | component: agent 8 | name: agent 9 | spec: 10 | ports: 11 | - name: http-web 12 | port: 8080 13 | protocol: TCP 14 | targetPort: http-web 15 | selector: 16 | app: kconmon 17 | component: agent 18 | sessionAffinity: None 19 | type: ClusterIP 20 | {{ end }} 21 | -------------------------------------------------------------------------------- /helmfile/charts/kconmon/templates/agent/servicemonitor.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.prometheus.enableServiceMonitor }} 2 | apiVersion: monitoring.coreos.com/v1 3 | kind: ServiceMonitor 4 | metadata: 5 | labels: 6 | {{ include "kconmon.app.labels.standard" . | indent 4 }} 7 | name: kconmon-service-monitor 8 | namespace: kconmon 9 | spec: 10 | endpoints: 11 | - interval: 5s 12 | path: /metrics 13 | port: http-web 14 | namespaceSelector: 15 | any: true 16 | selector: 17 | matchLabels: 18 | app: kconmon 19 | component: agent 20 | {{ end }} 21 | -------------------------------------------------------------------------------- /helmfile/charts/kconmon/templates/controller/clusterrole.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | labels: 5 | {{ include "kconmon.app.labels.standard" . | indent 4 }} 6 | name: kconmon-clusterrole 7 | rules: 8 | - apiGroups: 9 | - "" 10 | resources: 11 | - nodes 12 | verbs: 13 | - get 14 | - list 15 | - watch 16 | -------------------------------------------------------------------------------- /helmfile/charts/kconmon/templates/controller/clusterrolebinding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | labels: 5 | {{ include "kconmon.app.labels.standard" . | indent 4 }} 6 | name: kconmon-binding 7 | roleRef: 8 | apiGroup: rbac.authorization.k8s.io 9 | kind: ClusterRole 10 | name: kconmon-clusterrole 11 | subjects: 12 | - kind: ServiceAccount 13 | name: kconmon 14 | namespace: {{ .Release.Namespace }} 15 | -------------------------------------------------------------------------------- /helmfile/charts/kconmon/templates/controller/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | {{ include "kconmon.app.labels.standard" . | indent 4 }} 6 | name: controller 7 | spec: 8 | minReadySeconds: 15 9 | replicas: 2 10 | revisionHistoryLimit: 1 11 | selector: 12 | matchLabels: 13 | app: kconmon 14 | component: controller 15 | strategy: 16 | rollingUpdate: 17 | maxSurge: 100% 18 | maxUnavailable: 1 19 | type: RollingUpdate 20 | template: 21 | metadata: 22 | annotations: 23 | sidecar.istio.io/inject: "false" 24 | labels: 25 | app: kconmon 26 | component: controller 27 | spec: 28 | terminationGracePeriodSeconds: 15 29 | dnsConfig: 30 | options: 31 | - name: ndots 32 | value: "1" 33 | searches: 34 | - {{ .Release.Namespace }}.svc.cluster.local 35 | - svc.cluster.local 36 | - cluster.local 37 | serviceAccountName: kconmon 38 | containers: 39 | - name: agent 40 | image: "{{ required "Please specify the docker image" .Values.docker.image }}:{{ .Values.docker.tag | default .Chart.Version }}" 41 | imagePullPolicy: IfNotPresent 42 | args: 43 | - controller 44 | env: 45 | - name: CONTAINER_RESOURCE_REQUEST_MEMORY 46 | valueFrom: 47 | resourceFieldRef: 48 | divisor: "0" 49 | resource: requests.memory 50 | - name: CONTAINER_RESOURCE_LIMIT_MEMORY 51 | valueFrom: 52 | resourceFieldRef: 53 | divisor: "0" 54 | resource: limits.memory 55 | - name: DEPLOYMENT_NAMESPACE 56 | valueFrom: 57 | fieldRef: 58 | fieldPath: metadata.namespace 59 | - name: CONFIG 60 | value: {{ $.Values.config | toJson | quote }} 61 | ports: 62 | - name: http-web 63 | containerPort: 8080 64 | protocol: TCP 65 | readinessProbe: 66 | initialDelaySeconds: 2 67 | periodSeconds: 10 68 | successThreshold: 1 69 | timeoutSeconds: 1 70 | failureThreshold: 1 71 | httpGet: 72 | path: /readiness 73 | port: 8080 74 | resources: 75 | requests: 76 | cpu: 50m 77 | memory: 80Mi 78 | limits: 79 | memory: 128Mi 80 | -------------------------------------------------------------------------------- /helmfile/charts/kconmon/templates/controller/pdb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: policy/v1beta1 2 | kind: PodDisruptionBudget 3 | metadata: 4 | labels: 5 | {{ include "kconmon.app.labels.standard" . | indent 4 }} 6 | name: controller-pdb-percentage 7 | spec: 8 | maxUnavailable: 50% 9 | selector: 10 | matchLabels: 11 | app: kconmon 12 | component: controller 13 | -------------------------------------------------------------------------------- /helmfile/charts/kconmon/templates/controller/role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: Role 3 | metadata: 4 | labels: 5 | {{ include "kconmon.app.labels.standard" . | indent 4 }} 6 | name: kconmon 7 | rules: 8 | - apiGroups: 9 | - "" 10 | resources: 11 | - pods 12 | verbs: 13 | - get 14 | - list 15 | - watch 16 | -------------------------------------------------------------------------------- /helmfile/charts/kconmon/templates/controller/rolebinding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | labels: 5 | {{ include "kconmon.app.labels.standard" . | indent 4 }} 6 | name: kconmon 7 | roleRef: 8 | apiGroup: rbac.authorization.k8s.io 9 | kind: Role 10 | name: kconmon 11 | subjects: 12 | - kind: ServiceAccount 13 | name: kconmon 14 | namespace: {{ .Release.Namespace }} 15 | -------------------------------------------------------------------------------- /helmfile/charts/kconmon/templates/controller/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | {{ include "kconmon.app.labels.standard" . | indent 4 }} 6 | name: controller 7 | spec: 8 | ports: 9 | - name: http-web 10 | port: 80 11 | protocol: TCP 12 | targetPort: http-web 13 | selector: 14 | app: kconmon 15 | component: controller 16 | sessionAffinity: None 17 | type: ClusterIP 18 | -------------------------------------------------------------------------------- /helmfile/charts/kconmon/templates/controller/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: kconmon 5 | labels: 6 | {{ include "kconmon.app.labels.standard" . | indent 4 }} 7 | -------------------------------------------------------------------------------- /helmfile/charts/kconmon/values.yaml: -------------------------------------------------------------------------------- 1 | docker: 2 | image: stono/kconmon 3 | 4 | # Should we run an initContainer that enables core tcp connection setting tweaks 5 | # Note: This requires kernel 4+ 6 | enableTcpTweaks: true 7 | 8 | config: 9 | # What port should the server listen on 10 | port: 8080 11 | 12 | # What should the prometheus metrics be prefixed with 13 | metricsPrefix: kconmon 14 | 15 | # These are nodes where you do not wish to run the agent 16 | nodeAntiAffinity: 17 | - key: airflow 18 | value: true 19 | 20 | # This is the well known label on a node which identifies the zone 21 | failureDomainLabel: failure-domain.beta.kubernetes.io/zone 22 | 23 | # TCP test configuration 24 | tcp: 25 | interval: 5000 26 | timeout: 1000 27 | 28 | # UDP test configuration 29 | udp: 30 | interval: 5000 31 | # This is a per-packet timeout 32 | timeout: 250 33 | packets: 5 34 | 35 | dns: 36 | interval: 5000 37 | hosts: 38 | - www.google.com 39 | - kubernetes.default.svc.cluster.local 40 | 41 | resources: 42 | agent: 43 | # This will scale with the number of nodes in your cluster at loosely 1m per node 44 | # The minimum value is 25m 45 | cpu: 70m 46 | memory: 45Mi 47 | 48 | prometheus: 49 | # If you wish to create a service and a service monitor because you're using the prometheus operator, change this to true 50 | enableServiceMonitor: false 51 | -------------------------------------------------------------------------------- /helmfile/helmfile.yaml: -------------------------------------------------------------------------------- 1 | releases: 2 | - name: kconmon 3 | namespace: kconmon 4 | chart: ./charts/kconmon 5 | set: 6 | - name: docker.tag 7 | value: latest 8 | -------------------------------------------------------------------------------- /images/pretty.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stono/kconmon/4fcf38ecf67ad22425981a35375af5665e260021/images/pretty.png -------------------------------------------------------------------------------- /lib/apps/agent/controllers/index.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response } from 'express' 2 | import { IMetrics } from 'lib/apps/agent/metrics' 3 | import { ITester } from 'lib/tester' 4 | import { IDiscovery } from 'lib/discovery' 5 | 6 | export interface IIndexController { 7 | readiness(req: Request, res: Response): void 8 | metrics(req: Request, res: Response): void 9 | udp(req: Request, res: Response): void 10 | } 11 | 12 | export default class IndexController implements IIndexController { 13 | private metricsManager: IMetrics 14 | private tester: ITester 15 | private discovery: IDiscovery 16 | constructor( 17 | metricsManager: IMetrics, 18 | tester: ITester, 19 | discovery: IDiscovery 20 | ) { 21 | this.metricsManager = metricsManager 22 | this.tester = tester 23 | this.discovery = discovery 24 | } 25 | public readiness(req: Request, res: Response): void { 26 | res.set('Connection', 'close') 27 | res.status(200) 28 | res.end() 29 | res.connection.destroy() 30 | } 31 | 32 | public metrics(req: Request, res: Response): void { 33 | res.setHeader('content-type', 'text/plain') 34 | res.send(this.metricsManager.toString()) 35 | } 36 | 37 | public async udp(req: Request, res: Response): Promise { 38 | let agents = await this.discovery.agents() 39 | if (req.params.agent) { 40 | agents = agents.filter((agent) => agent.name === req.params.agent) 41 | } 42 | const results = await this.tester.runUDPTests(agents) 43 | res.json(results) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /lib/apps/agent/index.ts: -------------------------------------------------------------------------------- 1 | import * as path from 'path' 2 | import * as tsConfigPaths from 'tsconfig-paths/lib' 3 | import * as process from 'process' 4 | import got from 'got' 5 | import { Application } from 'express' 6 | const baseUrl = path.join(__dirname, '../../..') 7 | 8 | tsConfigPaths.register({ 9 | baseUrl, 10 | paths: {} 11 | }) 12 | 13 | import WebServer from 'lib/web-server' 14 | import Tester from 'lib/tester' 15 | import ServiceDiscovery from 'lib/discovery/service' 16 | import IndexController from 'lib/apps/agent/controllers' 17 | import IndexRoutes from 'lib/apps/agent/routes' 18 | import Metrics from './metrics' 19 | import * as os from 'os' 20 | import UDPServer from 'lib/udp/server' 21 | import config from 'lib/config' 22 | import Logger from 'lib/logger' 23 | import UDPClientFactory from 'lib/udp/clientFactory' 24 | const webServer = new WebServer(config) 25 | const discovery = new ServiceDiscovery(config, got) 26 | const metrics = new Metrics(config) 27 | const logger = new Logger('agent') 28 | const udpClientFactory = new UDPClientFactory(config) 29 | const udpServer = new UDPServer(config) 30 | const delay = (ms: number) => { 31 | return new Promise((resolve) => setTimeout(resolve, ms)) 32 | } 33 | 34 | ;(async () => { 35 | await discovery.start() 36 | await udpServer.start() 37 | await delay(5000) 38 | 39 | const me = await discovery.agent(os.hostname()) 40 | if (!me) { 41 | logger.error('failed to load agent metadata information!') 42 | process.exit(1) 43 | } 44 | logger.info(`loaded metadata`, me) 45 | const tester = new Tester( 46 | config, 47 | got, 48 | discovery, 49 | metrics, 50 | me, 51 | udpClientFactory 52 | ) 53 | const handlerInit = (app: Application): Promise => { 54 | const indexController = new IndexController(metrics, tester, discovery) 55 | new IndexRoutes().applyRoutes(app, indexController) 56 | return Promise.resolve() 57 | } 58 | await webServer.start(handlerInit) 59 | await tester.start() 60 | logger.info('agent started successfully') 61 | 62 | async function shutdown() { 63 | const shutdownPeriod = 7500 64 | logger.info(`stopping agent, will exit in ${shutdownPeriod}ms`) 65 | await tester.stop() 66 | await discovery.stop() 67 | setTimeout(async () => { 68 | await udpServer.stop() 69 | await webServer.stop() 70 | setTimeout(() => { 71 | process.exit(0) 72 | }, 1000) 73 | }, shutdownPeriod) 74 | } 75 | 76 | process.on('SIGINT', shutdown) 77 | process.on('SIGTERM', shutdown) 78 | process.on('unhandledRejection', (error) => { 79 | console.error('Unhandled Rejection!') 80 | console.error(error) 81 | process.exit(1) 82 | }) 83 | })() 84 | -------------------------------------------------------------------------------- /lib/apps/agent/metrics.ts: -------------------------------------------------------------------------------- 1 | export interface IMetrics { 2 | handleTCPTestResult(result: ITCPTestResult) 3 | handleUDPTestResult(result: IUDPTestResult) 4 | handleDNSTestResult(result: IDNSTestResult) 5 | resetTCPTestResults() 6 | resetUDPTestResults() 7 | toString() 8 | } 9 | 10 | import * as client from 'prom-client' 11 | import { IUDPTestResult, IDNSTestResult, ITCPTestResult } from 'lib/tester' 12 | import { IConfig } from 'lib/config' 13 | 14 | export default class Metrics implements IMetrics { 15 | private TCP: client.Counter 16 | private TCPDuration: client.Gauge 17 | private TCPConnect: client.Gauge 18 | 19 | private UDP: client.Counter 20 | private UDPDuration: client.Gauge 21 | private UDPVariance: client.Gauge 22 | private UDPLoss: client.Gauge 23 | 24 | private DNS: client.Counter 25 | private DNSDuration: client.Gauge 26 | 27 | constructor(config: IConfig) { 28 | client.register.clear() 29 | this.TCPConnect = new client.Gauge({ 30 | help: 'Time taken to establish the TCP socket', 31 | labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], 32 | name: `${config.metricsPrefix}_tcp_connect_milliseconds` 33 | }) 34 | 35 | this.TCPDuration = new client.Gauge({ 36 | help: 'Total time taken to complete the TCP test', 37 | labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], 38 | name: `${config.metricsPrefix}_tcp_duration_milliseconds` 39 | }) 40 | 41 | this.UDPDuration = new client.Gauge({ 42 | help: 'Average duration per packet', 43 | labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], 44 | name: `${config.metricsPrefix}_udp_duration_milliseconds` 45 | }) 46 | 47 | this.UDPVariance = new client.Gauge({ 48 | help: 'UDP variance between the slowest and fastest packet', 49 | labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], 50 | name: `${config.metricsPrefix}_udp_duration_variance_milliseconds` 51 | }) 52 | 53 | this.UDPLoss = new client.Gauge({ 54 | help: 'UDP packet loss', 55 | labelNames: ['source', 'destination', 'source_zone', 'destination_zone'], 56 | name: `${config.metricsPrefix}_udp_loss` 57 | }) 58 | 59 | this.DNS = new client.Counter({ 60 | help: 'DNS Test Results', 61 | labelNames: ['source', 'source_zone', 'host', 'result'], 62 | name: `${config.metricsPrefix}_dns_results_total` 63 | }) 64 | 65 | this.DNSDuration = new client.Gauge({ 66 | help: 'Total time taken to complete the DNS test', 67 | labelNames: ['source', 'source_zone', 'host'], 68 | name: `${config.metricsPrefix}_dns_duration_milliseconds` 69 | }) 70 | 71 | this.UDP = new client.Counter({ 72 | help: 'UDP Test Results', 73 | labelNames: [ 74 | 'source', 75 | 'destination', 76 | 'source_zone', 77 | 'destination_zone', 78 | 'result' 79 | ], 80 | name: `${config.metricsPrefix}_udp_results_total` 81 | }) 82 | 83 | this.TCP = new client.Counter({ 84 | help: 'TCP Test Results', 85 | labelNames: [ 86 | 'source', 87 | 'destination', 88 | 'source_zone', 89 | 'destination_zone', 90 | 'result' 91 | ], 92 | name: `${config.metricsPrefix}_tcp_results_total` 93 | }) 94 | } 95 | 96 | public handleDNSTestResult(result: IDNSTestResult): void { 97 | const source = result.source.nodeName 98 | this.DNS.labels(source, result.source.zone, result.host, result.result).inc( 99 | 1 100 | ) 101 | this.DNSDuration.labels( 102 | result.source.nodeName, 103 | result.source.zone, 104 | result.host 105 | ).set(result.duration) 106 | } 107 | 108 | public resetTCPTestResults() { 109 | this.TCPConnect.reset() 110 | this.TCPDuration.reset() 111 | } 112 | 113 | public resetUDPTestResults() { 114 | this.UDPDuration.reset() 115 | this.UDPLoss.reset() 116 | this.UDPVariance.reset() 117 | } 118 | 119 | public handleUDPTestResult(result: IUDPTestResult): void { 120 | const source = result.source.nodeName 121 | const destination = result.destination.nodeName 122 | const sourceZone = result.source.zone 123 | const destinationZone = result.destination.zone 124 | this.UDP.labels( 125 | source, 126 | destination, 127 | sourceZone, 128 | destinationZone, 129 | result.result 130 | ).inc(1) 131 | 132 | if (result.timings) { 133 | this.UDPDuration.labels( 134 | source, 135 | destination, 136 | sourceZone, 137 | destinationZone 138 | ).set(result.timings.average) 139 | 140 | this.UDPVariance.labels( 141 | source, 142 | destination, 143 | sourceZone, 144 | destinationZone 145 | ).set(result.timings.variance) 146 | 147 | this.UDPLoss.labels(source, destination, sourceZone, destinationZone).set( 148 | result.timings.loss 149 | ) 150 | } 151 | } 152 | 153 | public handleTCPTestResult(result: ITCPTestResult): void { 154 | const source = result.source.nodeName 155 | const destination = result.destination.nodeName 156 | const sourceZone = result.source.zone 157 | const destinationZone = result.destination.zone 158 | this.TCP.labels( 159 | source, 160 | destination, 161 | sourceZone, 162 | destinationZone, 163 | result.result 164 | ).inc(1) 165 | 166 | if (result.timings) { 167 | this.TCPConnect.labels( 168 | source, 169 | destination, 170 | sourceZone, 171 | destinationZone 172 | ).set( 173 | ((result.timings.connect || 174 | result.timings.socket || 175 | result.timings.start) - result.timings.start) as number 176 | ) 177 | this.TCPDuration.labels( 178 | source, 179 | destination, 180 | sourceZone, 181 | destinationZone 182 | ).set(result.timings.phases.total as number) 183 | } 184 | } 185 | 186 | public toString(): string { 187 | return client.register.metrics() 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /lib/apps/agent/routes/index.ts: -------------------------------------------------------------------------------- 1 | import IndexController from 'lib/apps/agent/controllers' 2 | import { Application } from 'express' 3 | import { IRoutes } from 'lib/web-server' 4 | 5 | export default class IndexRoutes implements IRoutes { 6 | public applyRoutes(app: Application, controller: IndexController): void { 7 | app.get('/readiness', (req, res) => { 8 | controller.readiness.bind(controller)(req, res) 9 | }) 10 | app.get('/metrics', (req, res) => { 11 | controller.metrics.bind(controller)(req, res) 12 | }) 13 | app.get('/test/udp/:agent', (req, res) => { 14 | controller.udp.bind(controller)(req, res) 15 | }) 16 | app.get('/test/udp', (req, res) => { 17 | controller.udp.bind(controller)(req, res) 18 | }) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /lib/apps/controller/controllers/index.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response } from 'express' 2 | import { IDiscovery } from 'lib/discovery' 3 | 4 | export interface IIndexController { 5 | readiness(req: Request, res: Response): void 6 | } 7 | 8 | export default class IndexController implements IIndexController { 9 | private discovery: IDiscovery 10 | constructor(discovery: IDiscovery) { 11 | this.discovery = discovery 12 | } 13 | 14 | public readiness(req: Request, res: Response): void { 15 | res.status(200).end() 16 | } 17 | 18 | public async agents(req: Request, res: Response): Promise { 19 | const agents = await this.discovery.agents() 20 | res.json(agents) 21 | } 22 | 23 | public async agent(req: Request, res: Response): Promise { 24 | const name = req.params.name 25 | if (!name) { 26 | res.sendStatus(400) 27 | return 28 | } 29 | const agent = await this.discovery.agent(name) 30 | if (!agent) { 31 | res.sendStatus(500) 32 | return 33 | } 34 | res.json(agent) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /lib/apps/controller/index.ts: -------------------------------------------------------------------------------- 1 | import * as path from 'path' 2 | import * as tsConfigPaths from 'tsconfig-paths/lib' 3 | import * as process from 'process' 4 | import { Application } from 'express' 5 | const baseUrl = path.join(__dirname, '../../..') 6 | 7 | tsConfigPaths.register({ 8 | baseUrl, 9 | paths: {} 10 | }) 11 | 12 | import WebServer from 'lib/web-server' 13 | import KubernetesDiscovery from 'lib/discovery/kubernetes' 14 | import IndexController from 'lib/apps/controller/controllers' 15 | import IndexRoutes from 'lib/apps/controller/routes' 16 | import config from 'lib/config' 17 | import Kubernetes from 'lib/kubernetes/client' 18 | import Logger from 'lib/logger' 19 | const kubernetes = new Kubernetes() 20 | const webServer = new WebServer(config) 21 | const discovery = new KubernetesDiscovery(config, kubernetes) 22 | 23 | const handlerInit = (app: Application): Promise => { 24 | const indexController = new IndexController(discovery) 25 | new IndexRoutes().applyRoutes(app, indexController) 26 | return Promise.resolve() 27 | } 28 | const logger = new Logger('controller') 29 | ;(async () => { 30 | await discovery.start() 31 | await webServer.start(handlerInit) 32 | logger.log('controller started successfully') 33 | })() 34 | 35 | async function shutdown() { 36 | const shutdownPeriod = 7500 37 | logger.info(`stopping controller, will exit in ${shutdownPeriod}ms`) 38 | setTimeout(async () => { 39 | await webServer.stop() 40 | await discovery.stop() 41 | setTimeout(() => { 42 | logger.info('controller stopped') 43 | process.exit(0) 44 | }, 1000) 45 | }, shutdownPeriod) 46 | } 47 | 48 | process.on('SIGINT', shutdown) 49 | process.on('SIGTERM', shutdown) 50 | process.on('unhandledRejection', (error) => { 51 | console.error('Unhandled Rejection!') 52 | console.error(error) 53 | process.exit(1) 54 | }) 55 | -------------------------------------------------------------------------------- /lib/apps/controller/routes/index.ts: -------------------------------------------------------------------------------- 1 | import IndexController from 'lib/apps/controller/controllers' 2 | import { Application } from 'express' 3 | import { IRoutes } from 'lib/web-server' 4 | 5 | export default class IndexRoutes implements IRoutes { 6 | public applyRoutes(app: Application, controller: IndexController): void { 7 | app.get('/readiness', (req, res) => { 8 | controller.readiness.bind(controller)(req, res) 9 | }) 10 | app.get('/agents', (req, res) => { 11 | controller.agents.bind(controller)(req, res) 12 | }) 13 | app.get('/agent/:name', (req, res) => { 14 | controller.agent.bind(controller)(req, res) 15 | }) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /lib/config/index.ts: -------------------------------------------------------------------------------- 1 | import Logger from 'lib/logger' 2 | 3 | const logger = new Logger('config') 4 | const env = process.env.CONFIG 5 | 6 | let json = {} 7 | if (env) { 8 | json = JSON.parse(env) 9 | } else { 10 | logger.warn('No environment configuration found, will be using defaults') 11 | } 12 | 13 | const getEnv = (key: string, defaultValue: any) => { 14 | return json[key] || defaultValue 15 | } 16 | 17 | interface ITestConfiguration { 18 | tcp: { 19 | interval: number 20 | timeout: number 21 | } 22 | udp: { 23 | interval: number 24 | timeout: number 25 | packets: number 26 | } 27 | dns: { 28 | interval: number 29 | hosts: string[] 30 | } 31 | } 32 | 33 | export interface IConfig { 34 | port: number 35 | metricsPrefix: string 36 | namespace: string 37 | environment: string 38 | failureDomainLabel: string 39 | nodeAntiAffinity: { 40 | [key: string]: string 41 | value: any 42 | }[] 43 | testConfig: ITestConfiguration 44 | } 45 | 46 | export class Config implements IConfig { 47 | public readonly port: number = getEnv('port', 8080) 48 | public readonly namespace: string = 49 | process.env.DEPLOYMENT_NAMESPACE || 'kconmon' 50 | public readonly metricsPrefix: string = getEnv('metricsPrefix', 'kconmon') 51 | public readonly environment: string = getEnv('environment', 'testing') 52 | public readonly failureDomainLabel: string = getEnv( 53 | 'failureDomainLabel', 54 | 'failure-domain.beta.kubernetes.io/zone' 55 | ) 56 | public readonly nodeAntiAffinity: { 57 | [key: string]: string 58 | value: any 59 | }[] = getEnv('nodeAntiAffinity', []) 60 | public readonly testConfig: ITestConfiguration = { 61 | tcp: getEnv('tcp', { interval: 5000, timeout: 1000 }), 62 | udp: getEnv('udp', { interval: 5000, timeout: 250, packets: 10 }), 63 | dns: getEnv('dns', { interval: 5000, hosts: [] }) 64 | } 65 | } 66 | 67 | const config = new Config() 68 | logger.info( 69 | `Configuration: port=${config.port}, namespace=${config.namespace}, failureDomainLabel=${config.failureDomainLabel}` 70 | ) 71 | export default config 72 | -------------------------------------------------------------------------------- /lib/discovery/index.ts: -------------------------------------------------------------------------------- 1 | export interface IDiscovery { 2 | start() 3 | stop() 4 | agents(): Promise 5 | agent(name: string): Promise 6 | } 7 | 8 | export interface IAgent { 9 | name: string 10 | nodeName: string 11 | ip: string 12 | zone: string 13 | } 14 | -------------------------------------------------------------------------------- /lib/discovery/kubernetes.ts: -------------------------------------------------------------------------------- 1 | import { IDiscovery, IAgent } from 'lib/discovery' 2 | import { IConfig } from 'lib/config' 3 | import { Models } from 'lib/kubernetes/models' 4 | import IKubernetesClient, { KubernetesEventType } from 'lib/kubernetes/client' 5 | import Logger from 'lib/logger' 6 | 7 | export type PodAgentMapper = (pod: Models.Core.IPod) => IAgent 8 | 9 | export interface IKubernetesDiscovery extends IDiscovery { 10 | reconcileNodes() 11 | } 12 | 13 | export default class KubernetesDiscovery implements IKubernetesDiscovery { 14 | private client: IKubernetesClient 15 | private logger = new Logger('discovery') 16 | private podCache: { [key: string]: IAgent } = {} 17 | private nodeCache: { 18 | [key: string]: { [key: string]: string } 19 | } = {} 20 | private config: IConfig 21 | constructor(config: IConfig, client: IKubernetesClient) { 22 | this.client = client 23 | this.config = config 24 | } 25 | 26 | public async agents(): Promise { 27 | return Object.keys(this.podCache).map((key) => { 28 | return this.podCache[key] 29 | }) 30 | } 31 | 32 | public async agent(name: string): Promise { 33 | const pod = await this.client.get( 34 | 'v1', 35 | 'Pod', 36 | this.config.namespace, 37 | name 38 | ) 39 | if (!pod) { 40 | return null 41 | } 42 | return this.mapToAgent(pod) 43 | } 44 | 45 | public async start(): Promise { 46 | const handleNodeEvent = async ( 47 | node: Models.Core.INode, 48 | eventType: KubernetesEventType 49 | ) => { 50 | if (eventType === KubernetesEventType.DELETED) { 51 | this.logger.info(`node ${node.metadata.name} was removed`) 52 | delete this.nodeCache[node.metadata.name as string] 53 | } 54 | if (eventType === KubernetesEventType.ADDED) { 55 | this.logger.info(`node ${node.metadata.name} was added`) 56 | this.nodeCache[node.metadata.name as string] = 57 | node.metadata.labels || {} 58 | } 59 | if (eventType === KubernetesEventType.MODIFIED) { 60 | this.nodeCache[node.metadata.name as string] = 61 | node.metadata.labels || {} 62 | } 63 | } 64 | await this.client.watch('v1', 'Node', handleNodeEvent) 65 | await this.reconcileNodes() 66 | 67 | const handlePodEvent = async ( 68 | pod: Models.Core.IPod, 69 | eventType: KubernetesEventType 70 | ) => { 71 | if (!pod.metadata.labels || pod.metadata.labels.component !== 'agent') { 72 | return 73 | } 74 | 75 | if (eventType === KubernetesEventType.DELETED) { 76 | const agent = this.podCache[pod.metadata.name as string] 77 | if (agent) { 78 | this.logger.info(`agent removed`, agent) 79 | delete this.podCache[pod.metadata.name as string] 80 | } 81 | } else { 82 | this.handleAgent(pod) 83 | } 84 | } 85 | await this.client.watch( 86 | 'v1', 87 | 'Pod', 88 | handlePodEvent, 89 | [KubernetesEventType.ALL], 90 | this.config.namespace 91 | ) 92 | await this.client.start() 93 | await this.reconcileAgents() 94 | } 95 | 96 | public async stop(): Promise { 97 | return this.client.stop() 98 | } 99 | 100 | private handleAgent(pod: Models.Core.IPod): void { 101 | if ( 102 | pod.status && 103 | pod.status.phase === 'Running' && 104 | pod.status.containerStatuses && 105 | pod.status.containerStatuses[0].ready 106 | ) { 107 | const result = this.mapToAgent(pod) 108 | if (!result) { 109 | return 110 | } 111 | if (!this.podCache[pod.metadata.name as string]) { 112 | this.podCache[pod.metadata.name as string] = result 113 | this.logger.info(`agent added`, result) 114 | } 115 | } else { 116 | delete this.podCache[pod.metadata.name as string] 117 | } 118 | } 119 | 120 | public async reconcileNodes(): Promise { 121 | this.logger.info('reconciling nodes from kubernetes') 122 | const nodes = await this.client.select('v1', 'Node') 123 | nodes.forEach((node) => { 124 | this.nodeCache[node.metadata.name] = node.metadata.labels || {} 125 | }) 126 | this.logger.info(`${nodes.length} nodes loaded`) 127 | } 128 | 129 | private async reconcileAgents(): Promise { 130 | this.logger.info('reconciling pods from kubernetes') 131 | const pods = await this.client.select( 132 | 'v1', 133 | 'Pod', 134 | this.config.namespace, 135 | `app=kconmon,component=agent` 136 | ) 137 | 138 | pods.forEach((pod) => { 139 | this.handleAgent(pod) 140 | }) 141 | 142 | Object.keys(this.podCache).forEach((key) => { 143 | if (!pods.find((pod) => pod.metadata.name === key)) { 144 | delete this.podCache[key] 145 | } 146 | }) 147 | 148 | this.logger.info(`${Object.keys(this.podCache).length} agents discovered`) 149 | } 150 | 151 | private mapToAgent(pod: Models.Core.IPod): IAgent | null { 152 | if (!pod.spec.nodeName) { 153 | return null 154 | } 155 | const node = this.nodeCache[pod.spec.nodeName] 156 | if (!node) { 157 | return null 158 | } 159 | let zone = 'unknown' 160 | const failureDomain = node[this.config.failureDomainLabel] 161 | if (failureDomain) { 162 | zone = failureDomain 163 | } else { 164 | this.logger.warn( 165 | `Unable to find failure domain label (${failureDomain}) on node` 166 | ) 167 | } 168 | 169 | return { 170 | name: pod.metadata.name as string, 171 | nodeName: pod.spec.nodeName as string, 172 | ip: pod.status?.podIP as string, 173 | zone 174 | } 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /lib/discovery/service.ts: -------------------------------------------------------------------------------- 1 | import { IDiscovery, IAgent } from 'lib/discovery' 2 | import { Got } from 'got/dist/source' 3 | import { IConfig } from 'lib/config' 4 | import Logger from 'lib/logger' 5 | 6 | export default class ServiceDiscovery implements IDiscovery { 7 | private got: Got 8 | private logger = new Logger('discovery') 9 | private lastResult: IAgent[] = [] 10 | private config: IConfig 11 | constructor(config: IConfig, got: Got) { 12 | this.got = got 13 | this.config = config 14 | } 15 | 16 | public async start(): Promise {} 17 | public async stop(): Promise {} 18 | public async agents(): Promise { 19 | try { 20 | const result = await this.got( 21 | `http://controller.${this.config.namespace}.svc.cluster.local./agents`, 22 | { 23 | timeout: 500, 24 | responseType: 'json', 25 | retry: { 26 | limit: 2 27 | } 28 | } 29 | ) 30 | this.lastResult = result.body 31 | } catch (ex) { 32 | this.logger.error( 33 | 'failed to retrieve current agent list from controller', 34 | ex 35 | ) 36 | } 37 | return this.lastResult 38 | } 39 | 40 | public async agent(name: string): Promise { 41 | try { 42 | const result = await this.got( 43 | `http://controller.${this.config.namespace}.svc.cluster.local./agent/${name}`, 44 | { 45 | timeout: 500, 46 | responseType: 'json', 47 | retry: { 48 | limit: 2 49 | } 50 | } 51 | ) 52 | return result.body 53 | } catch (ex) { 54 | this.logger.error( 55 | 'failed to retrieve real time agent information from controller', 56 | ex 57 | ) 58 | return null 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /lib/kubernetes/client.ts: -------------------------------------------------------------------------------- 1 | const Client = require('kubernetes-client').Client 2 | const { KubeConfig } = require('kubernetes-client') 3 | 4 | import * as fs from 'fs' 5 | import Request from './monkeypatch/client' 6 | import { Models } from './models' 7 | import { Readable } from 'stream' 8 | import Logger, { ILogger } from 'lib/logger' 9 | 10 | export enum KubernetesEventType { 11 | ALL, 12 | ADDED, 13 | MODIFIED, 14 | DELETED 15 | } 16 | 17 | export enum PatchType { 18 | MergePatch, 19 | JsonPatch 20 | } 21 | 22 | export interface IKubernetesWatchEvent { 23 | type: string 24 | object: T 25 | } 26 | 27 | interface IExecResult { 28 | stdout: string 29 | stderr: string 30 | error: string 31 | code: number 32 | } 33 | 34 | interface IWatchOptions { 35 | timeout: number 36 | qs: { 37 | resourceVersion?: string 38 | } 39 | } 40 | 41 | interface IGodaddyPostResult { 42 | messages: any[] 43 | } 44 | 45 | export interface IGodaddyClient { 46 | loadSpec() 47 | addCustomResourceDefinition(spec: Models.IExistingResource) 48 | apis: { 49 | [apiName: string]: { [version: string]: any } 50 | } 51 | api: { [version: string]: any } 52 | } 53 | 54 | export interface IGodaddyWatch { 55 | getObjectStream(options: { 56 | timeout?: number 57 | qs: { 58 | resourceVersion?: string 59 | } 60 | }): Promise 61 | } 62 | 63 | export interface IGodaddyApi { 64 | get(options?: { qs: { labelSelector: string } }) 65 | delete() 66 | post(body: any) 67 | patch(data: any) 68 | watch: any 69 | namespaces(name: string) 70 | exec: { 71 | post(options: { 72 | qs: { 73 | command: any 74 | container: string 75 | stdout: boolean 76 | stderr: boolean 77 | } 78 | }): Promise 79 | } 80 | } 81 | 82 | export enum PodPhase { 83 | Pending, 84 | Running, 85 | Succeeded, 86 | Failed, 87 | Unknown 88 | } 89 | 90 | declare type ResourceCallback = ( 91 | resource: T & Models.IExistingResource, 92 | eventType: KubernetesEventType 93 | ) => void 94 | declare type EventCallback = ( 95 | event: IKubernetesWatchEvent 96 | ) => void 97 | 98 | export interface IKubernetes { 99 | /** 100 | * Returns a single kubernetes resource 101 | * @param {string} api The Kubernetes API to target 102 | * @param {T['apiVersion']} apiVersion The version of that API 103 | * @param {T['kind']} kind The Kind of object to select 104 | * @param {string} namespace The namespace to go to 105 | * @param {string} name The name of the object 106 | * @returns {Promise<(T & Models.IExistingResource) | null>} The object, or undefined if not found 107 | */ 108 | get( 109 | apiVersion: T['apiVersion'], 110 | kind: T['kind'], 111 | namespace: string, 112 | name: string 113 | ): Promise<(T & Models.IExistingResource) | null> 114 | 115 | /** 116 | * Executes a command inside a pod, and returns the result 117 | * @param {string} namespace The namespace to go to 118 | * @param {string} name The name of the pod 119 | * @param {string} container The name of the container 120 | * @param {string[]} command The command to execute 121 | * @returns {Promise} An object containing the stdout, stderr and exit codes 122 | */ 123 | exec( 124 | namespace: string, 125 | name: string, 126 | container: string, 127 | command: string[] 128 | ): Promise 129 | 130 | /** 131 | * Returns a collection of kubernetes resources based on the selection criteria 132 | * @param {T['apiVersion']} apiVersion The version of that API 133 | * @param {T['kind']} kind The Kind of object to select 134 | * @param {string?} namespace (optional) The namespace to restrict to 135 | * @param {string?} labelSelector (optional) The label to select, eg app=your-app 136 | * @returns {Promise<(T & Models.IExistingResource)[]>} An array of Kubernetes resources 137 | */ 138 | select( 139 | apiVersion: T['apiVersion'], 140 | kind: T['kind'], 141 | namespace?: string | null | undefined, 142 | labelSelector?: string | null | undefined 143 | ): Promise<(T & Models.IExistingResource)[]> 144 | 145 | /** 146 | * Patch a kubernetes resource 147 | * @param {T['apiVersion']} apiVersion The version of that API 148 | * @param {T['kind']} kind The Kind of object to select 149 | * @param {string} namespace The namespace to go to 150 | * @param {string} name The name of the object 151 | * @param {PatchType} patchType The type of patch operation to run 152 | * @param {any} patch The patch to apply 153 | * @returns {Promise} A promise to indicate if the request was successful or not 154 | */ 155 | patch( 156 | apiVersion: T['apiVersion'], 157 | kind: T['kind'], 158 | namespace: string, 159 | name: string, 160 | patchType: PatchType.MergePatch, 161 | patch: any 162 | ): Promise 163 | patch( 164 | apiVersion: T['apiVersion'], 165 | kind: T['kind'], 166 | namespace: string, 167 | name: string, 168 | patchType: PatchType.JsonPatch, 169 | patch: jsonpatch.OpPatch[] 170 | ): Promise 171 | 172 | /** 173 | * Patch a kubernetes resource 174 | * @param {T['apiVersion']} apiVersion The version of that API 175 | * @param {T['kind']} kind The Kind of object to select 176 | * @param {string} namespace The namespace to go to 177 | * @param {string} name The name of the object 178 | * @param {string} type The status name 179 | * @param {string} status The status value 180 | * @returns {Promise} A promise to indicate if the request was successful or not 181 | */ 182 | addStatusCondition( 183 | apiVersion: T['apiVersion'], 184 | kind: T['kind'], 185 | namespace: string, 186 | name: string, 187 | type: string, 188 | status: string 189 | ): Promise 190 | 191 | /** 192 | * Create a new kubernetes resource 193 | * @param {T & Models.INewResource} manifest The manifest to create 194 | * @returns {Promise} A promise to indicate if the request was successful or not 195 | */ 196 | create( 197 | manifest: T & Models.INewResource 198 | ): Promise 199 | 200 | /** 201 | * Removes a kubernetes resource from the cluster 202 | * @param {T['apiVersion']} apiVersion The version of that API 203 | * @param {T['kind']} kind The Kind of object to select 204 | * @param {string} namespace The namespace to go to 205 | * @param {string} name The name of the object 206 | * @returns {Promise} A promise to indicate if the request was successful or not 207 | */ 208 | delete( 209 | apiVersion: T['apiVersion'], 210 | kind: T['kind'], 211 | namespace: string, 212 | name: string 213 | ): Promise 214 | 215 | /** 216 | * Watch the kubernetes API for a given resource type 217 | * Will handle auto reconnection 218 | * @param {T['apiVersion']} apiVersion The version of that API 219 | * @param {T['kind']} kind The Kind of object to select 220 | * @param {ResourceCallback} handler The handler that will be invoked with the resource 221 | * @param {KubernetesEventType[]} eventType The types to watch for (default: MODIFIED, ADDED, DELETED) 222 | * @param {string?} namespace The namespace to restrict the watch to 223 | * @returns {Promise} A promise to indicate if the watch was setup successfully or not 224 | */ 225 | watch( 226 | apiVersion: T['apiVersion'], 227 | kind: T['kind'], 228 | handler: ResourceCallback, 229 | eventTypes?: KubernetesEventType[], 230 | namespace?: string | null 231 | ): Promise 232 | 233 | /** 234 | * Starts all watch streams 235 | * @returns {Promise} A promise which returns when all watch operations have started 236 | */ 237 | start() 238 | 239 | /** 240 | * Stops all watch streams 241 | * @returns {Promise} A promise that returns when all watch operations have stopped 242 | */ 243 | stop() 244 | 245 | /** 246 | * Waits for a pod to enter a particular phase 247 | * @param {string} namespace The namespace to go to 248 | * @param {string} name The name of the object 249 | * @param {PodPhase?} phase The phase to wait for, defaults to Running 250 | * @param {number?} maxTime The maximum amount of time in milliseconds to wait 251 | * @returns {Promise} A promise that returns when the desired state is met 252 | */ 253 | waitForPodPhase( 254 | namespace: string, 255 | name: string, 256 | phase?: PodPhase, 257 | maxTime?: number 258 | ): Promise 259 | 260 | /** 261 | * Waits for something to rollout 262 | * @param {T['apiVersion']} apiVersion The version of that API 263 | * @param {T['kind']} kind The Kind of object to select 264 | * @param {string} namespace The namespace to go to 265 | * @param {string} name The name of the object 266 | * @param {number?} maxTime The maximum amount of time in milliseconds to wait 267 | * @returns {Promise} A promise that returns when all replicas are up to date 268 | */ 269 | waitForRollout( 270 | apiVersion: T['apiVersion'], 271 | kind: T['kind'], 272 | namespace: string, 273 | name: string, 274 | maxTime?: number 275 | ): Promise 276 | 277 | /** 278 | * Waits for a load balancer service to be assigned an external ip 279 | * 280 | * Non-load balancer services return immediately 281 | * @param {string} namespace The namespace to go to 282 | * @param {string} name The name of the object 283 | * @param {number?} maxTime The maximum amount of time in milliseconds to wait 284 | * @returns {Promise} A promise that returns once a load balancer ip is assigned 285 | */ 286 | waitForServiceLoadBalancerIp( 287 | namespace: string, 288 | name: string, 289 | maxTime?: number 290 | ): Promise 291 | 292 | /** 293 | * Trigger a rollout of a pod controller 294 | * @param {T['apiVersion']} apiVersion The version of that API 295 | * @param {T['kind']} kind the Kind of object to rollout 296 | * @param {string} namespace The namespace to go to 297 | * @param {string} name The name of the object 298 | * @returns {Promise} A promise that returns once a rollout has been triggered 299 | */ 300 | rollout< 301 | T extends 302 | | Models.Core.IDeployment 303 | | Models.Core.IStatefulSet 304 | | Models.Core.IDaemonSet 305 | >( 306 | apiVersion: T['apiVersion'], 307 | kind: T['kind'], 308 | namespace: string, 309 | name: string 310 | ): Promise 311 | } 312 | 313 | export default class Kubernetes implements IKubernetes { 314 | private client: IGodaddyClient 315 | private streamsToStart: (() => Promise)[] = [] 316 | private streams: Readable[] = [] 317 | private initialised = false 318 | private initialising = false 319 | private logger: ILogger 320 | private loadCustomResources = false 321 | 322 | // The timeout for GET, POST, PUT, PATCH, DELETE 323 | private requestTimeout = 5000 324 | // The timeout for WATCH 325 | private watchTimeout = 60000 326 | // How long will we wait for watch reconnections before erroring 327 | private watchReconnectTimeout = 300000 328 | 329 | constructor( 330 | options: { 331 | loadCustomResources?: boolean 332 | client?: IGodaddyClient 333 | } = {} 334 | ) { 335 | this.logger = new Logger('client') 336 | this.loadCustomResources = options.loadCustomResources || false 337 | 338 | const request = { timeout: this.requestTimeout } 339 | if (options.client) { 340 | this.client = options.client 341 | this.logger.info('using injected client') 342 | return 343 | } 344 | if (fs.existsSync('/var/run/secrets/kubernetes.io/serviceaccount')) { 345 | this.logger.info('using service account') 346 | const kubeconfig = new KubeConfig() 347 | kubeconfig.loadFromCluster() 348 | const backend = new Request({ kubeconfig, request }) 349 | this.client = new Client({ backend }) 350 | } else { 351 | this.logger.info('using kube config') 352 | const kubeconfig = new KubeConfig() 353 | kubeconfig.loadFromDefault() 354 | const backend = new Request({ kubeconfig, request }) 355 | this.client = new Client({ backend }) 356 | } 357 | } 358 | 359 | private async init(): Promise { 360 | if (this.initialised) { 361 | return 362 | } else if (this.initialising) { 363 | this.logger.debug( 364 | 'another instance of init is running, waiting for it to complete' 365 | ) 366 | let waitedFor = 0 367 | while (!this.initialised) { 368 | await this.sleep(50) 369 | waitedFor += 1 370 | if (waitedFor > 100) { 371 | throw new Error('waited 5000 ms for init to complete, it didnt') 372 | } 373 | } 374 | return 375 | } 376 | this.initialising = true 377 | this.logger.debug('loading kubernetes spec') 378 | await this.client.loadSpec() 379 | if (this.loadCustomResources) { 380 | this.logger.debug('spec loaded, loading crds') 381 | const query = await (this.client.apis['apiextensions.k8s.io'] 382 | .v1beta1 as any).customresourcedefinition.get() 383 | query.body.items.forEach((crd) => { 384 | this.client.addCustomResourceDefinition(crd) 385 | }) 386 | this.logger.debug(`${query.body.items.length} crds loaded`) 387 | } 388 | this.logger.debug('loading complete') 389 | this.initialised = true 390 | } 391 | 392 | private getApiParameters( 393 | apiVersion: T['apiVersion'], 394 | kind: T['kind'] 395 | ): { api: string; version: string; kind: string } { 396 | const [api, version] = 397 | apiVersion.indexOf('/') > -1 ? apiVersion.split('/') : ['', apiVersion] 398 | 399 | let kindLower = kind.toLowerCase() 400 | if (kindLower === 'networkpolicy') { 401 | kindLower = 'networkpolicie' 402 | } 403 | return { api, version, kind: kindLower } 404 | } 405 | 406 | private async getApi( 407 | apiVersion: T['apiVersion'], 408 | kind: T['kind'], 409 | namespace?: string, 410 | name?: string 411 | ): Promise { 412 | await this.init() 413 | const { api, version, kind: kindLower } = this.getApiParameters( 414 | apiVersion, 415 | kind 416 | ) 417 | 418 | this.logger.debug('api handler:', api, version, kindLower, namespace, name) 419 | let query = 420 | api === '' ? this.client.api[version] : this.client.apis[api][version] 421 | if (namespace) { 422 | query = query.namespaces(namespace) 423 | } 424 | const result = await query[kindLower] 425 | if (!name) { 426 | if (typeof result === 'undefined') { 427 | throw new Error(`No handler found for ${version}/${api}/${kindLower}`) 428 | } 429 | return result 430 | } 431 | if (typeof result === 'undefined') { 432 | throw new Error( 433 | `No handler found for ${version}/${api}/${kindLower}/${namespace}/${name}` 434 | ) 435 | } 436 | return result(name) 437 | } 438 | 439 | private sleep(ms: number) { 440 | return new Promise((resolve) => setTimeout(resolve, ms)) 441 | } 442 | 443 | private streamFor( 444 | apiVersion: T['apiVersion'], 445 | kind: T['kind'], 446 | wrappedHandler: EventCallback, 447 | namespace?: string | null 448 | ) { 449 | const { api, version, kind: kindLower } = this.getApiParameters( 450 | apiVersion, 451 | kind 452 | ) 453 | const key = `${api}:${version}:${kindLower}` 454 | const delayBetweenRetries = 1000 455 | 456 | let streamStarted = new Date(new Date().toUTCString()) 457 | let lastResourceVersion: string | null = null 458 | // The longest we ever want to wait really is 5 minutes as that 459 | // is the default configuration of etcds cache 460 | let reconnectRetriesTimeout: NodeJS.Timeout | null = null 461 | 462 | const handleEvent = ( 463 | event: IKubernetesWatchEvent 464 | ) => { 465 | if (!event.object || !event.object.metadata) { 466 | this.logger.debug(key, 'invalid resource returned', event) 467 | return 468 | } 469 | 470 | if (event.type === 'ERROR' && (event.object as any).code === 410) { 471 | // Resource has gone away, we're using a resourceVersion that is too old 472 | this.logger.debug( 473 | key, 474 | `the last seen resourceVersion: ${lastResourceVersion} was not found in etcd cache` 475 | ) 476 | lastResourceVersion = null 477 | return 478 | } 479 | 480 | const resourceCreationTimestamp = new Date( 481 | event.object.metadata.creationTimestamp as string 482 | ) 483 | 484 | /* If we are: 485 | * - In a replay state 486 | * - And the event is of type ADDED 487 | * - And the event creation date is < when the stream started 488 | * then we should ignore the event 489 | */ 490 | if ( 491 | lastResourceVersion === null && 492 | KubernetesEventType[event.type] === KubernetesEventType.ADDED && 493 | resourceCreationTimestamp < streamStarted 494 | ) { 495 | return 496 | } 497 | 498 | this.logger.debug( 499 | `saw resource: ${event.type} ${event.object.metadata.selfLink} ${event.object.metadata.resourceVersion}` 500 | ) 501 | 502 | // Increment the stream last seen version 503 | lastResourceVersion = event.object.metadata.resourceVersion 504 | 505 | wrappedHandler(event) 506 | } 507 | 508 | const result = 509 | api === '' ? this.client.api[version] : this.client.apis[api][version] 510 | 511 | const watchObject = namespace 512 | ? result.watch.namespaces(namespace) 513 | : result.watch 514 | 515 | if (!watchObject) { 516 | throw new Error(`No handler found for ${version}/${api}/${kindLower}`) 517 | } 518 | 519 | const startReconnectTimeout = () => { 520 | if (!reconnectRetriesTimeout) { 521 | reconnectRetriesTimeout = setTimeout(() => { 522 | this.logger.error( 523 | key, 524 | 'Failed to reconnect within timeout, exiting process' 525 | ) 526 | throw new Error( 527 | `Unable to reconnect ${key} to kubernetes after ${ 528 | this.watchReconnectTimeout / 1000 529 | }s` 530 | ) 531 | }, this.watchReconnectTimeout) 532 | } 533 | } 534 | 535 | const endReconnectTimeout = () => { 536 | if (reconnectRetriesTimeout) { 537 | clearTimeout(reconnectRetriesTimeout) 538 | reconnectRetriesTimeout = null 539 | } 540 | } 541 | 542 | const watch: IGodaddyWatch = watchObject[kindLower] 543 | const reconnect = async ( 544 | retryCount: number, 545 | streamPromise: (retryCount: number) => Promise 546 | ): Promise => { 547 | startReconnectTimeout() 548 | return streamPromise(retryCount) 549 | } 550 | 551 | /* eslint max-statements: off */ 552 | const streamPromise = async (previousRetryCount = 1): Promise => { 553 | let retryCount = previousRetryCount 554 | streamStarted = new Date(new Date().toUTCString()) 555 | 556 | const handleError = (stream: Readable) => { 557 | return async (err) => { 558 | this.destroyStream(stream) 559 | if (err.message === 'ESOCKETTIMEDOUT') { 560 | /* Represents our client timing out after successful connection, but seeing no data */ 561 | this.logger.debug(key, 'stream read timed out! Reconnecting...') 562 | endReconnectTimeout() 563 | reconnect(1, streamPromise) 564 | } else { 565 | /* Represents unexpected errors */ 566 | this.logger.error( 567 | key, 568 | 'stream encountered an error! Reconnecting...', 569 | { 570 | error: { 571 | message: err.message, 572 | name: err.name 573 | } 574 | } 575 | ) 576 | await this.sleep(delayBetweenRetries) 577 | reconnect(retryCount + 1, streamPromise) 578 | } 579 | } 580 | } 581 | 582 | const handleEnd = (stream: Readable) => { 583 | /* Represents kubernetes closing the stream */ 584 | return () => { 585 | this.logger.debug(key, 'stream was closed. Reconnecting...') 586 | this.destroyStream(stream) 587 | endReconnectTimeout() 588 | reconnect(1, streamPromise) 589 | } 590 | } 591 | 592 | let stream: Readable | undefined 593 | try { 594 | const watchOptions: IWatchOptions = { 595 | timeout: this.watchTimeout, 596 | qs: {} 597 | } 598 | 599 | if (lastResourceVersion) { 600 | watchOptions.qs.resourceVersion = lastResourceVersion 601 | } 602 | this.logger.debug( 603 | key, 604 | `stream starting (retry ${retryCount}, resourceVersion: ${lastResourceVersion}. Current active streams: ${this.streams.length}` 605 | ) 606 | stream = (await watch.getObjectStream(watchOptions)) as Readable 607 | this.streams.push(stream) 608 | stream.on('data', () => { 609 | endReconnectTimeout() 610 | retryCount = 1 611 | }) 612 | stream.on('data', handleEvent) 613 | stream.on('error', handleError(stream)) 614 | stream.on('end', handleEnd(stream)) 615 | } catch (ex) { 616 | if (ex) { 617 | this.logger.error(key, 'error setting up watch', ex) 618 | } 619 | await this.sleep(delayBetweenRetries) 620 | /* Represents unexpected errors in the stream */ 621 | if (stream) { 622 | this.destroyStream(stream) 623 | } 624 | reconnect(retryCount + 1, streamPromise) 625 | } 626 | } 627 | 628 | this.streamsToStart.push(streamPromise) 629 | } 630 | 631 | private async waitFor( 632 | apiVersion: T['apiVersion'], 633 | kind: T['kind'], 634 | namespace: string, 635 | name: string, 636 | condition: (resource: T & Models.IExistingResource) => boolean, 637 | maxTime = 10000 638 | ): Promise { 639 | const start = new Date() 640 | const isPresentAndConditionPasses = (resource) => { 641 | return resource ? condition(resource) : false 642 | } 643 | 644 | let resource = await this.get(apiVersion, kind, namespace, name) 645 | /* eslint no-await-in-loop: off */ 646 | while (!isPresentAndConditionPasses(resource)) { 647 | if (new Date().valueOf() - start.valueOf() > maxTime) { 648 | throw new Error('Timeout exceeded') 649 | } 650 | if (!resource) { 651 | throw new Error( 652 | `Failed to find a ${kind} named ${name} in ${namespace}` 653 | ) 654 | } 655 | await this.sleep(1000) 656 | resource = await this.get(apiVersion, kind, namespace, name) 657 | } 658 | } 659 | 660 | private destroyStream(stream: Readable) { 661 | try { 662 | stream.removeAllListeners() 663 | stream.destroy() 664 | } catch (ex) { 665 | this.logger.warn('Failed to destroy stream during cleanup', ex.message) 666 | } finally { 667 | this.streams = this.streams.filter((arrayItem) => arrayItem !== stream) 668 | } 669 | } 670 | 671 | public async waitForPodPhase( 672 | namespace: string, 673 | name: string, 674 | phase: PodPhase = PodPhase.Running, 675 | maxTime = 10000 676 | ): Promise { 677 | const condition = (pod: Models.Core.IPod) => { 678 | if (pod?.status?.phase) { 679 | return PodPhase[pod.status.phase] === phase 680 | } 681 | return false 682 | } 683 | return this.waitFor( 684 | 'v1', 685 | 'Pod', 686 | namespace, 687 | name, 688 | condition, 689 | maxTime 690 | ) 691 | } 692 | 693 | public async waitForRollout( 694 | apiVersion: T['apiVersion'], 695 | kind: T['kind'], 696 | namespace: string, 697 | name: string, 698 | maxTime = 10000 699 | ): Promise { 700 | const assertion = (resource: T) => { 701 | const progressingStatus = resource.status?.conditions?.find( 702 | (condition) => condition.type === 'Progressing' 703 | ) 704 | const isProgressing = 705 | progressingStatus && 706 | progressingStatus.status === 'True' && 707 | progressingStatus.reason === 'NewReplicaSetAvailable' 708 | const isUpdated = 709 | resource.status?.updatedReplicas === resource.spec.replicas 710 | const hasSameReplicas = 711 | resource.status?.replicas === resource.spec.replicas 712 | return (isProgressing && isUpdated && hasSameReplicas) || false 713 | } 714 | return this.waitFor( 715 | apiVersion, 716 | kind, 717 | namespace, 718 | name, 719 | assertion, 720 | maxTime 721 | ) 722 | } 723 | 724 | public async waitForServiceLoadBalancerIp( 725 | namespace: string, 726 | name: string, 727 | maxTime = 60000 728 | ): Promise { 729 | const condition = (service: Models.Core.IService) => { 730 | if (service.spec.type !== 'LoadBalancer') { 731 | return true 732 | } 733 | return (service.status?.loadBalancer?.ingress?.length || 0) > 0 734 | } 735 | return this.waitFor( 736 | 'v1', 737 | 'Service', 738 | namespace, 739 | name, 740 | condition, 741 | maxTime 742 | ) 743 | } 744 | 745 | public async exec( 746 | namespace: string, 747 | name: string, 748 | container: string, 749 | command: string[] 750 | ): Promise { 751 | try { 752 | const api = await this.getApi( 753 | 'v1', 754 | 'Pod', 755 | namespace, 756 | name 757 | ) 758 | const res = await api.exec.post({ 759 | qs: { 760 | command, 761 | container, 762 | stdout: true, 763 | stderr: true 764 | } 765 | }) 766 | const isEmpty = (item: any) => { 767 | return typeof item === 'undefined' || item === 'null' || item === '' 768 | } 769 | 770 | const filterByType = (type: string) => { 771 | return res.messages 772 | .filter((item) => item.channel === type) 773 | .map((item) => item.message) 774 | .filter((item) => !isEmpty(item)) 775 | .join('\n') 776 | } 777 | const stdout = filterByType('stdout') 778 | const stderr = filterByType('stderr') 779 | const error = filterByType('error') 780 | const result = { 781 | stdout: stdout.trim(), 782 | stderr: stderr.trim(), 783 | error, 784 | code: 0 785 | } 786 | 787 | const codeMatch = error.match( 788 | /command terminated with non-zero exit code: Error executing in Docker Container: (\d+)/ 789 | ) 790 | if (codeMatch) { 791 | result.code = parseInt(codeMatch[1], 10) 792 | } 793 | return result 794 | } catch (ex) { 795 | this.logger.error('Error executing command on kubernetes', ex) 796 | throw ex 797 | } 798 | } 799 | 800 | public async delete( 801 | apiVersion: T['apiVersion'], 802 | kind: T['kind'], 803 | namespace: string, 804 | name: string 805 | ): Promise { 806 | try { 807 | const api = await this.getApi(apiVersion, kind, namespace, name) 808 | await api.delete() 809 | } catch (ex) { 810 | if (ex.code !== 200) { 811 | this.logger.error('Error deleting item from kubernetes', ex) 812 | throw ex 813 | } 814 | } 815 | } 816 | 817 | public async get( 818 | apiVersion: T['apiVersion'], 819 | kind: T['kind'], 820 | namespace: string, 821 | name: string 822 | ): Promise<(T & Models.IExistingResource) | null> { 823 | try { 824 | const api = await this.getApi(apiVersion, kind, namespace, name) 825 | const result = await api.get() 826 | return result.body 827 | } catch (ex) { 828 | if (ex.code !== 404) { 829 | this.logger.error('Error getting item from kubernetes', ex) 830 | throw new Error(ex) 831 | } 832 | return null 833 | } 834 | } 835 | 836 | public async select( 837 | apiVersion: T['apiVersion'], 838 | kind: T['kind'], 839 | namespace?: string, 840 | labelSelector?: string 841 | ): Promise<(T & Models.IExistingResource)[]> { 842 | const api = await this.getApi(apiVersion, kind, namespace) 843 | const result = labelSelector 844 | ? await api.get({ 845 | qs: { 846 | labelSelector 847 | } 848 | }) 849 | : await api.get() 850 | 851 | if (result.statusCode !== 200) { 852 | throw new Error( 853 | `Non-200 status code returned from Kubernetes API (${result.statusCode})` 854 | ) 855 | } 856 | return result.body.items 857 | } 858 | 859 | public async create( 860 | manifest: T & Models.INewResource 861 | ): Promise { 862 | const kindHandler = await this.getApi( 863 | manifest.apiVersion, 864 | manifest.kind, 865 | manifest.metadata.namespace 866 | ) 867 | return kindHandler.post({ body: manifest }) 868 | } 869 | 870 | public async addStatusCondition( 871 | apiVersion: T['apiVersion'], 872 | kind: T['kind'], 873 | namespace: string, 874 | name: string, 875 | type: string, 876 | status: string 877 | ): Promise { 878 | const patch = [ 879 | { 880 | op: 'add', 881 | path: '/status/conditions/-', 882 | value: { type, status } 883 | } 884 | ] 885 | return this.patch( 886 | apiVersion, 887 | kind, 888 | namespace, 889 | name, 890 | PatchType.JsonPatch, 891 | patch 892 | ) 893 | } 894 | 895 | public async patch( 896 | apiVersion: T['apiVersion'], 897 | kind: T['kind'], 898 | namespace: string, 899 | name: string, 900 | patchType: PatchType, 901 | patch: any 902 | ): Promise { 903 | const patchTypeMappings = { 904 | 0: 'application/merge-patch+json', 905 | 1: 'application/json-patch+json' 906 | } 907 | const contentType = patchTypeMappings[patchType] 908 | 909 | if (!contentType) { 910 | throw new Error( 911 | `Unable to match patch ${PatchType[patchType]} to a content type` 912 | ) 913 | } 914 | 915 | const patchMutated: any = { 916 | headers: { 917 | accept: 'application/json', 918 | 'content-type': contentType 919 | }, 920 | body: patch 921 | } 922 | 923 | // If this is a JSON patch to add a status condition, then mutate the url 924 | if ( 925 | patchType === PatchType.JsonPatch && 926 | patch[0].op === 'add' && 927 | patch[0].path === '/status/conditions/-' 928 | ) { 929 | let plural = kind.toLowerCase() 930 | if (plural.slice(-1) !== 's') { 931 | plural += 's' 932 | } 933 | 934 | const date = new Date().toISOString().split('.')[0] 935 | patch[0].value.lastTransitionTime = `${date}.000000Z` 936 | 937 | const pathname = `/api/v1/namespaces/${namespace}/${plural}/${name}/status` 938 | patchMutated.pathname = pathname 939 | } 940 | 941 | const api = await this.getApi(apiVersion, kind, namespace, name) 942 | const result = await api.patch(patchMutated) 943 | 944 | if (result.statusCode !== 200) { 945 | throw new Error( 946 | `Non-200 status code returned from Kubernetes API (${result.statusCode})` 947 | ) 948 | } 949 | } 950 | 951 | public async watch( 952 | apiVersion: T['apiVersion'], 953 | kind: T['kind'], 954 | handler: ResourceCallback, 955 | eventTypes: KubernetesEventType[] = [KubernetesEventType.ALL], 956 | namespace?: string | null 957 | ) { 958 | await this.init() 959 | const wrappedHandler: EventCallback = ( 960 | event: IKubernetesWatchEvent 961 | ) => { 962 | const eventType: KubernetesEventType = KubernetesEventType[event.type] 963 | if ( 964 | eventTypes.indexOf(eventType) > -1 || 965 | eventTypes.indexOf(KubernetesEventType.ALL) > -1 966 | ) { 967 | handler(event.object, eventType) 968 | } 969 | } 970 | this.streamFor(apiVersion, kind, wrappedHandler, namespace) 971 | } 972 | 973 | public async start(): Promise { 974 | await this.init() 975 | this.logger.log('starting all streams') 976 | for (const stream of this.streamsToStart) { 977 | await stream() 978 | } 979 | } 980 | 981 | public async stop(): Promise { 982 | this.logger.log('stopping all streams') 983 | for (const stream of this.streams) { 984 | await this.destroyStream(stream) 985 | } 986 | } 987 | 988 | public async rollout< 989 | T extends 990 | | Models.Core.IDeployment 991 | | Models.Core.IStatefulSet 992 | | Models.Core.IDaemonSet 993 | >( 994 | apiVersion: T['apiVersion'], 995 | kind: T['kind'], 996 | namespace: string, 997 | name: string 998 | ): Promise { 999 | const patch = { 1000 | spec: { 1001 | template: { 1002 | metadata: { 1003 | annotations: { 1004 | 'node-at-kubernetes/restartedAt': `${new Date().getTime()}` 1005 | } 1006 | } 1007 | } 1008 | } 1009 | } 1010 | return this.patch( 1011 | apiVersion, 1012 | kind, 1013 | namespace, 1014 | name, 1015 | PatchType.MergePatch, 1016 | patch 1017 | ) 1018 | } 1019 | } 1020 | -------------------------------------------------------------------------------- /lib/kubernetes/models.ts: -------------------------------------------------------------------------------- 1 | import { 2 | DaemonSet, 3 | DaemonSetSpec, 4 | Deployment, 5 | DeploymentSpec, 6 | StatefulSet, 7 | StatefulSetSpec 8 | } from 'kubernetes-types/apps/v1' 9 | import { 10 | Event, 11 | Node, 12 | NodeSpec, 13 | Pod, 14 | PodSpec, 15 | Secret, 16 | Service, 17 | ServiceSpec 18 | } from 'kubernetes-types/core/v1' 19 | import { Ingress, IngressSpec } from 'kubernetes-types/extensions/v1beta1' 20 | import { ObjectMeta } from 'kubernetes-types/meta/v1' 21 | import { 22 | NetworkPolicy, 23 | NetworkPolicySpec 24 | } from 'kubernetes-types/networking/v1' 25 | import { XOR } from 'ts-xor' 26 | 27 | export namespace Models { 28 | type Metadata = Omit & { 29 | namespace: string 30 | } 31 | 32 | /** 33 | * Represents core fields all objects share 34 | */ 35 | export interface IBaseResource { 36 | kind: string 37 | apiVersion: string 38 | metadata: Metadata 39 | } 40 | 41 | /** 42 | * Generic intersection types which take 'kubernetes-types' 43 | * interfaces and make their IBaseResource keys mandatory 44 | */ 45 | type BaseResource = Omit & IBaseResource 46 | type BaseResourceWithSpec = BaseResource & { spec: R } 47 | 48 | /** 49 | * Additional fields that are found on already existing resources 50 | */ 51 | export interface IExistingResource extends IBaseResource { 52 | metadata: { 53 | namespace: string 54 | name: string 55 | selfLink: string 56 | creationTimestamp: string 57 | resourceVersion: string 58 | uid: string 59 | labels?: { [key: string]: string } 60 | annotations?: { [key: string]: string } 61 | deletionTimestamp?: string 62 | } 63 | } 64 | 65 | /** 66 | * Additional fields required for creating new objects 67 | */ 68 | export interface INewResource extends IBaseResource { 69 | metadata: Metadata & XOR<{ name: string }, { generateName: string }> 70 | } 71 | 72 | export namespace Core { 73 | /** 74 | * Represents v1.NetworkPolicy 75 | */ 76 | export interface INetworkPolicy 77 | extends BaseResourceWithSpec {} 78 | 79 | /** 80 | * Represents v1.Pod 81 | */ 82 | export interface IPod extends BaseResourceWithSpec {} 83 | 84 | /** 85 | * Represents v1.Secret 86 | */ 87 | export interface ISecret extends BaseResource {} 88 | 89 | /** 90 | * Represents v1.Deployment 91 | */ 92 | export interface IDeployment 93 | extends BaseResourceWithSpec {} 94 | 95 | /** 96 | * Represents v1.StatefulSet 97 | */ 98 | export interface IStatefulSet 99 | extends BaseResourceWithSpec {} 100 | 101 | /** 102 | * Represents v1.DaemonSet 103 | */ 104 | export interface IDaemonSet 105 | extends BaseResourceWithSpec {} 106 | 107 | /** 108 | * Represents v1.Service 109 | */ 110 | export interface IService 111 | extends BaseResourceWithSpec {} 112 | 113 | /** 114 | * Represents networking.k8s.io/v1beta1.Ingress or extensions/v1beta1.Ingress 115 | */ 116 | export interface IIngress 117 | extends BaseResourceWithSpec, IngressSpec> { 118 | apiVersion: 'networking.k8s.io/v1beta1' | 'extensions/v1beta1' 119 | } 120 | 121 | /** 122 | * Represents v1.Node 123 | */ 124 | export interface INode extends BaseResourceWithSpec {} 125 | 126 | /** 127 | * Represents v1.Event 128 | */ 129 | export interface IEvent extends BaseResource {} 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /lib/kubernetes/monkeypatch/client.ts: -------------------------------------------------------------------------------- 1 | const { 2 | convertKubeconfig 3 | } = require('kubernetes-client/backends/request/config') 4 | const deprecate = require('depd')('kubernetes-client') 5 | const JSONStream = require('json-stream') 6 | const pump = require('pump') 7 | const qs = require('qs') 8 | const request = require('request') 9 | const urljoin = require('url-join') 10 | const WebSocket = require('ws') 11 | 12 | /* eslint no-underscore-dangle: off */ 13 | /** 14 | * Refresh whatever authentication {type} is. 15 | * @param {String} type - type of authentication 16 | * @param {Object} config - auth provider config 17 | * @returns {Promise} with request friendly auth object 18 | */ 19 | function refreshAuth(type, config) { 20 | return new Promise((resolve, reject) => { 21 | const provider = require(`kubernetes-client/backends/request/auth-providers/${type}.js`) 22 | provider 23 | .refresh(config) 24 | .then((result) => { 25 | const auth = { 26 | bearer: result 27 | } 28 | 29 | return resolve(auth) 30 | }) 31 | .catch((err) => reject(err)) 32 | }) 33 | } 34 | 35 | const execChannels = ['stdin', 'stdout', 'stderr', 'error', 'resize'] 36 | 37 | /** 38 | * Determine whether a failed Kubernetes API response is asking for an upgrade 39 | * @param {object} body - response body object from Kubernetes 40 | * @property {string} status - request status 41 | * @property {number} code - previous request's response code 42 | * @property {message} message - previous request response message 43 | * @returns {boolean} Upgrade the request 44 | */ 45 | 46 | function isUpgradeRequired(body) { 47 | return ( 48 | body.status === 'Failure' && 49 | body.code === 400 && 50 | body.message === 'Upgrade request required' 51 | ) 52 | } 53 | 54 | /** 55 | * Upgrade a request into a Websocket transaction & process the result 56 | * @param {ApiRequestOptions} options - Options object 57 | * @param {callback} cb - The callback that handles the response 58 | */ 59 | 60 | function upgradeRequest(options, cb) { 61 | const queryParams = qs.stringify(options.qs, { indices: false }) 62 | const wsUrl = urljoin(options.baseUrl, options.uri, `?${queryParams}`) 63 | const protocol = 'base64.channel.k8s.io' 64 | 65 | // Passing authorization header 66 | options.headers = { 67 | ...options.headers, 68 | authorization: `Bearer ${options.auth.bearer}` 69 | } 70 | const ws = new WebSocket(wsUrl, protocol, options) 71 | 72 | const messages: { channel: string; message: string }[] = [] 73 | ws.on('message', (msg) => { 74 | const channel = execChannels[msg.slice(0, 1)] 75 | const message = Buffer.from(msg.slice(1), 'base64').toString('ascii') 76 | messages.push({ channel, message }) 77 | }) 78 | 79 | ws.on('error', (err) => { 80 | err.messages = messages 81 | cb(err, messages) 82 | }) 83 | 84 | ws.on('close', (code, reason) => { 85 | cb(null, { 86 | messages, 87 | body: messages.map(({ message }) => message).join(''), 88 | code, 89 | reason 90 | }) 91 | }) 92 | 93 | return ws 94 | } 95 | 96 | class ErrorWithCode extends Error { 97 | public readonly code: number 98 | public readonly statusCode: number 99 | constructor(msg: string, code: number, statusCode: number) { 100 | super(msg) 101 | this.code = code 102 | this.statusCode = statusCode 103 | } 104 | } 105 | 106 | export default class Request { 107 | private requestOptions: any 108 | private authProvider: any 109 | /** 110 | * Internal representation of HTTP request object. 111 | * 112 | * @param {object} options - Options object 113 | * @param {string} options.url - Kubernetes API URL 114 | * @param {object} options.auth - request library auth object 115 | * @param {string} options.ca - Certificate authority 116 | * @param {string} options.cert - Client certificate 117 | * @param {string} options.key - Client key 118 | * @param {boolean} options.insecureSkipTlsVerify - Skip the validity check 119 | * on the server's certificate. 120 | */ 121 | constructor(options) { 122 | this.requestOptions = options.request || {} 123 | 124 | let convertedOptions 125 | /* eslint no-negated-condition: off */ 126 | if (!options.kubeconfig) { 127 | deprecate( 128 | 'Request() without a .kubeconfig option, see ' + 129 | 'https://github.com/godaddy/kubernetes-client/blob/master/merging-with-kubernetes.md' 130 | ) 131 | convertedOptions = options 132 | } else { 133 | convertedOptions = convertKubeconfig(options.kubeconfig) 134 | } 135 | 136 | this.requestOptions.qsStringifyOptions = { indices: false } 137 | this.requestOptions.baseUrl = convertedOptions.url 138 | this.requestOptions.ca = convertedOptions.ca 139 | this.requestOptions.cert = convertedOptions.cert 140 | this.requestOptions.key = convertedOptions.key 141 | if ('insecureSkipTlsVerify' in convertedOptions) { 142 | this.requestOptions.strictSSL = !convertedOptions.insecureSkipTlsVerify 143 | } 144 | if ('timeout' in convertedOptions) { 145 | this.requestOptions.timeout = convertedOptions.timeout 146 | } 147 | 148 | this.authProvider = { 149 | type: null 150 | } 151 | if (convertedOptions.auth) { 152 | this.requestOptions.auth = convertedOptions.auth 153 | if (convertedOptions.auth.provider) { 154 | this.requestOptions.auth = convertedOptions.auth.request 155 | this.authProvider = convertedOptions.auth.provider 156 | } 157 | } 158 | } 159 | 160 | _request(options, cb) { 161 | const auth = this.authProvider 162 | return request(options, (err, res, body) => { 163 | if (err) { 164 | return cb(err) 165 | } 166 | 167 | if (body && isUpgradeRequired(body)) { 168 | return upgradeRequest(options, cb) 169 | } 170 | 171 | // Refresh auth if 401 or 403 172 | if ((res.statusCode === 401 || res.statusCode === 403) && auth.type) { 173 | return refreshAuth(auth.type, auth.config) 174 | .then((newAuth) => { 175 | this.requestOptions.auth = newAuth 176 | options.auth = newAuth 177 | return request(options, (requestErr, requestRes, requestBody) => { 178 | if (requestErr) { 179 | return cb(requestErr) 180 | } 181 | return cb(null, { 182 | statusCode: requestRes.statusCode, 183 | requestBody 184 | }) 185 | }) 186 | }) 187 | .catch((refreshAuthErr) => cb(refreshAuthErr)) 188 | } 189 | 190 | return cb(null, { statusCode: res.statusCode, body }) 191 | }) 192 | } 193 | 194 | async getLogByteStream(options) { 195 | return this.http(Object.assign({ stream: true }, options)) 196 | } 197 | 198 | async getWatchObjectStream(options) { 199 | const jsonStream = new JSONStream() 200 | const stream = this.http(Object.assign({ stream: true }, options)) 201 | stream.once('error', (err) => { 202 | jsonStream.emit('error', err) 203 | }) 204 | jsonStream.destroy = function () { 205 | stream.abort() 206 | stream.destroy() 207 | } 208 | pump(stream, jsonStream) 209 | return jsonStream 210 | } 211 | 212 | /** 213 | * @param {object} options - Options object 214 | * @param {Stream} options.stdin - optional stdin Readable stream 215 | * @param {Stream} options.stdout - optional stdout Writeable stream 216 | * @param {Stream} options.stderr - optional stdout Writeable stream 217 | * @returns {Promise} Promise resolving to a Kubernetes V1 Status object and a WebSocket 218 | */ 219 | async getWebSocket() { 220 | throw new Error('Request.getWebSocket not implemented') 221 | } 222 | 223 | /** 224 | * @typedef {object} ApiRequestOptions 225 | * @property {object} body - Request body 226 | * @property {object} headers - Headers object 227 | * @property {string} path - version-less path 228 | * @property {object} qs - {@link https://www.npmjs.com/package/request#requestoptions-callback| 229 | * request query parameter} 230 | */ 231 | 232 | /** 233 | * Invoke a REST request against the Kubernetes API server 234 | * @param {ApiRequestOptions} options - Options object 235 | * @param {callback} cb - The callback that handles the response 236 | * @returns {Stream} If cb is falsy, return a stream 237 | */ 238 | http(options) { 239 | const uri = options.pathname 240 | const requestOptions = Object.assign( 241 | { 242 | method: options.method, 243 | uri, 244 | body: options.body, 245 | json: 'json' in options ? Boolean(options.json) : true, 246 | qs: options.parameters || options.qs, 247 | headers: options.headers 248 | }, 249 | this.requestOptions 250 | ) 251 | 252 | if (options.timeout) { 253 | requestOptions.timeout = options.timeout 254 | } 255 | 256 | if (options.noAuth) { 257 | delete requestOptions.auth 258 | } 259 | 260 | if (options.stream) { 261 | return request(requestOptions) 262 | } 263 | 264 | return new Promise((resolve, reject) => { 265 | this._request(requestOptions, (err, res) => { 266 | if (err) { 267 | return reject(err) 268 | } 269 | if (res.statusCode < 200 || res.statusCode > 299) { 270 | const error = new ErrorWithCode( 271 | res.body.message || res.body, 272 | res.statusCode, 273 | res.statusCode 274 | ) 275 | return reject(error) 276 | } 277 | return resolve(res) 278 | }) 279 | }) 280 | } 281 | } 282 | -------------------------------------------------------------------------------- /lib/logger/index.ts: -------------------------------------------------------------------------------- 1 | export interface ILogger { 2 | debug(...args): void 3 | info(...args): void 4 | log(...args): void 5 | warn(...args): void 6 | error(...args): void 7 | } 8 | 9 | export enum LogLevel { 10 | debug = 1, 11 | info = 2, 12 | warn = 3, 13 | error = 4, 14 | none = 5 15 | } 16 | 17 | export interface IConsole { 18 | log(message?: any, ...optionalParams: any[]): void 19 | } 20 | 21 | export default class Logger implements ILogger { 22 | private module: string 23 | private logLevel: LogLevel = LogLevel.info 24 | private writer: IConsole 25 | 26 | constructor( 27 | moduleName: string, 28 | logLevel?: LogLevel, 29 | writer: IConsole = console 30 | ) { 31 | this.writer = writer 32 | this.module = moduleName 33 | const isNull = (item): boolean => { 34 | return typeof item === 'undefined' || item === null 35 | } 36 | if (isNull(logLevel) && process.env.LOG_LEVEL) { 37 | const stringLevel = process.env.LOG_LEVEL.toString().toLowerCase() 38 | const level = LogLevel[stringLevel] 39 | if (isNull(level)) { 40 | throw new Error(`unknown log level: ${stringLevel}`) 41 | } 42 | this.logLevel = level 43 | } else { 44 | this.logLevel = logLevel || LogLevel.info 45 | } 46 | 47 | this.info = this.info.bind(this) 48 | } 49 | 50 | debug(...args): void { 51 | this.writeLog.bind(this)(LogLevel.debug, args) 52 | } 53 | info(...args): void { 54 | this.writeLog.bind(this)(LogLevel.info, args) 55 | } 56 | log(...args): void { 57 | this.writeLog.bind(this)(LogLevel.info, args) 58 | } 59 | warn(...args): void { 60 | this.writeLog.bind(this)(LogLevel.warn, args) 61 | } 62 | error(...args): void { 63 | this.writeLog.bind(this)(LogLevel.error, args) 64 | } 65 | 66 | private writeLog(level: LogLevel, args: any[]): void { 67 | if (level < this.logLevel) { 68 | return 69 | } 70 | this.logJson(level, args) 71 | } 72 | 73 | private logJson(level: LogLevel, args: any[]): void { 74 | const stringMessages: string[] = [] 75 | const payload: any = { 76 | timestamp: new Date().toISOString(), 77 | level: LogLevel[level], 78 | module: this.module, 79 | message: '' 80 | } 81 | 82 | args.forEach((arg) => { 83 | if (typeof arg === 'undefined' || arg === null) { 84 | return 85 | } 86 | if (typeof arg === 'object') { 87 | if (arg instanceof Error) { 88 | payload.error = { 89 | name: arg.name, 90 | message: arg.message 91 | } as any 92 | if (arg.stack) { 93 | const lines = arg.stack.split('\n', 2) 94 | payload.error.stack = lines[lines.length - 1].trim() 95 | } 96 | } else { 97 | /* If the argument is a type object, then loop over its keys 98 | and add them to the payload, unless they're a reserved key 99 | such as timestamp, module, level or message 100 | */ 101 | Object.assign(payload, arg) 102 | } 103 | return 104 | } 105 | stringMessages.push(arg) 106 | }) 107 | payload.message = stringMessages.join(' ') 108 | 109 | if (payload.message === '' && payload.error) { 110 | payload.message = payload.error.message 111 | } 112 | 113 | this.writer.log(JSON.stringify(payload)) 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /lib/tester/index.ts: -------------------------------------------------------------------------------- 1 | import { Got } from 'got/dist/source' 2 | import { IDiscovery, IAgent } from 'lib/discovery' 3 | import { PlainResponse } from 'got/dist/source/core' 4 | import { IMetrics } from 'lib/apps/agent/metrics' 5 | import { IUDPPingResult } from 'lib/udp/client' 6 | import { IConfig } from 'lib/config' 7 | import Logger, { ILogger } from 'lib/logger' 8 | import * as dns from 'dns' 9 | import { IUdpClientFactory as IUDPClientFactory } from 'lib/udp/clientFactory' 10 | 11 | export interface ITester { 12 | start() 13 | stop() 14 | runUDPTests(agents: IAgent[]): Promise 15 | runTCPTests(agents: IAgent[]): Promise 16 | runDNSTests(): Promise 17 | } 18 | 19 | interface ITestResult { 20 | source: IAgent 21 | destination: IAgent 22 | result: 'pass' | 'fail' 23 | } 24 | 25 | export interface IDNSTestResult { 26 | source: IAgent 27 | host: string 28 | duration: number 29 | result: 'pass' | 'fail' 30 | } 31 | 32 | export interface IUDPTestResult extends ITestResult { 33 | timings?: IUDPPingResult 34 | } 35 | 36 | export interface ITCPTestResult extends ITestResult { 37 | timings?: PlainResponse['timings'] 38 | } 39 | 40 | export default class Tester implements ITester { 41 | private got: Got 42 | private discovery: IDiscovery 43 | private logger: ILogger = new Logger('tester') 44 | private metrics: IMetrics 45 | private me: IAgent 46 | private running = false 47 | private config: IConfig 48 | private resolver = new dns.promises.Resolver() 49 | private readonly udpClientFactory: IUDPClientFactory 50 | 51 | constructor( 52 | config: IConfig, 53 | got: Got, 54 | discovery: IDiscovery, 55 | metrics: IMetrics, 56 | me: IAgent, 57 | udpClientFactory: IUDPClientFactory 58 | ) { 59 | this.got = got 60 | this.discovery = discovery 61 | this.metrics = metrics 62 | this.me = me 63 | this.config = config 64 | this.udpClientFactory = udpClientFactory 65 | } 66 | 67 | public async start(): Promise { 68 | const delay = (ms: number) => { 69 | return new Promise((resolve) => setTimeout(resolve, ms)) 70 | } 71 | 72 | const jitter = () => { 73 | const rand = Math.random() * (500 - 50) 74 | return Math.floor(rand + 100) 75 | } 76 | 77 | this.running = true 78 | let agents = await this.discovery.agents() 79 | const agentUpdateLoop = async () => { 80 | while (this.running) { 81 | agents = await this.discovery.agents() 82 | await delay(5000) 83 | } 84 | } 85 | const tcpEventLoop = async () => { 86 | while (this.running) { 87 | this.metrics.resetTCPTestResults() 88 | await this.runTCPTests(agents) 89 | await delay(this.config.testConfig.tcp.interval + jitter()) 90 | } 91 | } 92 | const udpEventLoop = async () => { 93 | while (this.running) { 94 | this.metrics.resetUDPTestResults() 95 | await this.runUDPTests(agents) 96 | await delay(this.config.testConfig.udp.interval + jitter()) 97 | } 98 | } 99 | const dnsEventLoop = async () => { 100 | while (this.running) { 101 | await this.runDNSTests() 102 | await delay(this.config.testConfig.dns.interval + jitter()) 103 | } 104 | } 105 | agentUpdateLoop() 106 | tcpEventLoop() 107 | udpEventLoop() 108 | dnsEventLoop() 109 | } 110 | 111 | public async stop(): Promise { 112 | this.running = false 113 | } 114 | 115 | public async runDNSTests(): Promise { 116 | const promises = this.config.testConfig.dns.hosts.map( 117 | async (host): Promise => { 118 | const hrstart = process.hrtime() 119 | try { 120 | const result = await this.resolver.resolve4(host) 121 | const hrend = process.hrtime(hrstart) 122 | const mapped: IDNSTestResult = { 123 | source: this.me, 124 | host, 125 | duration: hrend[1] / 1000000, 126 | result: result && result.length > 0 ? 'pass' : 'fail' 127 | } 128 | this.metrics.handleDNSTestResult(mapped) 129 | return mapped 130 | } catch (ex) { 131 | this.logger.error(`dns test for ${host} failed`, ex) 132 | const hrend = process.hrtime(hrstart) 133 | const mapped: IDNSTestResult = { 134 | source: this.me, 135 | host, 136 | duration: hrend[1] / 1000000, 137 | result: 'fail' 138 | } 139 | this.metrics.handleDNSTestResult(mapped) 140 | return mapped 141 | } 142 | } 143 | ) 144 | const result = await Promise.allSettled(promises) 145 | return result 146 | .filter((r) => r.status === 'fulfilled') 147 | .map((i) => (i as PromiseFulfilledResult).value) 148 | } 149 | 150 | public async runUDPTests(agents: IAgent[]): Promise { 151 | const results: IUDPTestResult[] = [] 152 | this.udpClientFactory.generateClientsForAgents(agents) 153 | 154 | const testAgent = async (agent: IAgent): Promise => { 155 | const client = this.udpClientFactory.clientFor(agent) 156 | try { 157 | const result = await client.ping( 158 | this.config.testConfig.udp.timeout, 159 | this.config.testConfig.udp.packets 160 | ) 161 | if (result.loss > 0) { 162 | this.logger.warn('packet loss detected', result) 163 | } 164 | const testResult: IUDPTestResult = { 165 | source: this.me, 166 | destination: agent, 167 | timings: result, 168 | result: result.loss > 0 ? 'fail' : 'pass' 169 | } 170 | results.push(testResult) 171 | this.metrics.handleUDPTestResult(testResult) 172 | } catch (ex) { 173 | this.logger.error('Failed to execute UDP test', ex) 174 | const testResult: IUDPTestResult = { 175 | source: this.me, 176 | destination: agent, 177 | result: 'fail' 178 | } 179 | results.push(testResult) 180 | this.metrics.handleUDPTestResult(testResult) 181 | } 182 | } 183 | const promises = agents.map(testAgent) 184 | await Promise.allSettled(promises) 185 | 186 | return results 187 | } 188 | 189 | public async runTCPTests(agents: IAgent[]): Promise { 190 | const testAgent = async (agent: IAgent): Promise => { 191 | try { 192 | const url = `http://${agent.ip}:${this.config.port}/readiness` 193 | const result = await this.got(url, { 194 | timeout: this.config.testConfig.tcp.timeout 195 | }) 196 | const mappedResult: ITCPTestResult = { 197 | source: this.me, 198 | destination: agent, 199 | timings: result.timings, 200 | result: result.statusCode === 200 ? 'pass' : 'fail' 201 | } 202 | this.metrics.handleTCPTestResult(mappedResult) 203 | return mappedResult 204 | } catch (ex) { 205 | this.logger.warn( 206 | `test failed`, 207 | { 208 | source: this.me, 209 | destination: agent 210 | }, 211 | ex 212 | ) 213 | const failResult: ITCPTestResult = { 214 | source: this.me, 215 | destination: agent, 216 | result: 'fail' 217 | } 218 | this.metrics.handleTCPTestResult(failResult) 219 | return failResult 220 | } 221 | } 222 | const promises = agents.map(testAgent) 223 | const result = await Promise.allSettled(promises) 224 | return result 225 | .filter((r) => r.status === 'fulfilled') 226 | .map((i) => (i as PromiseFulfilledResult).value) 227 | } 228 | } 229 | -------------------------------------------------------------------------------- /lib/udp/client.ts: -------------------------------------------------------------------------------- 1 | import * as udp from 'dgram' 2 | import Logger, { ILogger } from 'lib/logger' 3 | 4 | export interface IUDPClient { 5 | ping(timeoutMs: number, times: number): Promise 6 | destroy(): void 7 | } 8 | 9 | export interface IUDPSinglePingResult { 10 | success: boolean 11 | duration: number 12 | } 13 | 14 | export interface IUDPPingResult { 15 | results: IUDPSinglePingResult[] 16 | success: boolean 17 | min: number 18 | max: number 19 | average: number 20 | duration: number 21 | variance: number 22 | loss: number 23 | } 24 | 25 | const delay = (ms: number) => { 26 | return new Promise((resolve) => setTimeout(resolve, ms)) 27 | } 28 | 29 | export default class UDPClient implements IUDPClient { 30 | private logger: ILogger = new Logger('udp-client') 31 | private host: string 32 | private port: number 33 | private client: udp.Socket 34 | 35 | constructor(host: string, port: number) { 36 | this.host = host 37 | this.port = port 38 | this.client = udp.createSocket('udp4') 39 | } 40 | 41 | public destroy() { 42 | try { 43 | this.client.close() 44 | } catch (ex) { 45 | this.logger.error('failed to close client', ex) 46 | } 47 | } 48 | 49 | public async ping(timeoutMs: number, times: number): Promise { 50 | const results: IUDPSinglePingResult[] = [] 51 | const sendPing = () => { 52 | return new Promise((resolve, reject) => { 53 | const timeout = setTimeout(() => { 54 | this.logger.error('Timed out waiting for udp response') 55 | reject(new Error('TIMEOUT')) 56 | }, timeoutMs) 57 | 58 | const data = Buffer.from('ping') 59 | let hrstart = process.hrtime() 60 | 61 | this.client.on('message', () => { 62 | const hrend = process.hrtime(hrstart) 63 | clearTimeout(timeout) 64 | resolve(hrend[1] / 1000000) 65 | }) 66 | 67 | hrstart = process.hrtime() 68 | this.client.send(data, this.port, this.host, (error) => { 69 | if (error) { 70 | this.logger.error('failed to send udp packet', error) 71 | reject(error) 72 | } 73 | }) 74 | }) 75 | .then((duration: number) => { 76 | return { 77 | success: true, 78 | duration 79 | } 80 | }) 81 | .catch(() => { 82 | return { 83 | success: false, 84 | duration: timeoutMs 85 | } 86 | }) 87 | .finally(() => { 88 | this.client.removeAllListeners() 89 | }) 90 | } 91 | 92 | try { 93 | // send one noddy packet to warm code path and reduce variance 94 | // due to the first test being slightly slower 95 | await sendPing().catch() 96 | for (let i = 0; i < times; i += 1) { 97 | await sendPing().then((result) => { 98 | results.push(result) 99 | }) 100 | await delay(50) 101 | } 102 | } catch (ex) { 103 | this.logger.error('failed to ping', ex) 104 | } finally { 105 | this.client.removeAllListeners() 106 | } 107 | 108 | const durations = results.map((result) => result.duration).sort() 109 | const returnValue = { 110 | results, 111 | success: true, 112 | loss: 113 | (results.filter((result) => !result.success).length / results.length) * 114 | 100, 115 | min: durations[0] as number, 116 | max: durations.reverse()[0] as number, 117 | average: durations.reduce((a, b) => a + b, 0) / durations.length, 118 | variance: 0, 119 | duration: durations.reduce((a, b) => a + b, 0) 120 | } 121 | returnValue.success = returnValue.loss === 0 122 | returnValue.variance = returnValue.max - returnValue.min 123 | return returnValue 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /lib/udp/clientFactory.ts: -------------------------------------------------------------------------------- 1 | import { IAgent } from 'lib/discovery' 2 | import UDPClient, { IUDPClient } from 'lib/udp/client' 3 | import Logger from 'lib/logger' 4 | import { IConfig } from 'lib/config' 5 | 6 | export interface IUdpClientFactory { 7 | generateClientsForAgents(agents: IAgent[]): void 8 | clientFor(agent: IAgent): IUDPClient 9 | } 10 | 11 | export default class UDPClientFactory implements IUdpClientFactory { 12 | private clients: { [key: string]: IUDPClient } = {} 13 | private readonly logger = new Logger('udp-client-factory') 14 | private readonly config: IConfig 15 | constructor(config: IConfig) { 16 | this.config = config 17 | } 18 | 19 | public generateClientsForAgents(agents: IAgent[]) { 20 | agents.forEach((agent) => { 21 | if (!this.clients[agent.ip]) { 22 | this.logger.info(`new udp client created for ${agent.ip}`) 23 | this.clients[agent.ip] = new UDPClient(agent.ip, this.config.port) 24 | } 25 | }) 26 | Object.keys(this.clients).forEach((ip) => { 27 | const agent = agents.find((a) => a.ip === ip) 28 | if (!agent) { 29 | this.logger.info(`udp client removed for ${ip}`) 30 | this.clients[ip].destroy() 31 | delete this.clients[ip] 32 | } 33 | }) 34 | } 35 | 36 | public clientFor(agent: IAgent): IUDPClient { 37 | return this.clients[agent.ip] 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /lib/udp/server.ts: -------------------------------------------------------------------------------- 1 | import * as udp from 'dgram' 2 | import { IConfig } from 'lib/config' 3 | import Logger, { ILogger } from 'lib/logger' 4 | 5 | export interface IUDPServer { 6 | start(): void 7 | stop(): void 8 | } 9 | 10 | export default class UDPServer implements IUDPServer { 11 | private server: udp.Socket 12 | private logger: ILogger = new Logger('udp-server') 13 | private config: IConfig 14 | 15 | constructor(config: IConfig) { 16 | this.config = config 17 | this.server = udp.createSocket('udp4') 18 | 19 | this.server.on('error', (error) => { 20 | throw error 21 | }) 22 | 23 | this.server.on('message', (msg, info) => { 24 | this.server.send(msg, info.port, info.address, (error) => { 25 | if (error) { 26 | this.logger.error('UDP reply failed', error) 27 | } 28 | }) 29 | }) 30 | } 31 | 32 | /* eslint max-statements: off */ 33 | public async start(): Promise { 34 | return new Promise((resolve) => { 35 | this.logger.info(`starting udp server on port ${this.config.port}`) 36 | this.server.bind(this.config.port, '0.0.0.0', () => { 37 | this.logger.info(`udp server started`) 38 | resolve() 39 | }) 40 | }) 41 | } 42 | 43 | public async stop(): Promise { 44 | this.logger.info('stopping udp server') 45 | return new Promise((resolve) => { 46 | this.server.close(() => { 47 | this.logger.info('udp server stopped') 48 | resolve() 49 | }) 50 | }) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /lib/web-server/index.ts: -------------------------------------------------------------------------------- 1 | import * as express from 'express' 2 | import * as stoppable from 'stoppable' 3 | import { IConfig } from 'lib/config' 4 | import Logger, { ILogger } from 'lib/logger' 5 | 6 | export type ApplyRoutesHandler = (app: express.Application) => Promise 7 | export interface IWebServer { 8 | start(handlerInit: ApplyRoutesHandler): void 9 | stop(): void 10 | } 11 | export interface IRoutes { 12 | /** Apply the routes to the express server */ 13 | applyRoutes(app: express.Application, controller): void 14 | } 15 | 16 | export default class WebServer implements IWebServer { 17 | private http 18 | private logger: ILogger 19 | private config: IConfig 20 | 21 | constructor(config: IConfig) { 22 | this.config = config 23 | this.logger = new Logger('web-server') 24 | } 25 | 26 | /* eslint max-statements: off */ 27 | public async start(handlerInit: ApplyRoutesHandler): Promise { 28 | this.logger.info(`starting web server on port ${this.config.port}`) 29 | 30 | const app: express.Application = express() 31 | app.set('etag', false) 32 | app.disable('etag') 33 | app.disable('x-powered-by') 34 | 35 | await handlerInit(app) 36 | return new Promise((resolve) => { 37 | const server = app.listen(this.config.port, () => { 38 | this.logger.info(`web server started`) 39 | }) 40 | this.http = stoppable(server, 10000) 41 | server.keepAliveTimeout = 1000 * (60 * 6) 42 | server.on('connection', (socket) => { 43 | // Disable Nagles 44 | socket.setNoDelay(true) 45 | socket.setTimeout(600 * 60 * 1000) 46 | }) 47 | resolve() 48 | }) 49 | } 50 | 51 | public async stop(): Promise { 52 | const logger = this.logger 53 | 54 | return new Promise((resolve) => { 55 | logger.info('stopping web server') 56 | this.http.stop(() => { 57 | logger.info('web server stopped') 58 | resolve() 59 | }) 60 | }) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kconmon", 3 | "description": "A project for testing & monitor kubernetes nodes with TCP and UDP", 4 | "version": "0.0.0", 5 | "homepage": "https://github.com/Stono/kconmon", 6 | "author": { 7 | "name": "Karl Stoney", 8 | "email": "me@karlstoney.com" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "git://github.com/Stono/kconmon.git" 13 | }, 14 | "bugs": { 15 | "url": "https://github.com/Stono/kconmon/issues" 16 | }, 17 | "main": "lib/index.ts", 18 | "engines": { 19 | "node": ">= 14.0.0" 20 | }, 21 | "scripts": { 22 | "build": "./node_modules/grunt/bin/grunt build", 23 | "test": "./node_modules/grunt/bin/grunt test" 24 | }, 25 | "dependencies": { 26 | "express": "4.17.1", 27 | "got": "11.5.1", 28 | "prom-client": "12.0.0", 29 | "stoppable": "1.1.0", 30 | "tsconfig-paths": "3.9.0", 31 | "kubernetes-client": "9.0.0", 32 | "kubernetes-types": "1.17.0-beta.1", 33 | "ts-xor": "1.0.8" 34 | }, 35 | "devDependencies": { 36 | "@types/express": "4.17.7", 37 | "@types/json-patch": "0.0.30", 38 | "@types/mocha": "8.0.0", 39 | "@types/node": "14.0.26", 40 | "@typescript-eslint/eslint-plugin": "3.7.0", 41 | "@typescript-eslint/parser": "3.7.0", 42 | "eslint": "7.5.0", 43 | "eslint-config-prettier": "6.11.0", 44 | "eslint-plugin-mocha": "7.0.1", 45 | "eslint-plugin-prettier": "3.1.4", 46 | "grunt": "1.2.1", 47 | "grunt-contrib-copy": "1.0.0", 48 | "grunt-contrib-watch": "1.1.0", 49 | "grunt-env": "1.0.1", 50 | "grunt-eslint": "23.0.0", 51 | "grunt-exec": "3.0.0", 52 | "grunt-mocha-test": "0.13.3", 53 | "grunt-ts": "6.0.0-beta.22", 54 | "mocha": "8.0.1", 55 | "prettier": "2.0.5", 56 | "should": "13.2.3", 57 | "testdouble": "3.16.1", 58 | "ts-node": "8.10.2", 59 | "typescript": "3.9.7" 60 | }, 61 | "keywords": [], 62 | "license": "Apache-2.0" 63 | } 64 | -------------------------------------------------------------------------------- /samples/ping.ts: -------------------------------------------------------------------------------- 1 | import * as path from 'path' 2 | import * as tsConfigPaths from 'tsconfig-paths/lib' 3 | const baseUrl = path.join(__dirname, '../') 4 | 5 | tsConfigPaths.register({ 6 | baseUrl, 7 | paths: {} 8 | }) 9 | 10 | import UDPServer from 'lib/udp/server' 11 | import UDPClient from 'lib/udp/client' 12 | import config from 'lib/config' 13 | const server = new UDPServer(config) 14 | 15 | server.start() 16 | 17 | const client = new UDPClient() 18 | ;(async () => { 19 | console.log(await client.ping('127.0.0.1', 8081, 1, 20)) 20 | })() 21 | -------------------------------------------------------------------------------- /screenshots/grafana.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stono/kconmon/4fcf38ecf67ad22425981a35375af5665e260021/screenshots/grafana.png -------------------------------------------------------------------------------- /test/discovery/kubernetes.test.ts: -------------------------------------------------------------------------------- 1 | import * as td from 'testdouble' 2 | import { IConfig } from 'lib/config' 3 | import KubernetesDiscovery, { 4 | IKubernetesDiscovery 5 | } from 'lib/discovery/kubernetes' 6 | import IKubernetesClient from 'lib/kubernetes/client' 7 | import { Models } from 'lib/kubernetes/models' 8 | import * as should from 'should' 9 | 10 | describe('Discovery: Kubernetes', () => { 11 | let sut: IKubernetesDiscovery 12 | let client: IKubernetesClient 13 | before(() => { 14 | const config = td.object() 15 | config.namespace = 'test' 16 | config.port = 8080 17 | config.failureDomainLabel = 'failure-domain.beta.kubernetes.io/zone' 18 | client = td.object() 19 | sut = new KubernetesDiscovery(config, client) 20 | }) 21 | 22 | it('should get the details of a single agent', async () => { 23 | const pod = td.object() 24 | pod.metadata = td.object() 25 | pod.metadata.name = 'bacon-pod' 26 | pod.spec.nodeName = 'bacon-node' 27 | 28 | const podStatusAsAny = pod.status as any 29 | podStatusAsAny.podIP = '1.2.3.4' 30 | 31 | const node = td.object() 32 | node.metadata = td.object() 33 | node.metadata.name = 'bacon-node' 34 | node.metadata.labels = { 35 | 'failure-domain.beta.kubernetes.io/zone': 'europe-west4-a' 36 | } 37 | 38 | td.when(client.select('v1', 'Node')).thenResolve([node]) 39 | td.when( 40 | client.get('v1', 'Pod', 'test', 'kconmon-g88mt') 41 | ).thenResolve(pod) 42 | 43 | await sut.reconcileNodes() 44 | const agent = await sut.agent('kconmon-g88mt') 45 | should(agent).eql({ 46 | name: 'bacon-pod', 47 | nodeName: 'bacon-node', 48 | ip: '1.2.3.4', 49 | zone: 'europe-west4-a' 50 | }) 51 | }) 52 | }) 53 | -------------------------------------------------------------------------------- /test/discovery/service.test.ts: -------------------------------------------------------------------------------- 1 | import { IDiscovery } from 'lib/discovery' 2 | import ServiceDiscovery from 'lib/discovery/service' 3 | import * as td from 'testdouble' 4 | import { IConfig } from 'lib/config' 5 | import { Got } from 'got/dist/source' 6 | import TestHelpers from 'test/helpers' 7 | import * as should from 'should' 8 | 9 | describe('Discovery: Service', () => { 10 | let sut: IDiscovery 11 | let got: Got 12 | before(() => { 13 | const config = td.object() 14 | config.namespace = 'test' 15 | config.port = 8080 16 | got = td.function() 17 | sut = new ServiceDiscovery(config, got) 18 | }) 19 | 20 | it('should get the current list of agents', async () => { 21 | const httpOptions = { 22 | timeout: 500, 23 | responseType: 'json', 24 | retry: { limit: 2 } 25 | } 26 | const response = { 27 | statusCode: 200, 28 | body: TestHelpers.loadSample('agents') 29 | } 30 | td.when( 31 | got( 32 | 'http://controller.test.svc.cluster.local./agents', 33 | httpOptions as any 34 | ) 35 | ).thenResolve(response) 36 | const agents = await sut.agents() 37 | should(agents).eql(response.body) 38 | }) 39 | 40 | it('should get the details of a single agent', async () => { 41 | const httpOptions = { 42 | timeout: 500, 43 | responseType: 'json', 44 | retry: { limit: 2 } 45 | } 46 | const response = { 47 | statusCode: 200, 48 | body: TestHelpers.loadSample('agent') 49 | } 50 | td.when( 51 | got( 52 | 'http://controller.test.svc.cluster.local./agent/kconmon-g88mt', 53 | httpOptions as any 54 | ) 55 | ).thenResolve(response) 56 | const agents = await sut.agent('kconmon-g88mt') 57 | should(agents).eql(response.body) 58 | }) 59 | }) 60 | -------------------------------------------------------------------------------- /test/helpers.ts: -------------------------------------------------------------------------------- 1 | import * as path from 'path' 2 | import * as fs from 'fs' 3 | export default class TestHelpers { 4 | public static loadSample(file: string): any { 5 | const location = path.join(__dirname, 'samples/', `${file}.json`) 6 | return JSON.parse(fs.readFileSync(location).toString()) 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /test/kubernetes/client.test.ts: -------------------------------------------------------------------------------- 1 | import * as should from 'should' 2 | import Kubernetes, { 3 | IKubernetes, 4 | IKubernetesWatchEvent, 5 | KubernetesEventType, 6 | IGodaddyClient, 7 | IGodaddyWatch 8 | } from '../../lib/kubernetes/client' 9 | import { Models } from '../../lib/kubernetes/models' 10 | import * as td from 'testdouble' 11 | import TestHelpers from '../helpers' 12 | import { EventEmitter } from 'events' 13 | import { Readable } from 'stream' 14 | 15 | /* eslint no-invalid-this: off */ 16 | describe('Really complicated watch logic', () => { 17 | let client: IKubernetes 18 | let godaddy: IGodaddyClient 19 | let watch: IGodaddyWatch 20 | 21 | beforeEach(() => { 22 | godaddy = td.object() 23 | client = new Kubernetes({ 24 | client: godaddy 25 | }) 26 | 27 | watch = td.object() 28 | godaddy.api = { 29 | v1: { 30 | watch: { 31 | node: watch 32 | } 33 | } 34 | } 35 | }) 36 | 37 | it('should have a timeout of 60s', (done) => { 38 | td.when(watch.getObjectStream(td.matchers.anything())).thenDo((options) => { 39 | should(options.timeout).eql(60000) 40 | done() 41 | return td.object() 42 | }) 43 | client.watch('v1', 'Node', () => {}) 44 | client.start() 45 | }) 46 | 47 | it('should initially set up a watch, with no resourceVersion', (done) => { 48 | td.when(watch.getObjectStream(td.matchers.anything())).thenDo((options) => { 49 | should(options.qs).eql({}) 50 | done() 51 | return td.object() 52 | }) 53 | client.watch('v1', 'Node', () => {}) 54 | client.start() 55 | }) 56 | 57 | it('if we only see replay events, should resume from unset', (done) => { 58 | const readable = td.object() 59 | const emitter = new EventEmitter() 60 | 61 | let counter = 0 62 | td.when(watch.getObjectStream({ timeout: 60000, qs: {} })).thenDo(() => { 63 | counter += 1 64 | if (counter === 2) { 65 | done() 66 | } 67 | return readable 68 | }) 69 | 70 | const handleNodeEvent = () => {} 71 | client.watch( 72 | 'v1', 73 | 'Node', 74 | handleNodeEvent, 75 | [KubernetesEventType.ALL], 76 | null 77 | ) 78 | td.when(readable.on(td.matchers.anything(), td.matchers.anything())).thenDo( 79 | (event, handler) => { 80 | emitter.on(event, handler) 81 | } 82 | ) 83 | ;(async () => { 84 | await client.start() 85 | const incrementedModel: IKubernetesWatchEvent = TestHelpers.loadSample( 86 | 'node' 87 | ) 88 | incrementedModel.object.metadata.resourceVersion = '226835920' 89 | incrementedModel.type = 'ADDED' 90 | emitter.emit('data', incrementedModel) 91 | emitter.emit('end') 92 | })() 93 | }) 94 | 95 | it('if we see both replay and real events, should resume from the last real resourceVersion we saw', (done) => { 96 | const readable = td.object() 97 | const emitter = new EventEmitter() 98 | const model: IKubernetesWatchEvent = TestHelpers.loadSample( 99 | 'node' 100 | ) 101 | 102 | td.when(watch.getObjectStream({ timeout: 60000, qs: {} })).thenDo(() => { 103 | return readable 104 | }) 105 | 106 | td.when( 107 | watch.getObjectStream({ 108 | timeout: 60000, 109 | qs: { resourceVersion: '226835919' } 110 | }) 111 | ).thenDo(() => { 112 | done() 113 | return readable 114 | }) 115 | 116 | const handleNodeEvent = () => {} 117 | client.watch('v1', 'Node', handleNodeEvent) 118 | td.when(readable.on(td.matchers.anything(), td.matchers.anything())).thenDo( 119 | (event, handler) => { 120 | emitter.on(event, handler) 121 | } 122 | ) 123 | ;(async () => { 124 | await client.start() 125 | 126 | const syntheticAddedEvent: IKubernetesWatchEvent = TestHelpers.loadSample( 127 | 'node' 128 | ) 129 | syntheticAddedEvent.object.metadata.resourceVersion = '226835936' 130 | syntheticAddedEvent.type = 'ADDED' 131 | emitter.emit('data', syntheticAddedEvent) 132 | emitter.emit('data', model) 133 | emitter.emit('end') 134 | })() 135 | }) 136 | 137 | it('if the last version we saw is gone, we should resume from unset', (done) => { 138 | const readable = td.object() 139 | const emitter = new EventEmitter() 140 | const model: IKubernetesWatchEvent = TestHelpers.loadSample( 141 | 'node' 142 | ) 143 | 144 | let counter = 0 145 | td.when(watch.getObjectStream({ timeout: 60000, qs: {} })).thenDo(() => { 146 | counter += 1 147 | if (counter === 2) { 148 | td.verify(watch.getObjectStream(td.matchers.anything()), { times: 3 }) 149 | done() 150 | } 151 | return readable 152 | }) 153 | 154 | td.when( 155 | watch.getObjectStream({ 156 | timeout: 60000, 157 | qs: { resourceVersion: '226835919' } 158 | }) 159 | ).thenDo(() => { 160 | emitter.emit('data', { 161 | type: 'ERROR', 162 | object: { 163 | metadata: {}, 164 | code: 410 165 | } 166 | }) 167 | emitter.emit('end') 168 | return readable 169 | }) 170 | 171 | const handleNodeEvent = () => {} 172 | client.watch('v1', 'Node', handleNodeEvent) 173 | td.when(readable.on(td.matchers.anything(), td.matchers.anything())).thenDo( 174 | (event, handler) => { 175 | emitter.on(event, handler) 176 | } 177 | ) 178 | ;(async () => { 179 | await client.start() 180 | emitter.emit('data', model) 181 | emitter.emit('end') 182 | })() 183 | }) 184 | }) 185 | -------------------------------------------------------------------------------- /test/logger/logger.test.ts: -------------------------------------------------------------------------------- 1 | import Logger, { LogLevel, IConsole, ILogger } from '../../lib/logger' 2 | import * as should from 'should' 3 | 4 | describe('Logger', () => { 5 | /* eslint init-declarations: off */ 6 | let lastMessage: any | undefined, logger: ILogger 7 | let messageProcessor 8 | before(() => { 9 | const mockConsole: IConsole = { 10 | log: (msg) => { 11 | messageProcessor(msg) 12 | } 13 | } 14 | logger = new Logger('test-logger', LogLevel.debug, mockConsole) 15 | }) 16 | 17 | it('should throw an error if you use an unknown log level with the environment variable', () => { 18 | process.env.LOG_LEVEL = 'bacon' 19 | try { 20 | const sut = new Logger('test-logger') 21 | should(sut).not.be.empty() 22 | } catch (ex) { 23 | should(ex.message).eql('unknown log level: bacon') 24 | } 25 | process.env.LOG_LEVEL = 'none' 26 | }) 27 | 28 | it('should accept valid log levels via the environment variable', () => { 29 | const sut = new Logger('test-logger', LogLevel.info) 30 | should(sut).not.be.empty() 31 | }) 32 | 33 | describe('Logging', () => { 34 | before(() => { 35 | messageProcessor = (msg: string): void => { 36 | lastMessage = JSON.parse(msg) 37 | } 38 | }) 39 | const commonTests = (level: string, message = 'testing'): void => { 40 | should(lastMessage.level).eql(level) 41 | should(lastMessage.timestamp).not.be.empty() 42 | should(lastMessage.module).eql('test-logger') 43 | should(lastMessage.message).eql(message) 44 | } 45 | 46 | it('should handle null', () => { 47 | logger.info(null) 48 | should(lastMessage.message).eql('') 49 | }) 50 | 51 | it('should write debug messages', () => { 52 | logger.debug('testing') 53 | commonTests('debug') 54 | }) 55 | 56 | it('if just an exception is passed, it should take the error message and put it on the message field', () => { 57 | logger.debug(new Error('bang')) 58 | should(lastMessage.message).eql('bang') 59 | }) 60 | 61 | it('should write info messages', () => { 62 | logger.info('testing') 63 | commonTests('info') 64 | }) 65 | 66 | it('should write log as info messages', () => { 67 | logger.log('testing') 68 | commonTests('info') 69 | }) 70 | 71 | it('should write warn messages', () => { 72 | logger.warn('testing') 73 | commonTests('warn') 74 | }) 75 | 76 | it('should write error messages', () => { 77 | logger.error('testing') 78 | commonTests('error') 79 | }) 80 | 81 | it('should map complex objects to properties', () => { 82 | logger.log('testing', { 83 | some: 'object', 84 | with: 'fields' 85 | }) 86 | commonTests('info') 87 | should(lastMessage.some).eql('object') 88 | should(lastMessage.with).eql('fields') 89 | }) 90 | 91 | it('should map multiple strings into a single message', () => { 92 | logger.log( 93 | 'testing', 94 | { 95 | some: 'object', 96 | with: 'fields' 97 | }, 98 | 'some', 99 | 'string' 100 | ) 101 | commonTests('info', 'testing some string') 102 | }) 103 | 104 | it('should map error objects to properties', () => { 105 | logger.log('testing', new Error('boom')) 106 | commonTests('info') 107 | should(lastMessage.error.message).eql('boom') 108 | should(lastMessage.error.name).eql('Error') 109 | /* eslint no-unused-expressions: off */ 110 | should(lastMessage.error.stack).not.be.empty 111 | }) 112 | }) 113 | }) 114 | -------------------------------------------------------------------------------- /test/samples/agent.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kconmon-g88mt", 3 | "nodeName": "gke-pool-20191114-b473a77c-8n7l", 4 | "ip": "10.206.0.123", 5 | "zone": "europe-west4-a" 6 | } 7 | -------------------------------------------------------------------------------- /test/samples/agents.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "kconmon-g88mt", 4 | "nodeName": "gke-pool-20191114-b473a77c-8n7l", 5 | "ip": "10.206.0.123", 6 | "zone": "europe-west4-a" 7 | }, 8 | { 9 | "name": "kconmon-2qjfl", 10 | "nodeName": "gke-pool-20191114-19632ad8-n7lh", 11 | "ip": "10.206.4.69", 12 | "zone": "europe-west4-b" 13 | }, 14 | { 15 | "name": "kconmon-2qpvp", 16 | "nodeName": "gke-pool-20191114-19632ad8-de5d", 17 | "ip": "10.206.3.109", 18 | "zone": "europe-west4-c" 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /test/samples/node.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "UPDATED", 3 | "object": { 4 | "apiVersion": "v1", 5 | "kind": "Node", 6 | "metadata": { 7 | "annotations": { 8 | "container.googleapis.com/instance_id": "7484356131799260764", 9 | "node.alpha.kubernetes.io/ttl": "0", 10 | "projectcalico.org/IPv4IPIPTunnelAddr": "10.206.5.1", 11 | "volumes.kubernetes.io/controller-managed-attach-detach": "true" 12 | }, 13 | "creationTimestamp": "2020-05-20T07:48:23Z", 14 | "labels": { 15 | "beta.kubernetes.io/arch": "amd64", 16 | "beta.kubernetes.io/fluentd-ds-ready": "true", 17 | "beta.kubernetes.io/instance-type": "n2-custom-4-6144", 18 | "beta.kubernetes.io/masq-agent-ds-ready": "true", 19 | "beta.kubernetes.io/os": "linux", 20 | "cloud.google.com/gke-nodepool": "core-20191114", 21 | "cloud.google.com/gke-os-distribution": "cos", 22 | "failure-domain.beta.kubernetes.io/region": "europe-west4", 23 | "failure-domain.beta.kubernetes.io/zone": "europe-west4-a", 24 | "kubernetes.io/arch": "amd64", 25 | "kubernetes.io/hostname": "gke-delivery-platform-core-20191114-d0aec4bb-6pzr", 26 | "kubernetes.io/os": "linux", 27 | "node.kubernetes.io/masq-agent-ds-ready": "true", 28 | "projectcalico.org/ds-ready": "true", 29 | "purpose": "core" 30 | }, 31 | "name": "gke-delivery-platform-core-20191114-d0aec4bb-6pzr", 32 | "resourceVersion": "226835919", 33 | "selfLink": "/api/v1/nodes/gke-delivery-platform-core-20191114-d0aec4bb-6pzr", 34 | "uid": "dd7c4a43-87a7-4117-bf75-fbfdb94471b0" 35 | }, 36 | "spec": { 37 | "podCIDR": "10.206.5.0/24", 38 | "providerID": "gce://at-delivery-platform-testing/europe-west4-a/gke-delivery-platform-core-20191114-d0aec4bb-6pzr", 39 | "taints": [ 40 | { 41 | "effect": "NoSchedule", 42 | "key": "key", 43 | "value": "core" 44 | } 45 | ] 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /test/tester.test.ts: -------------------------------------------------------------------------------- 1 | import Tester, { ITester } from 'lib/tester' 2 | import * as td from 'testdouble' 3 | import { IConfig } from 'lib/config' 4 | import { IDiscovery, IAgent } from 'lib/discovery' 5 | import { IMetrics } from 'lib/apps/agent/metrics' 6 | import * as should from 'should' 7 | import { Got } from 'got/dist/source' 8 | import { IUdpClientFactory } from 'lib/udp/clientFactory' 9 | import { IUDPClient, IUDPPingResult } from 'lib/udp/client' 10 | 11 | describe('Tester', () => { 12 | let sut: ITester 13 | let config: IConfig 14 | let got: Got 15 | let udpClientFactory: IUdpClientFactory 16 | 17 | before(async () => { 18 | config = td.object() 19 | config.testConfig.udp.timeout = 500 20 | config.testConfig.udp.packets = 1 21 | config.testConfig.tcp.timeout = 500 22 | config.port = 8080 23 | }) 24 | 25 | beforeEach(async () => { 26 | const discovery = td.object() 27 | const metrics = td.object() 28 | const me = td.object() 29 | got = td.function() 30 | udpClientFactory = td.object() 31 | sut = new Tester(config, got, discovery, metrics, me, udpClientFactory) 32 | }) 33 | 34 | it('should do a dns test', async () => { 35 | config.testConfig.dns.hosts = ['www.google.com'] 36 | const result = await sut.runDNSTests() 37 | should(result[0].result).eql('pass') 38 | }) 39 | 40 | it('should do a udp test', async () => { 41 | const udpClient = td.object() 42 | const udpPingResult = td.object() 43 | udpPingResult.success = true 44 | td.when( 45 | udpClient.ping( 46 | config.testConfig.udp.timeout, 47 | config.testConfig.udp.packets 48 | ) 49 | ).thenResolve(udpPingResult) 50 | const agent = td.object() 51 | agent.ip = '127.0.0.1' 52 | agent.name = 'local' 53 | agent.nodeName = 'some-node' 54 | agent.zone = 'some-zone' 55 | td.when(udpClientFactory.clientFor(agent)).thenReturn(udpClient) 56 | 57 | const result = await sut.runUDPTests([agent]) 58 | should(result[0].result).eql('pass') 59 | }) 60 | 61 | it('should should capture a failed ping as an fail', async () => { 62 | const udpClient = td.object() 63 | const udpPingResult = td.object() 64 | udpPingResult.success = true 65 | td.when( 66 | udpClient.ping( 67 | config.testConfig.udp.timeout, 68 | config.testConfig.udp.packets 69 | ) 70 | ).thenReject(new Error('boom')) 71 | const agent = td.object() 72 | agent.ip = '127.0.0.1' 73 | agent.name = 'local' 74 | agent.nodeName = 'some-node' 75 | agent.zone = 'some-zone' 76 | td.when(udpClientFactory.clientFor(agent)).thenReturn(udpClient) 77 | 78 | const result = await sut.runUDPTests([agent]) 79 | should(result[0].result).eql('fail') 80 | }) 81 | 82 | it('should do a tcp test', async () => { 83 | td.when( 84 | got('http://127.0.0.1:8080/readiness', { timeout: 500 }) 85 | ).thenResolve({ statusCode: 200 }) 86 | 87 | const agent = td.object() 88 | agent.ip = '127.0.0.1' 89 | agent.name = 'local' 90 | agent.nodeName = 'some-node' 91 | agent.zone = 'some-zone' 92 | const result = await sut.runTCPTests([agent]) 93 | should(result[0].result).eql('pass') 94 | }) 95 | 96 | it('should capture a 5xx code as a fail', async () => { 97 | td.when( 98 | got('http://127.0.0.1:8080/readiness', { timeout: 500 }) 99 | ).thenResolve({ statusCode: 500 }) 100 | 101 | const agent = td.object() 102 | agent.ip = '127.0.0.1' 103 | agent.name = 'local' 104 | agent.nodeName = 'some-node' 105 | agent.zone = 'some-zone' 106 | const result = await sut.runTCPTests([agent]) 107 | should(result[0].result).eql('fail') 108 | }) 109 | 110 | it('should capture a failed tcp test as a fail', async () => { 111 | td.when( 112 | got('http://127.0.0.1:8080/readiness', { timeout: 500 }) 113 | ).thenReject(new Error('boom')) 114 | 115 | const agent = td.object() 116 | agent.ip = '127.0.0.1' 117 | agent.name = 'local' 118 | agent.nodeName = 'some-node' 119 | agent.zone = 'some-zone' 120 | const result = await sut.runTCPTests([agent]) 121 | should(result[0].result).eql('fail') 122 | }) 123 | }) 124 | -------------------------------------------------------------------------------- /test/udp.test.ts: -------------------------------------------------------------------------------- 1 | import UDPServer, { IUDPServer } from 'lib/udp/server' 2 | import * as td from 'testdouble' 3 | import { IConfig } from 'lib/config' 4 | import UDPClient, { IUDPClient } from 'lib/udp/client' 5 | import * as should from 'should' 6 | 7 | describe('UDP Server', () => { 8 | let server: IUDPServer 9 | let client: IUDPClient 10 | before(() => { 11 | const config = td.object() 12 | config.port = 8080 13 | server = new UDPServer(config) 14 | client = new UDPClient('127.0.0.1', config.port) 15 | }) 16 | after(async () => { 17 | return server.stop() 18 | }) 19 | it('should accept udp packets', async () => { 20 | await server.start() 21 | const result = await client.ping(50, 1) 22 | should(result.results.length).eql(1) 23 | should(result.results[0].success).eql(true) 24 | should(result.success).eql(true) 25 | should(Object.keys(result).sort()).eql( 26 | [ 27 | 'average', 28 | 'duration', 29 | 'loss', 30 | 'max', 31 | 'min', 32 | 'results', 33 | 'success', 34 | 'variance' 35 | ].sort() 36 | ) 37 | }) 38 | }) 39 | -------------------------------------------------------------------------------- /test/webserver.test.ts: -------------------------------------------------------------------------------- 1 | import WebServer, { IWebServer } from 'lib/web-server' 2 | import * as td from 'testdouble' 3 | import * as express from 'express' 4 | import { IConfig } from 'lib/config' 5 | import got from 'got' 6 | import * as should from 'should' 7 | 8 | describe('Web Server', () => { 9 | let sut: IWebServer 10 | before(() => { 11 | const config = td.object() 12 | config.port = 8080 13 | sut = new WebServer(config) 14 | }) 15 | after(async () => { 16 | return sut.stop() 17 | }) 18 | it('should start and stop', async () => { 19 | await sut.start( 20 | (app: express.Application): Promise => { 21 | app.get('/', (req, res) => { 22 | res.sendStatus(200) 23 | }) 24 | return Promise.resolve() 25 | } 26 | ) 27 | const result = await got('http://127.0.0.1:8080') 28 | should(result.statusCode).eql(200) 29 | }) 30 | }) 31 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "target": "es6", 5 | "rootDir": "./", 6 | "outDir": "./built", 7 | "baseUrl": "./", 8 | "allowJs": true, 9 | "checkJs": true, 10 | "alwaysStrict": true, 11 | "noImplicitAny": false, 12 | "noImplicitThis": true, 13 | "strictNullChecks": true, 14 | "strictFunctionTypes": true, 15 | "strictPropertyInitialization": true, 16 | "noEmitOnError": false, 17 | "sourceMap": true, 18 | "lib": ["ES2015", "ES2016", "ES2017", "ESNext"] 19 | }, 20 | "include": ["lib/**/*", "test/**/*"], 21 | "exclude": ["node_modules"] 22 | } 23 | --------------------------------------------------------------------------------