├── kubernetes ├── netsil-ns.yaml ├── README.md ├── configmaps │ ├── integrations-configmap.yml │ ├── collector.yaml │ └── auto-configmap.yml ├── netsil-svc.yaml ├── rbac-setup.yaml ├── 1.5 │ └── collector.yaml ├── collector.yaml ├── collector-rbac.yaml ├── netsil-rc.yaml ├── netsil.yml └── kube-netsil │ └── kube-netsil.sh ├── docker ├── README.md ├── install-collectors.sh ├── install-netsil.sh └── docker-compose.yml ├── cloudformation ├── README.md └── netsil-cloudformation.json ├── mesos-marathon ├── README.md ├── netsil-dcos-collectors.json └── netsil-dcos.json ├── bin ├── README.md ├── make-images-public.sh ├── copy-machine-images.sh └── update.sh ├── LICENSE ├── README.md └── ecs └── netsil-collectors.json /kubernetes/netsil-ns.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: netsil 5 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | ## Installation 2 | Please refer to our [documentation](https://docs.netsil.com/setup-guide/installation/docker) for installation instructions. 3 | -------------------------------------------------------------------------------- /kubernetes/README.md: -------------------------------------------------------------------------------- 1 | ## Installation 2 | Please refer to our [documentation](https://docs.netsil.com/setup-guide/installation/k8s) for installation instructions. 3 | -------------------------------------------------------------------------------- /cloudformation/README.md: -------------------------------------------------------------------------------- 1 | ## Installation 2 | Please refer to our [documentation](https://docs.netsil.com/setup-guide/installation/aws) for installation instructions. 3 | -------------------------------------------------------------------------------- /mesos-marathon/README.md: -------------------------------------------------------------------------------- 1 | ## Installation 2 | Please refer to our [documentation](https://docs.netsil.com/setup-guide/installation/mesos-marathon) for installation instructions. 3 | -------------------------------------------------------------------------------- /bin/README.md: -------------------------------------------------------------------------------- 1 | ## To update the version number (example) 2 | PREVIOUS_VERSION=1.4.5 NEXT_VERSION=1.5.0 ./bin/update.sh version 3 | 4 | ## To update the latest tag (example) 5 | PREVIOUS_VERSION=1.4.5 NEXT_VERSION=1.5.0 ./bin/update.sh tag 6 | -------------------------------------------------------------------------------- /kubernetes/configmaps/integrations-configmap.yml: -------------------------------------------------------------------------------- 1 | kind: ConfigMap 2 | apiVersion: v1 3 | metadata: 4 | name: integrations 5 | namespace: epoch 6 | data: 7 | kubernetes.yaml: | 8 | init_config: 9 | 10 | instances: 11 | - port: 4194 12 | -------------------------------------------------------------------------------- /docker/install-collectors.sh: -------------------------------------------------------------------------------- 1 | NETSIL_HOST=${NETSIL_HOST:-"127.0.0.1"} 2 | docker run -td \ 3 | --name=netsil_collectors \ 4 | --net=host \ 5 | -v /var/run/docker.sock:/var/run/docker.sock:ro \ 6 | -v /proc/:/host/proc/:ro \ 7 | -v /sys/fs/cgroup/:/host/sys/fs/cgroup:ro \ 8 | --cap-add=NET_RAW \ 9 | --cap-add=NET_ADMIN \ 10 | --ulimit core=0 \ 11 | -e DEPLOY_ENV="docker" \ 12 | -e SD_BACKEND="docker" \ 13 | -e NETSIL_SP_HOST=${NETSIL_HOST} \ 14 | netsil/collectors:latest 15 | -------------------------------------------------------------------------------- /kubernetes/netsil-svc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | namespace: netsil 5 | name: netsil 6 | spec: 7 | selector: 8 | app: netsil 9 | component: netsil 10 | ports: 11 | - name: http 12 | port: 80 13 | - name: https 14 | port: 443 15 | nodePort: 30443 16 | - name: port2000 17 | port: 2000 18 | - name: port2001 19 | port: 2001 20 | - name: port2003 21 | port: 2003 22 | - name: port2003udp 23 | port: 2003 24 | protocol: UDP 25 | type: NodePort 26 | -------------------------------------------------------------------------------- /bin/make-images-public.sh: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/bash 2 | 3 | # TODO: Generate this list from the copy-images script output in the future 4 | declare -A IMAGE_IDS 5 | 6 | for f in *.json ; do 7 | region=${f%-*} 8 | image_id=$(cat $f | jq -r '.ImageId') 9 | IMAGE_IDS[$region]=${image_id} 10 | done 11 | 12 | # TODO: Can this command be run before the AMI itself is ready? 13 | for i in ${!IMAGE_IDS[@]} 14 | do 15 | aws --region ${i} ec2 modify-image-attribute \ 16 | --image-id ${IMAGE_IDS[$i]} \ 17 | --launch-permission "{\"Add\":[{\"Group\":\"all\"}]}" 18 | echo "Made ${IMAGE_IDS[$i]} public" 19 | done 20 | 21 | 22 | for i in ${!IMAGE_IDS[@]} 23 | do 24 | aws --region ${i} ec2 describe-image-attribute \ 25 | --image-id ${IMAGE_IDS[$i]} \ 26 | --attribute launchPermission 27 | done 28 | -------------------------------------------------------------------------------- /docker/install-netsil.sh: -------------------------------------------------------------------------------- 1 | docker run -td \ 2 | --name netsil_aoc \ 3 | -v /opt/netsil/lite/ceph/conf:/etc/ceph \ 4 | -v /opt/netsil/lite/ceph/data:/var/lib/ceph-data \ 5 | -v /opt/netsil/lite/druid/realtime-segments:/opt/netsil/druid/realtime-segments \ 6 | -v /opt/netsil/lite/druid/indexCache:/var/tmp/druid/indexCache \ 7 | -v /opt/netsil/lite/elasticsearch/data:/usr/share/elasticsearch/data \ 8 | -v /opt/netsil/lite/kafka/kafka-log-dir:/opt/netsil/kafka/kafka-log-dir \ 9 | -v /opt/netsil/lite/license-manager/licenses:/opt/netsil/license-manager/licenses \ 10 | -v /opt/netsil/lite/mysql/data:/var/lib/mysql \ 11 | -v /opt/netsil/lite/redis:/var/lib/redis \ 12 | -v /opt/netsil/lite/zookeeper/data:/opt/netsil/zookeeper/data \ 13 | -p 80:80 \ 14 | -p 443:443 \ 15 | -p 8443:8443 \ 16 | -p 2000:2000 \ 17 | -p 2001:2001 \ 18 | -p 2003:2003 \ 19 | -p 3003:3003 \ 20 | -p 2003:2003/udp \ 21 | --log-driver=none \ 22 | netsil/netsil:latest \ 23 | /root/startup.sh 24 | -------------------------------------------------------------------------------- /bin/copy-machine-images.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Source AMI is: " ${SOURCE_AMI} 4 | echo "Source region is: " ${SOURCE_REGION} 5 | AMI_NAME=${AMI_NAME:-"Netsil - Stable ${NETSIL_VERSION_NUMBER}"} 6 | 7 | dryrun_s3="" 8 | dryrun_ami="" 9 | if [[ ${DO_DRY} == "yes" ]] ; then 10 | dryrun_s3="--dryrun" 11 | dryrun_ami="--dry-run" 12 | fi 13 | 14 | # The aws ec2 copy command doesn't block, and returns the image id in stdout 15 | function copy_ami() { 16 | dest_region=$1 17 | aws ec2 copy-image ${dryrun_ami} \ 18 | --source-image-id ${SOURCE_AMI} \ 19 | --source-region ${SOURCE_REGION} \ 20 | --region ${dest_region} \ 21 | --name "${AMI_NAME}" 22 | } 23 | 24 | DEST_REGIONS=( 25 | us-east-1 \ 26 | us-east-2 \ 27 | us-west-1 \ 28 | eu-west-1 \ 29 | eu-central-1 \ 30 | ap-southeast-1 \ 31 | ap-northeast-1 \ 32 | ap-south-1 \ 33 | sa-east-1 \ 34 | ) 35 | 36 | 37 | # Copy AMIs 38 | for region in ${DEST_REGIONS[@]} 39 | do 40 | rm -f "${region}-ami.json" 41 | copy_ami $region > "${region}-ami.json" 42 | done 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | netsil: 4 | image: netsil/netsil:latest 5 | restart: unless-stopped 6 | command: ["/root/startup.sh"] 7 | volumes: 8 | - /opt/netsil/lite/ceph/conf:/etc/ceph 9 | - /opt/netsil/lite/ceph/data:/var/lib/ceph-data 10 | - /opt/netsil/lite/druid/realtime-segments:/opt/netsil/druid/realtime-segments 11 | - /opt/netsil/lite/druid/indexCache:/var/tmp/druid/indexCache 12 | - /opt/netsil/lite/elasticsearch/data:/usr/share/elasticsearch/data 13 | - /opt/netsil/lite/kafka/kafka-log-dir:/opt/netsil/kafka/kafka-log-dir 14 | - /opt/netsil/lite/license-manager/licenses:/opt/netsil/license-manager/licenses 15 | - /opt/netsil/lite/mysql/data:/var/lib/mysql 16 | - /opt/netsil/lite/redis:/var/lib/redis 17 | - /opt/netsil/lite/zookeeper/data:/opt/netsil/zookeeper/data 18 | environment: 19 | - DEPLOY_ENV=docker-compose 20 | ports: 21 | - '80:80' 22 | - '443:443' 23 | - '2000:2000' 24 | - '2001:2001' 25 | - '2003:2003' 26 | - '2003:2003/udp' 27 | logging: 28 | driver: none 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## About 2 | Netsil Application Operations Center (AOC) is a next-gen observability and analytics tool for modern cloud applications. The Netsil AOC helps SREs and DevOps improve the reliability and performance of API and microservices-driven production applications. 3 | 4 | At the heart of the AOC is an auto-discovered service topology map. It visualizes service dependencies and operational metrics so practitioners can work collaboratively across teams. 5 | 6 | ## Introduction 7 | Installation is done in two parts: **Netsil AOC** and the **Netsil Collectors**. 8 | 9 | The collectors are installed on your application instances (VMs or containerized environments) and mirror the service interactions & metrics back to Netsil AOC for real-time analysis. 10 | 11 | You may install the collectors in a variety of environments, independent of the AOC environment. 12 | For instance, you may run the AOC as a standalone docker container, but install the collectors in Kubernetes. 13 | 14 | ## Documentation 15 | Refer to our [documentation](https://docs.netsil.com) site for installation instructions, API definitions, user guides, and more. 16 | 17 | ## Support 18 | For help please join our public [slack channel](http://slack.netsil.com) or email support@netsil.com 19 | -------------------------------------------------------------------------------- /kubernetes/rbac-setup.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: ClusterRole 3 | apiVersion: rbac.authorization.k8s.io/v1beta1 4 | metadata: 5 | name: epoch 6 | namespace: epoch 7 | rules: 8 | - nonResourceURLs: 9 | - "/version" 10 | - "/healthz" 11 | verbs: ["get"] 12 | - apiGroups: [""] 13 | resources: 14 | - "nodes" 15 | - "namespaces" 16 | - "events" 17 | - "services" 18 | verbs: ["get", "list"] 19 | - apiGroups: [""] 20 | resources: 21 | - "pods" 22 | verbs: ["get", "watch", "list"] 23 | - apiGroups: [""] 24 | resources: 25 | - "configmaps" 26 | resourceNames: ["configmap-volume", "configmap-auto-conf"] 27 | verbs: ["get", "delete", "update"] 28 | - apiGroups: [""] 29 | resources: 30 | - "configmaps" 31 | verbs: ["create"] 32 | --- 33 | apiVersion: v1 34 | kind: ServiceAccount 35 | metadata: 36 | name: epoch 37 | namespace: epoch 38 | automountServiceAccountToken: true 39 | --- 40 | kind: ClusterRoleBinding 41 | apiVersion: rbac.authorization.k8s.io/v1beta1 42 | metadata: 43 | name: epoch 44 | subjects: 45 | - kind: ServiceAccount 46 | name: epoch 47 | namespace: epoch 48 | roleRef: 49 | kind: ClusterRole 50 | name: epoch 51 | apiGroup: rbac.authorization.k8s.io 52 | -------------------------------------------------------------------------------- /mesos-marathon/netsil-dcos-collectors.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "netsil-collectors", 3 | "container": { 4 | "type": "DOCKER", 5 | "docker": { 6 | "network": "HOST", 7 | "image": "netsil/collectors:latest", 8 | "parameters": [ 9 | { "key": "cap-add", "value": "NET_RAW" }, 10 | { "key": "cap-add", "value": "NET_ADMIN" } 11 | ] 12 | }, 13 | "volumes": [ 14 | { 15 | "containerPath": "/var/run/docker.sock", 16 | "hostPath": "/var/run/docker.sock", 17 | "mode": "RO" 18 | }, 19 | { 20 | "containerPath": "/host/proc/", 21 | "hostPath": "/proc/", 22 | "mode": "RO" 23 | }, 24 | { 25 | "containerPath": "/host/sys/fs/cgroup", 26 | "hostPath": "/sys/fs/cgroup/", 27 | "mode": "RO" 28 | } 29 | ] 30 | }, 31 | "cpus": 0.2, 32 | "mem": 512, 33 | "instances": 1, 34 | "acceptedResourceRoles": ["*", "slave_public"], 35 | "env": { 36 | "NETSIL_SP_HOST": "netsil.marathon.mesos", 37 | "DEPLOY_ENV": "docker" 38 | }, 39 | "constraints": [ 40 | ["hostname", "UNIQUE"] 41 | ] 42 | } 43 | -------------------------------------------------------------------------------- /bin/update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ -z "${NEXT_VERSION}" ]] ; then 4 | echo "NEXT_VERSION not set." 5 | exit 1 6 | fi 7 | 8 | 9 | update_version() { 10 | if [[ -z "${PREVIOUS_VERSION}" ]] ; then 11 | echo "PREVIOUS_VERSION not set." 12 | exit 1 13 | fi 14 | 15 | manifests=( 16 | ./docker/docker-compose.yml 17 | ./docker/install-netsil.sh 18 | ./docker/install-collectors.sh 19 | ./kubernetes/collector.yaml 20 | ./kubernetes/1.5/collector.yaml 21 | ./kubernetes/configmaps/collector.yaml 22 | ./kubernetes/netsil-rc.yaml 23 | ./kubernetes/netsil.yml 24 | ./kubernetes/kube-netsil/kube-netsil.sh 25 | ./mesos-marathon/netsil-dcos.json 26 | ./mesos-marathon/netsil-dcos-collectors.json 27 | ) 28 | 29 | for i in "${manifests[@]}" 30 | do 31 | sed -i.bak "s/stable-${PREVIOUS_VERSION}/stable-${NEXT_VERSION}/g" $i 32 | done 33 | } 34 | 35 | update_tag() { 36 | git tag -d latest 37 | git tag -a "latest" -m "Release ${NEXT_VERSION}" 38 | git tag -a "${NEXT_VERSION}" -m "Release ${NEXT_VERSION}" 39 | git push origin --tags -f 40 | } 41 | 42 | 43 | case "$1" in 44 | version) 45 | update_version 46 | ;; 47 | tag) 48 | update_tag 49 | ;; 50 | *) 51 | echo $"Usage: $0 {version|tag}" 52 | exit 1 53 | esac 54 | 55 | -------------------------------------------------------------------------------- /kubernetes/1.5/collector.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | metadata: 4 | namespace: netsil 5 | name: collector 6 | labels: 7 | app: netsil 8 | component: collector 9 | spec: 10 | template: 11 | metadata: 12 | labels: 13 | app: netsil 14 | component: collector 15 | spec: 16 | hostNetwork: true 17 | dnsPolicy: Default 18 | containers: 19 | - name: collector 20 | image: netsil/collectors:latest 21 | command: ["/bin/bash","-c","while true ; do NETSIL_SP_HOST=$NETSIL_SERVICE_HOST /opt/netsil/collectors/start.sh ; echo Exiting, possibly to upgrade ; sleep 5 ; done"] 22 | securityContext: 23 | capabilities: 24 | add: 25 | - NET_RAW 26 | - NET_ADMIN 27 | env: 28 | # Please DO NOT prepend http:// or https:// to the NETSIL_SERVICE_HOST value 29 | - name: NETSIL_SERVICE_HOST 30 | value: ${your_netsil_ip} 31 | - name: NETSIL_ORGANIZATION_ID 32 | value: ${organizationId} 33 | - name: DEPLOY_ENV 34 | value: 'docker' 35 | - name: KUBERNETES 36 | value: "yes" 37 | - name: SD_BACKEND 38 | value: docker 39 | volumeMounts: 40 | - name: cgroup 41 | mountPath: /host/sys/fs/cgroup/ 42 | readOnly: true 43 | - name: proc 44 | mountPath: /host/proc/ 45 | readOnly: true 46 | - name: docker-sock 47 | mountPath: /var/run/docker.sock 48 | readOnly: true 49 | volumes: 50 | - name: cgroup 51 | hostPath: 52 | path: /sys/fs/cgroup/ 53 | - name: proc 54 | hostPath: 55 | path: /proc/ 56 | - name: docker-sock 57 | hostPath: 58 | path: /var/run/docker.sock 59 | -------------------------------------------------------------------------------- /kubernetes/collector.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | metadata: 4 | namespace: netsil 5 | name: collector 6 | labels: 7 | app: netsil 8 | component: collector 9 | spec: 10 | minReadySeconds: 0 11 | updateStrategy: 12 | type: RollingUpdate 13 | rollingUpdate: 14 | maxUnavailable: 1 15 | template: 16 | metadata: 17 | labels: 18 | app: netsil 19 | component: collector 20 | spec: 21 | hostNetwork: true 22 | dnsPolicy: ClusterFirstWithHostNet 23 | containers: 24 | - name: collector 25 | image: netsil/collectors:latest 26 | command: ["/bin/bash","-c","while true ; do NETSIL_SP_HOST=$NETSIL_SERVICE_HOST /opt/netsil/collectors/start.sh ; echo Exiting, possibly to upgrade ; sleep 5 ; done"] 27 | securityContext: 28 | capabilities: 29 | add: 30 | - NET_RAW 31 | - NET_ADMIN 32 | env: 33 | # Please DO NOT prepend http:// or https:// to the NETSIL_SERVICE_HOST value 34 | - name: NETSIL_SERVICE_HOST 35 | value: ${your_netsil_ip} 36 | - name: NETSIL_ORGANIZATION_ID 37 | value: ${organizationId} 38 | - name: DEPLOY_ENV 39 | value: 'docker' 40 | - name: KUBERNETES 41 | value: "yes" 42 | - name: SD_BACKEND 43 | value: docker 44 | volumeMounts: 45 | - name: cgroup 46 | mountPath: /host/sys/fs/cgroup/ 47 | readOnly: true 48 | - name: proc 49 | mountPath: /host/proc/ 50 | readOnly: true 51 | - name: docker-sock 52 | mountPath: /var/run/docker.sock 53 | readOnly: true 54 | volumes: 55 | - name: cgroup 56 | hostPath: 57 | path: /sys/fs/cgroup/ 58 | - name: proc 59 | hostPath: 60 | path: /proc/ 61 | - name: docker-sock 62 | hostPath: 63 | path: /var/run/docker.sock 64 | -------------------------------------------------------------------------------- /kubernetes/collector-rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | metadata: 4 | namespace: netsil 5 | name: collector 6 | labels: 7 | app: netsil 8 | component: collector 9 | spec: 10 | minReadySeconds: 0 11 | updateStrategy: 12 | type: RollingUpdate 13 | rollingUpdate: 14 | maxUnavailable: 1 15 | template: 16 | metadata: 17 | labels: 18 | app: netsil 19 | component: collector 20 | spec: 21 | hostNetwork: true 22 | dnsPolicy: ClusterFirstWithHostNet 23 | serviceAccountName: netsil 24 | containers: 25 | - name: collector 26 | image: netsil/collectors:latest 27 | command: ["/bin/bash","-c","while true ; do NETSIL_SP_HOST=$NETSIL_SERVICE_HOST /opt/netsil/collectors/start.sh ; echo Exiting, possibly to upgrade ; sleep 5 ; done"] 28 | securityContext: 29 | capabilities: 30 | add: 31 | - NET_RAW 32 | - NET_ADMIN 33 | env: 34 | #- name: NETSIL_SERVICE_HOST 35 | # value: "" 36 | #- name: NETSIL_ORGANIZATION_ID 37 | # value: "" 38 | - name: DEPLOY_ENV 39 | value: "docker" 40 | - name: KUBERNETES 41 | value: "yes" 42 | - name: SD_BACKEND 43 | value: "docker" 44 | - name: KUBERNETES_KUBELET_HOST 45 | valueFrom: 46 | fieldRef: 47 | fieldPath: status.hostIP 48 | volumeMounts: 49 | - name: cgroup 50 | mountPath: /host/sys/fs/cgroup/ 51 | readOnly: true 52 | - name: proc 53 | mountPath: /host/proc/ 54 | readOnly: true 55 | - name: docker-sock 56 | mountPath: /var/run/docker.sock 57 | readOnly: true 58 | volumes: 59 | - name: cgroup 60 | hostPath: 61 | path: /sys/fs/cgroup/ 62 | - name: proc 63 | hostPath: 64 | path: /proc/ 65 | - name: docker-sock 66 | hostPath: 67 | path: /var/run/docker.sock 68 | -------------------------------------------------------------------------------- /kubernetes/configmaps/collector.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | metadata: 4 | namespace: netsil 5 | name: collector 6 | labels: 7 | app: netsil 8 | component: collector 9 | spec: 10 | minReadySeconds: 0 11 | updateStrategy: 12 | type: RollingUpdate 13 | rollingUpdate: 14 | maxUnavailable: 1 15 | template: 16 | metadata: 17 | labels: 18 | app: netsil 19 | component: collector 20 | spec: 21 | hostNetwork: true 22 | dnsPolicy: ClusterFirstWithHostNet 23 | containers: 24 | - name: collector 25 | image: netsil/collectors:latest 26 | command: ["/bin/bash","-c","while true ; do NETSIL_SP_HOST=$NETSIL_SERVICE_HOST /opt/netsil/collectors/start.sh ; echo Exiting, possibly to upgrade ; sleep 5 ; done"] 27 | securityContext: 28 | capabilities: 29 | add: 30 | - NET_RAW 31 | - NET_ADMIN 32 | env: 33 | # Please DO NOT prepend http:// or https:// to the NETSIL_SERVICE_HOST value 34 | - name: NETSIL_SERVICE_HOST 35 | value: ${your_netsil_ip} 36 | - name: NETSIL_ORGANIZATION_ID 37 | value: ${organizationId} 38 | - name: DEPLOY_ENV 39 | value: "docker" 40 | - name: KUBERNETES 41 | value: "yes" 42 | - name: SD_BACKEND 43 | value: "docker" 44 | volumeMounts: 45 | - name: cgroup 46 | mountPath: /host/sys/fs/cgroup/ 47 | readOnly: true 48 | - name: proc 49 | mountPath: /host/proc/ 50 | readOnly: true 51 | - name: docker-sock 52 | mountPath: /var/run/docker.sock 53 | readOnly: true 54 | - name: configmap-volume 55 | mountPath: /conf.d/ 56 | - name: configmap-auto-conf 57 | mountPath: /etc/netsil-dd-agent/conf.d/auto_conf/ 58 | volumes: 59 | - name: cgroup 60 | hostPath: 61 | path: /sys/fs/cgroup/ 62 | - name: proc 63 | hostPath: 64 | path: /proc/ 65 | - name: docker-sock 66 | hostPath: 67 | path: /var/run/docker.sock 68 | - name: configmap-volume 69 | configMap: 70 | name: integrations 71 | - name: configmap-auto-conf 72 | configMap: 73 | name: auto-conf 74 | -------------------------------------------------------------------------------- /ecs/netsil-collectors.json: -------------------------------------------------------------------------------- 1 | { 2 | "executionRoleArn":null, 3 | "containerDefinitions":[ 4 | { 5 | "dnsSearchDomains":null, 6 | "logConfiguration":null, 7 | "entryPoint":null, 8 | "portMappings":[ 9 | 10 | ], 11 | "command":null, 12 | "linuxParameters":{ 13 | "capabilities":{ 14 | "add":[ 15 | "NET_RAW", 16 | "NET_ADMIN" 17 | ], 18 | "drop":null 19 | }, 20 | "sharedMemorySize":null, 21 | "tmpfs":null, 22 | "devices":null, 23 | "initProcessEnabled":null 24 | }, 25 | "cpu":100, 26 | "environment":[ 27 | { 28 | "name":"SD_BACKEND", 29 | "value":"docker" 30 | }, 31 | { 32 | "name":"DEPLOY_ENV", 33 | "value":"docker" 34 | }, 35 | { 36 | "name":"NETSIL_ORGANIZATION_ID", 37 | "value":${organizationId} 38 | }, 39 | { 40 | "name":"SAMPLINGRATE", 41 | "value":"100" 42 | }, 43 | { 44 | "name":"NETSIL_SP_HOST", 45 | "value":${your_netsil_ip} 46 | } 47 | ], 48 | "ulimits":[ 49 | { 50 | "name":"core", 51 | "softLimit":0, 52 | "hardLimit":0 53 | } 54 | ], 55 | "dnsServers":null, 56 | "mountPoints":[ 57 | 58 | ], 59 | "workingDirectory":null, 60 | "dockerSecurityOptions":null, 61 | "memory":1024, 62 | "memoryReservation":null, 63 | "volumesFrom":[ 64 | 65 | ], 66 | "image":"netsil/collectors:latest", 67 | "disableNetworking":null, 68 | "healthCheck":null, 69 | "essential":true, 70 | "links":null, 71 | "hostname":null, 72 | "extraHosts":null, 73 | "user":null, 74 | "readonlyRootFilesystem":null, 75 | "dockerLabels":null, 76 | "privileged":null, 77 | "name":"netsil" 78 | } 79 | ], 80 | "memory":null, 81 | "taskRoleArn":null, 82 | "family":"netsil-task", 83 | "requiresCompatibilities":[ 84 | 85 | ], 86 | "networkMode":"host", 87 | "cpu":null, 88 | "volumes":[ 89 | { 90 | "name":"docker_sock", 91 | "host":{ 92 | "sourcePath":"/var/run/docker.sock" 93 | } 94 | }, 95 | { 96 | "name":"proc", 97 | "host":{ 98 | "sourcePath":"/proc/" 99 | } 100 | }, 101 | { 102 | "name":"cgroup", 103 | "host":{ 104 | "sourcePath":"/cgroup/" 105 | } 106 | } 107 | ], 108 | "placementConstraints":[ 109 | 110 | ] 111 | } 112 | -------------------------------------------------------------------------------- /kubernetes/netsil-rc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ReplicationController 3 | metadata: 4 | namespace: netsil 5 | name: netsil 6 | spec: 7 | replicas: 1 8 | selector: 9 | app: netsil 10 | component: netsil 11 | template: 12 | metadata: 13 | name: stream-processor 14 | labels: 15 | app: netsil 16 | component: netsil 17 | version: stable 18 | spec: 19 | containers: 20 | - name: netsil 21 | image: netsil/netsil:latest 22 | command: 23 | - /root/startup.sh 24 | ports: 25 | - containerPort: 80 26 | - containerPort: 443 27 | - containerPort: 2000 28 | - containerPort: 2001 29 | - containerPort: 2003 30 | - containerPort: 2003 31 | protocol: UDP 32 | volumeMounts: 33 | - name: alerts 34 | mountPath: /opt/netsil/alerts/data 35 | - name: ceph-conf 36 | mountPath: /etc/ceph 37 | - name: ceph-data 38 | mountPath: /var/lib/ceph-data 39 | - name: druid-index-cache 40 | mountPath: /var/tmp/druid/indexCache 41 | - name: druid-realtime-segments 42 | mountPath: /opt/netsil/druid/realtime-segments 43 | - name: elasticsearch 44 | mountPath: /usr/share/elasticsearch/data 45 | - name: kafka-logs 46 | mountPath: /opt/netsil/kafka/kafka-log-dir 47 | - name: licenses 48 | mountPath: /opt/netsil/lite/license-manager/licenses 49 | - name: mysql 50 | mountPath: /var/lib/mysql 51 | - name: redis 52 | mountPath: /var/lib/redis 53 | - name: user-persistence 54 | mountPath: /opt/netsil/user-persistence/data 55 | - name: zookeeper 56 | mountPath: /opt/netsil/zookeeper/data 57 | volumes: 58 | - name: alerts 59 | hostPath: 60 | path: /opt/netsil/lite/alerts/data 61 | - name: ceph-conf 62 | hostPath: 63 | path: /opt/netsil/lite/ceph/conf 64 | - name: ceph-data 65 | hostPath: 66 | path: /opt/netsil/lite/ceph/data 67 | - name: druid-index-cache 68 | hostPath: 69 | path: /var/tmp/druid/indexCache 70 | - name: druid-realtime-segments 71 | hostPath: 72 | path: /opt/netsil/lite/druid/realtime-segments 73 | - name: elasticsearch 74 | hostPath: 75 | path: /opt/netsil/lite/elasticsearch/data 76 | - name: licenses 77 | hostPath: 78 | path: /opt/netsil/lite/license-manager/licenses 79 | - name: kafka-logs 80 | hostPath: 81 | path: /opt/netsil/lite/kafka/kafka-log-dir 82 | - name: mysql 83 | hostPath: 84 | path: /opt/netsil/lite/mysql/data 85 | - name: redis 86 | hostPath: 87 | path: /opt/netsil/lite/redis 88 | - name: user-persistence 89 | hostPath: 90 | path: /opt/netsil/lite/user-persistence/data 91 | - name: zookeeper 92 | hostPath: 93 | path: /opt/netsil/lite/zookeeper/data 94 | -------------------------------------------------------------------------------- /kubernetes/configmaps/auto-configmap.yml: -------------------------------------------------------------------------------- 1 | kind: ConfigMap 2 | apiVersion: v1 3 | metadata: 4 | name: auto-conf 5 | namespace: epoch 6 | data: 7 | apache.yaml: | 8 | docker_images: 9 | - httpd 10 | 11 | init_config: 12 | 13 | instances: 14 | - apache_status_url: http://%%host%%/server-status?auto 15 | consul.yaml: | 16 | docker_images: 17 | - consul 18 | 19 | init_config: 20 | 21 | instances: 22 | - url: "http://%%host%%:%%port%%" 23 | catalog_checks: yes 24 | new_leader_checks: yes 25 | # service_whitelist: 26 | couch.yaml: | 27 | docker_images: 28 | - couchdb 29 | 30 | init_config: 31 | 32 | instances: 33 | - server: http://%%host%%:%%port%% 34 | couchbase.yaml: | 35 | docker_images: 36 | - couchbase 37 | 38 | init_config: 39 | 40 | instances: 41 | - server: http://%%host%%:%%port%% 42 | user: Administrator 43 | password: password 44 | elastic.yaml: | 45 | docker_images: 46 | - elasticsearch 47 | 48 | init_config: 49 | 50 | instances: 51 | - url: "http://%%host%%:%%port_0%%" 52 | etcd.yaml: | 53 | docker_images: 54 | - etcd 55 | 56 | init_config: 57 | 58 | instances: 59 | - url: "http://%%host%%:%%port_0%%" 60 | kube_dns.yaml: | 61 | docker_images: 62 | - kubedns-amd64 63 | - k8s-dns-kube-dns-amd64 64 | 65 | init_config: 66 | 67 | instances: 68 | - prometheus_endpoint: "http://%%host%%:10055/metrics" 69 | tags: 70 | - "dns-pod:%%host%%" 71 | kubernetes_state.yaml: | 72 | docker_images: 73 | - kube-state-metrics 74 | 75 | init_config: 76 | 77 | instances: 78 | # To enable Kube State metrics you must specify the url exposing the API 79 | - kube_state_url: http://%%host%%:%%port%%/metrics 80 | # Tags are reported as set by kube-state-metrics. If you want to translate 81 | # them to other tags, you can use the labels_mapper dictionary 82 | # labels_mapper: 83 | # namespace: kube_namespace 84 | kyototycoon.yaml: | 85 | docker_images: 86 | - kyototycoon 87 | 88 | init_config: 89 | 90 | instances: 91 | - report_url: http://%%host%%:%%port%%/rpc/report 92 | mcache.yaml: | 93 | docker_images: 94 | - memcached 95 | 96 | init_config: 97 | 98 | instances: 99 | - url: "%%host%%" 100 | port: "%%port%%" 101 | redisdb.yaml: | 102 | docker_images: 103 | - redis 104 | 105 | init_config: 106 | 107 | instances: 108 | - host: "%%host%%" 109 | port: "%%port%%" 110 | riak.yaml: | 111 | docker_images: 112 | - riak 113 | 114 | init_config: 115 | 116 | instances: 117 | - url: http://%%host%%:%%port%%/stats 118 | -------------------------------------------------------------------------------- /mesos-marathon/netsil-dcos.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "netsil", 3 | "cpus": 4, 4 | "mem": 16000, 5 | "disk": 0, 6 | "instances": 1, 7 | "acceptedResourceRoles": [ 8 | "*" 9 | ], 10 | "container": { 11 | "type": "DOCKER", 12 | "volumes": [ 13 | { 14 | "containerPath": "zk_data", 15 | "mode": "RW", 16 | "persistent": { 17 | "size": 250 18 | } 19 | }, 20 | { 21 | "containerPath": "/opt/netsil/lite/zookeeper/data", 22 | "hostPath": "zk_data", 23 | "mode": "RW" 24 | }, 25 | { 26 | "containerPath": "mysql_data", 27 | "mode": "RW", 28 | "persistent": { 29 | "size": 250 30 | } 31 | }, 32 | { 33 | "containerPath": "/var/lib/mysql", 34 | "hostPath": "mysql_data", 35 | "mode": "RW" 36 | }, 37 | { 38 | "containerPath": "es_data", 39 | "mode": "RW", 40 | "persistent": { 41 | "size": 500 42 | } 43 | }, 44 | { 45 | "containerPath": "/usr/share/elasticsearch/data", 46 | "hostPath": "es_data", 47 | "mode": "RW" 48 | }, 49 | { 50 | "containerPath": "redis_data", 51 | "mode": "RW", 52 | "persistent": { 53 | "size": 500 54 | } 55 | }, 56 | { 57 | "containerPath": "/var/lib/redis", 58 | "hostPath": "redis_data", 59 | "mode": "RW" 60 | }, 61 | { 62 | "containerPath": "kafka-log-dir", 63 | "mode": "RW", 64 | "persistent": { 65 | "size": 500 66 | } 67 | }, 68 | { 69 | "containerPath": "/opt/netsil/kafka/kafka-log-dir", 70 | "hostPath": "kafka-log-dir", 71 | "mode": "RW" 72 | }, 73 | { 74 | "containerPath": "realtime-segments", 75 | "mode": "RW", 76 | "persistent": { 77 | "size": 500 78 | } 79 | }, 80 | { 81 | "containerPath": "/opt/netsil/druid/realtime-segments", 82 | "hostPath": "realtime-segments", 83 | "mode": "RW" 84 | }, 85 | { 86 | "containerPath": "ceph-data", 87 | "mode": "RW", 88 | "persistent": { 89 | "size": 500 90 | } 91 | }, 92 | { 93 | "containerPath": "/var/lib/ceph-data", 94 | "hostPath": "ceph-data", 95 | "mode": "RW" 96 | }, 97 | { 98 | "containerPath": "ceph-conf", 99 | "mode": "RW", 100 | "persistent": { 101 | "size": 500 102 | } 103 | }, 104 | { 105 | "containerPath": "/etc/ceph", 106 | "hostPath": "ceph-conf", 107 | "mode": "RW" 108 | }, 109 | { 110 | "containerPath": "licenses", 111 | "mode": "RW", 112 | "persistent": { 113 | "size": 10 114 | } 115 | }, 116 | { 117 | "containerPath": "/opt/netsil/license-manager/licenses", 118 | "hostPath": "licenses", 119 | "mode": "RW" 120 | }, 121 | { 122 | "containerPath": "indexCache", 123 | "mode": "RW", 124 | "persistent": { 125 | "size": 10 126 | } 127 | }, 128 | { 129 | "containerPath": "/var/tmp/druid/indexCache", 130 | "hostPath": "indexCache", 131 | "mode": "RW" 132 | } 133 | ], 134 | "docker": { 135 | "image": "netsil/netsil:stable-latest", 136 | "network": "BRIDGE", 137 | "portMappings": [ 138 | { 139 | "containerPort": 80, 140 | "hostPort": 30443, 141 | "protocol": "tcp" 142 | }, 143 | { 144 | "containerPort": 2000, 145 | "hostPort": 2000, 146 | "protocol": "tcp" 147 | }, 148 | { 149 | "containerPort": 2001, 150 | "hostPort": 2001, 151 | "protocol": "tcp" 152 | }, 153 | { 154 | "containerPort": 2003, 155 | "hostPort": 2003, 156 | "protocol": "udp,tcp" 157 | } 158 | ], 159 | "privileged": false, 160 | "parameters": [], 161 | "forcePullImage": false 162 | } 163 | }, 164 | "args":[ 165 | "/root/startup.sh" 166 | ], 167 | "labels": { 168 | "DCOS_SERVICE_NAME": "netsil", 169 | "DCOS_SERVICE_SCHEME": "http", 170 | "DCOS_SERVICE_PORT_INDEX": "0" 171 | }, 172 | "residency": { 173 | "taskLostBehavior": "WAIT_FOREVER", 174 | "relaunchEscalationTimeoutSeconds": 100 175 | }, 176 | "upgradeStrategy": { 177 | "minimumHealthCapacity": 0, 178 | "maximumOverCapacity": 0 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /kubernetes/netsil.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: netsil 5 | --- 6 | apiVersion: v1 7 | kind: ReplicationController 8 | metadata: 9 | namespace: netsil 10 | name: netsil 11 | spec: 12 | replicas: 1 13 | selector: 14 | app: netsil 15 | component: netsil 16 | template: 17 | metadata: 18 | name: stream-processor 19 | labels: 20 | app: netsil 21 | component: netsil 22 | version: stable 23 | spec: 24 | containers: 25 | - name: netsil 26 | image: netsil/netsil:latest 27 | command: 28 | - /root/startup.sh 29 | ports: 30 | - containerPort: 80 31 | - containerPort: 443 32 | - containerPort: 2000 33 | - containerPort: 2001 34 | - containerPort: 2003 35 | - containerPort: 2003 36 | protocol: UDP 37 | volumeMounts: 38 | - name: alerts 39 | mountPath: /opt/netsil/alerts/data 40 | - name: ceph-conf 41 | mountPath: /etc/ceph 42 | - name: ceph-data 43 | mountPath: /var/lib/ceph-data 44 | - name: druid-index-cache 45 | mountPath: /var/tmp/druid/indexCache 46 | - name: druid-realtime-segments 47 | mountPath: /opt/netsil/druid/realtime-segments 48 | - name: elasticsearch 49 | mountPath: /usr/share/elasticsearch/data 50 | - name: kafka-logs 51 | mountPath: /opt/netsil/kafka/kafka-log-dir 52 | - name: licenses 53 | mountPath: /opt/netsil/lite/license-manager/licenses 54 | - name: mysql 55 | mountPath: /var/lib/mysql 56 | - name: redis 57 | mountPath: /var/lib/redis 58 | - name: user-persistence 59 | mountPath: /opt/netsil/user-persistence/data 60 | - name: zookeeper 61 | mountPath: /opt/netsil/zookeeper/data 62 | volumes: 63 | - name: alerts 64 | hostPath: 65 | path: /opt/netsil/lite/alerts/data 66 | - name: ceph-conf 67 | hostPath: 68 | path: /opt/netsil/lite/ceph/conf 69 | - name: ceph-data 70 | hostPath: 71 | path: /opt/netsil/lite/ceph/data 72 | - name: druid-index-cache 73 | hostPath: 74 | path: /var/tmp/druid/indexCache 75 | - name: druid-realtime-segments 76 | hostPath: 77 | path: /opt/netsil/lite/druid/realtime-segments 78 | - name: elasticsearch 79 | hostPath: 80 | path: /opt/netsil/lite/elasticsearch/data 81 | - name: licenses 82 | hostPath: 83 | path: /opt/netsil/lite/license-manager/licenses 84 | - name: kafka-logs 85 | hostPath: 86 | path: /opt/netsil/lite/kafka/kafka-log-dir 87 | - name: mysql 88 | hostPath: 89 | path: /opt/netsil/lite/mysql/data 90 | - name: redis 91 | hostPath: 92 | path: /opt/netsil/lite/redis 93 | - name: user-persistence 94 | hostPath: 95 | path: /opt/netsil/lite/user-persistence/data 96 | - name: zookeeper 97 | hostPath: 98 | path: /opt/netsil/lite/zookeeper/data 99 | --- 100 | apiVersion: v1 101 | kind: Service 102 | metadata: 103 | namespace: netsil 104 | name: netsil 105 | spec: 106 | selector: 107 | app: netsil 108 | component: netsil 109 | ports: 110 | - name: http 111 | port: 80 112 | - name: https 113 | port: 443 114 | nodePort: 30443 115 | - name: port2000 116 | port: 2000 117 | - name: port2001 118 | port: 2001 119 | - name: port2003 120 | port: 2003 121 | - name: port2003udp 122 | port: 2003 123 | protocol: UDP 124 | type: NodePort 125 | --- 126 | apiVersion: extensions/v1beta1 127 | kind: DaemonSet 128 | metadata: 129 | namespace: netsil 130 | name: collector 131 | labels: 132 | app: netsil 133 | component: collector 134 | spec: 135 | template: 136 | metadata: 137 | labels: 138 | app: netsil 139 | component: collector 140 | spec: 141 | hostNetwork: true 142 | containers: 143 | - name: collector 144 | image: netsil/collectors:latest 145 | command: ["/bin/bash","-c","while true ; do NETSIL_SP_HOST=$NETSIL_SERVICE_HOST /opt/netsil/collectors/start.sh ; echo Exiting, possibly to upgrade ; sleep 5 ; done"] 146 | securityContext: 147 | capabilities: 148 | add: 149 | - NET_RAW 150 | - NET_ADMIN 151 | env: 152 | #- name: NETSIL_SERVICE_HOST 153 | # value: '' 154 | - name: NETSIL_TRAFFIC_PORT 155 | value: '2003' 156 | - name: NETSIL_INFRA_PORT 157 | value: '2001' 158 | - name: DEPLOY_ENV 159 | value: 'docker' 160 | - name: SAMPLINGRATE 161 | value: '32' 162 | volumeMounts: 163 | - name: cgroup 164 | mountPath: /host/sys/fs/cgroup/ 165 | readOnly: true 166 | - name: proc 167 | mountPath: /host/proc/ 168 | readOnly: true 169 | - name: docker-sock 170 | mountPath: /var/run/docker.sock 171 | readOnly: true 172 | volumes: 173 | - name: cgroup 174 | hostPath: 175 | path: /sys/fs/cgroup/ 176 | - name: proc 177 | hostPath: 178 | path: /proc/ 179 | - name: docker-sock 180 | hostPath: 181 | path: /var/run/docker.sock 182 | -------------------------------------------------------------------------------- /cloudformation/netsil-cloudformation.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "AWS Cloudformation script to deploy Netsil AOC", 4 | "Parameters": { 5 | "InstanceType": { 6 | "Description": "Netsil AOC EC2 instance type", 7 | "Type": "String", 8 | "Default": "m4.2xlarge", 9 | "AllowedValues": [ 10 | "t2.xlarge", 11 | "t2.2xlarge", 12 | "m4.xlarge", 13 | "m4.2xlarge", 14 | "m4.4xlarge", 15 | "c4.2xlarge", 16 | "c4.4xlarge", 17 | "r4.xlarge", 18 | "r4.2xlarge" 19 | ], 20 | "ConstraintDescription": "Must be a valid EC2 instance type." 21 | }, 22 | "VolumeSize": { 23 | "Description": "Storage size of your instance. Must be at least 120 GB.", 24 | "Type": "Number", 25 | "MinValue": "120", 26 | "Default": "500", 27 | "ConstraintDescription": "Size must be greater than or equal to 120 GB." 28 | }, 29 | "KeyName": { 30 | "Description": "Name of an existing EC2 KeyPair to enable SSH access to the instances", 31 | "Type": "AWS::EC2::KeyPair::KeyName", 32 | "ConstraintDescription": "Must be the name of an existing EC2 KeyPair." 33 | }, 34 | "AssignPublicIP": { 35 | "Description": "You may want to choose if you intend to access Netsil AOC behind a load balancer.", 36 | "AllowedValues": ["true", "false"], 37 | "Default": "true", 38 | "Type": "String" 39 | }, 40 | "VPC": { 41 | "Description": "You must launch Netsil AOC in the same VPC as that of the applications you wish to monitor, or setup VPC peering when using a separate VPC.", 42 | "Type": "AWS::EC2::VPC::Id", 43 | "ConstraintDescription": "Must be the name of an existing VPC." 44 | }, 45 | "Subnet": { 46 | "Description": "Ensure that your subnet is within your chosen VPC.", 47 | "Type": "AWS::EC2::Subnet::Id", 48 | "ConstraintDescription": "Must be the name of an existing subnet that is within the VPC you selected." 49 | }, 50 | "AdminLocation": { 51 | "Description": "Specify a public IP address range to allow SSH and HTTP(S) access to the EC2 instance.", 52 | "Type": "String", 53 | "MinLength": "9", 54 | "MaxLength": "18", 55 | "Default": "0.0.0.0/0", 56 | "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})", 57 | "ConstraintDescription": "Must be a valid IP CIDR range of the form x.x.x.x/x." 58 | }, 59 | "SGsourceVpcCidr": { 60 | "Description": "Specify a private IP address range to restrict inbound traffic to the AOC from your application instances.", 61 | "Default": "0.0.0.0/0", 62 | "Type": "String", 63 | "MinLength": "9", 64 | "MaxLength": "18", 65 | "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})", 66 | "ConstraintDescription": "Must be a valid IP CIDR range of the form x.x.x.x/x." 67 | } 68 | }, 69 | "Mappings": { 70 | "AWSInstanceType2Arch": { 71 | "t2.xlarge": { 72 | "Arch": "HVM64" 73 | }, 74 | "t2.2xlarge": { 75 | "Arch": "HVM64" 76 | }, 77 | "m4.xlarge": { 78 | "Arch": "HVM64" 79 | }, 80 | "m4.2xlarge": { 81 | "Arch": "HVM64" 82 | }, 83 | "m4.4xlarge": { 84 | "Arch": "HVM64" 85 | }, 86 | "c4.2xlarge": { 87 | "Arch": "HVM64" 88 | }, 89 | "c4.4xlarge": { 90 | "Arch": "HVM64" 91 | }, 92 | "r4.xlarge": { 93 | "Arch": "HVM64" 94 | }, 95 | "r4.2xlarge": { 96 | "Arch": "HVM64" 97 | } 98 | }, 99 | "AWSRegionArch2AMI": { 100 | "us-east-1": { 101 | "HVM64": "ami-afe069d5" 102 | }, 103 | "us-east-2": { 104 | "HVM64": "ami-4ee6c82b" 105 | }, 106 | "us-west-1": { 107 | "HVM64": "ami-011f2761" 108 | }, 109 | "us-west-2": { 110 | "HVM64": "ami-40409238" 111 | }, 112 | "eu-west-1": { 113 | "HVM64": "ami-5a922623" 114 | }, 115 | "eu-central-1": { 116 | "HVM64": "ami-8229aaed" 117 | }, 118 | "ap-southeast-1": { 119 | "HVM64": "ami-90396ef3" 120 | }, 121 | "ap-northeast-1": { 122 | "HVM64": "ami-67803201" 123 | }, 124 | "ap-south-1": { 125 | "HVM64": "ami-e2c08f8d" 126 | }, 127 | "sa-east-1": { 128 | "HVM64": "ami-2c6e2b40" 129 | } 130 | } 131 | }, 132 | 133 | "Resources": { 134 | "NetsilAOC": { 135 | "Type": "AWS::EC2::Instance", 136 | "Properties": { 137 | "ImageId": { 138 | "Fn::FindInMap": [ 139 | "AWSRegionArch2AMI", 140 | { 141 | "Ref": "AWS::Region" 142 | }, 143 | { 144 | "Fn::FindInMap": [ 145 | "AWSInstanceType2Arch", 146 | { 147 | "Ref": "InstanceType" 148 | }, 149 | "Arch" 150 | ] 151 | } 152 | ] 153 | }, 154 | "InstanceType": { 155 | "Ref": "InstanceType" 156 | }, 157 | "NetworkInterfaces": [ 158 | { 159 | "GroupSet": [ 160 | { 161 | "Ref": "NetsilAOCPrivateSecurityGroup" 162 | }, 163 | { 164 | "Ref": "NetsilAOCPublicSecurityGroup" 165 | } 166 | ], 167 | "AssociatePublicIpAddress": { "Ref": "AssignPublicIP" }, 168 | "DeviceIndex": "0", 169 | "DeleteOnTermination": "true", 170 | "SubnetId": { 171 | "Ref": "Subnet" 172 | } 173 | } 174 | ], 175 | "KeyName": { 176 | "Ref": "KeyName" 177 | }, 178 | "BlockDeviceMappings": [ 179 | { 180 | "DeviceName": "/dev/xvda", 181 | "Ebs": { 182 | "VolumeType": "gp2", 183 | "VolumeSize": { "Ref": "VolumeSize" } 184 | } 185 | } 186 | ] 187 | }, 188 | "DependsOn": [ 189 | "NetsilAOCPublicSecurityGroup", 190 | "NetsilAOCPrivateSecurityGroup" 191 | ] 192 | }, 193 | 194 | "NetsilAOCPrivateSecurityGroup": { 195 | "Type": "AWS::EC2::SecurityGroup", 196 | "Properties": { 197 | "GroupDescription": "Private VPC between Netsil and Collectors.", 198 | "SecurityGroupIngress": [ 199 | { 200 | "IpProtocol": "tcp", 201 | "FromPort": "2001", 202 | "ToPort": "2001", 203 | "CidrIp": { 204 | "Ref": "SGsourceVpcCidr" 205 | } 206 | }, 207 | { 208 | "IpProtocol": "tcp", 209 | "FromPort": "2003", 210 | "ToPort": "2003", 211 | "CidrIp": { 212 | "Ref": "SGsourceVpcCidr" 213 | } 214 | }, 215 | { 216 | "IpProtocol": "udp", 217 | "FromPort": "2003", 218 | "ToPort": "2003", 219 | "CidrIp": { 220 | "Ref": "SGsourceVpcCidr" 221 | } 222 | }, 223 | { 224 | "IpProtocol": "tcp", 225 | "FromPort": "3003", 226 | "ToPort": "3003", 227 | "CidrIp": { 228 | "Ref": "SGsourceVpcCidr" 229 | } 230 | } 231 | ], 232 | "VpcId": {"Ref": "VPC"} 233 | } 234 | }, 235 | 236 | "NetsilAOCPublicSecurityGroup": { 237 | "Type": "AWS::EC2::SecurityGroup", 238 | "Properties": { 239 | "GroupDescription": "Ports for public access of Netsil AOC UI", 240 | "SecurityGroupIngress": [ 241 | { 242 | "IpProtocol": "tcp", 243 | "FromPort": "80", 244 | "ToPort": "80", 245 | "CidrIp": { 246 | "Ref": "AdminLocation" 247 | } 248 | }, 249 | { 250 | "IpProtocol": "tcp", 251 | "FromPort": "443", 252 | "ToPort": "443", 253 | "CidrIp": { 254 | "Ref": "AdminLocation" 255 | } 256 | }, 257 | { 258 | "IpProtocol": "tcp", 259 | "FromPort": "2000", 260 | "ToPort": "2000", 261 | "CidrIp": { 262 | "Ref": "AdminLocation" 263 | } 264 | }, 265 | { 266 | "IpProtocol": "tcp", 267 | "FromPort": "22", 268 | "ToPort": "22", 269 | "CidrIp": { 270 | "Ref": "AdminLocation" 271 | } 272 | } 273 | ], 274 | "VpcId": {"Ref": "VPC"} 275 | } 276 | } 277 | }, 278 | 279 | "Metadata" : { 280 | "AWS::CloudFormation::Interface" : { 281 | "ParameterGroups" : [ 282 | { 283 | "Label" : { "default":"Amazon EC2 Configuration" }, 284 | "Parameters" : [ "InstanceType", "VolumeSize", "KeyName" ] 285 | }, 286 | { 287 | "Label" : { "default" : "Network Configuration" }, 288 | "Parameters" : [ "VPC", "Subnet", "AssignPublicIP" ] 289 | }, 290 | { 291 | "Label" : { "default" : "Security Group Configuration" }, 292 | "Parameters" : [ "AdminLocation", "SGsourceVpcCidr" ] 293 | } 294 | ], 295 | "ParameterLabels" : { 296 | "VPC" : { "default" : "VPC to Deploy To" }, 297 | "Subnet" : { "default" : "Subnet to Deploy To" }, 298 | "AdminLocation": { "default": "Public CIDR Range" }, 299 | "SGsourceVpcCidr": { "default": "Private CIDR Range" }, 300 | "AssignPublicIP": { "default": "Assign a Public IP Address"} 301 | } 302 | } 303 | }, 304 | 305 | "Outputs": { 306 | "WebsiteURL": { 307 | "Value": { 308 | "Fn::Join": [ 309 | "", 310 | [ 311 | "https://", 312 | { 313 | "Fn::GetAtt": [ 314 | "NetsilAOC", 315 | "PublicDnsName" 316 | ] 317 | } 318 | ] 319 | ] 320 | }, 321 | "Description": "Address of the Netsil AOC Web UI. Please wait around ten minutes after stack creation for this link to be available." 322 | } 323 | } 324 | } 325 | -------------------------------------------------------------------------------- /kubernetes/kube-netsil/kube-netsil.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # 5 | # Script to automate AOC installation on a Kubernetes cluster 6 | # 7 | # Pre-requisites: 8 | # 1. In order to install AOC server, atleast one node in the k8s cluster should have minimum 4 CPU and 16 GB RAM 9 | # 2. kubectl should point to the cluster 10 | # 11 | 12 | cleanup(){ 13 | clear; 14 | 15 | echo "=========== Cluster Information ================="; 16 | kubectl cluster-info 17 | echo "PROMPT: Are you sure you want to remove AOC installation from this cluster [y/n][n]:" 18 | read choice; 19 | if [[ "${choice}" == "y" ]]; then 20 | kubectl delete daemonset collector -n netsil 21 | kubectl delete svc netsil -n netsil 22 | kubectl delete deploy netsil -n netsil 23 | kubectl delete ns netsil 24 | exit 0; 25 | fi 26 | } 27 | 28 | getNodeList(){ 29 | NodeList=$(kubectl get nodes --output=jsonpath={.items..metadata.name}) 30 | } 31 | 32 | checkdnsPolicy(){ 33 | KUBE_SERVER_VERSION=$(kubectl version | grep Server | awk -F "," '{print $2}' | cut -d":" -f2 | tr -cd '[:alnum:]') 34 | if [[ ${KUBE_SERVER_VERSION} -lt 6 ]]; then 35 | DISPLAY_VERSION_WARNING="1" 36 | dnsPolicy='Default' 37 | else 38 | DISPLAY_VERSION_WARNING="0" 39 | dnsPolicy='ClusterFirstWithHostNet' 40 | fi 41 | } 42 | 43 | checkCPURAM(){ 44 | 45 | PRINT=${1} 46 | getNodeList 47 | n=0 48 | for node in $NodeList 49 | do 50 | CPU_count=$(kubectl get nodes --output=jsonpath={.items[${n}].status.allocatable.cpu}) 51 | RAM_count=$(kubectl get nodes --output=jsonpath={.items[${n}].status.allocatable.memory}) 52 | # Removing the Ki in RAM count 53 | RAM_count=${RAM_count%??} 54 | if [[ "$PRINT" != "" ]]; then 55 | echo "${node} ${CPU_count} CPU(s) $(( ${RAM_count} / 1024 )) MB"; 56 | fi 57 | if [[ ${CPU_count} -gt 3 && ${RAM_count} -gt 15000000 ]]; then 58 | AOC_Server=${node} 59 | echo "INFO : AOC server will be installed on node $AOC_Server"; 60 | break; 61 | fi 62 | n=$(( $n + 1 )) 63 | done 64 | } 65 | 66 | checkns(){ 67 | # 68 | # Check for netsil namespace 69 | # Create namespace if it does not exist 70 | # 71 | nslist=$(kubectl get ns --output=jsonpath={.items..metadata.name}) 72 | for ns in $nslist 73 | do 74 | if [[ "$ns" == "netsil" ]]; then 75 | ns_found=True; 76 | fi 77 | done 78 | if [[ "${ns_found}" != "True" ]]; then 79 | kubectl create ns netsil 80 | fi 81 | } 82 | 83 | installAOCServer(){ 84 | 85 | # Delete deployment if it exists already 86 | kubectl get deployment netsil --namespace netsil > /dev/null 2>&1 && kubectl delete deployment netsil --namespace netsil 87 | # Generate AOC Server deployment 88 | cat <aoc-server-deployment.yaml 89 | apiVersion: extensions/v1beta1 90 | kind: Deployment 91 | metadata: 92 | namespace: netsil 93 | name: netsil 94 | spec: 95 | replicas: 1 96 | template: 97 | metadata: 98 | name: stream-processor 99 | labels: 100 | app: netsil 101 | component: netsil 102 | version: stable 103 | spec: 104 | containers: 105 | - name: netsil 106 | image: netsil/netsil:latest 107 | command: 108 | - /root/startup.sh 109 | ports: 110 | - containerPort: 80 111 | - containerPort: 443 112 | - containerPort: 2000 113 | - containerPort: 2001 114 | - containerPort: 2003 115 | - containerPort: 2003 116 | protocol: UDP 117 | volumeMounts: 118 | - name: alerts 119 | mountPath: /var/lib/netsil/alerts/data 120 | - name: ceph-conf 121 | mountPath: /etc/ceph 122 | - name: ceph-data 123 | mountPath: /var/lib/ceph-data 124 | - name: druid-index-cache 125 | mountPath: /var/tmp/druid/indexCache 126 | - name: druid-realtime-segments 127 | mountPath: /var/lib/netsil/druid/realtime-segments 128 | - name: elasticsearch 129 | mountPath: /usr/share/elasticsearch/data 130 | - name: kafka-logs 131 | mountPath: /var/lib/netsil/kafka/kafka-log-dir 132 | - name: licenses 133 | mountPath: /var/lib/netsil/lite/license-manager/licenses 134 | - name: mysql 135 | mountPath: /var/lib/mysql 136 | - name: redis 137 | mountPath: /var/lib/redis 138 | - name: user-persistence 139 | mountPath: /var/lib/netsil/user-persistence/data 140 | - name: zookeeper 141 | mountPath: /var/lib/netsil/zookeeper/data 142 | volumes: 143 | - name: alerts 144 | hostPath: 145 | path: /var/lib/netsil/lite/alerts/data 146 | - name: ceph-conf 147 | hostPath: 148 | path: /var/lib/netsil/lite/ceph/conf 149 | - name: ceph-data 150 | hostPath: 151 | path: /var/lib/netsil/lite/ceph/data 152 | - name: druid-index-cache 153 | hostPath: 154 | path: /var/tmp/druid/indexCache 155 | - name: druid-realtime-segments 156 | hostPath: 157 | path: /var/lib/netsil/lite/druid/realtime-segments 158 | - name: elasticsearch 159 | hostPath: 160 | path: /var/lib/netsil/lite/elasticsearch/data 161 | - name: licenses 162 | hostPath: 163 | path: /var/lib/netsil/lite/license-manager/licenses 164 | - name: kafka-logs 165 | hostPath: 166 | path: /var/lib/netsil/lite/kafka/kafka-log-dir 167 | - name: mysql 168 | hostPath: 169 | path: /var/lib/netsil/lite/mysql/data 170 | - name: redis 171 | hostPath: 172 | path: /var/lib/netsil/lite/redis 173 | - name: user-persistence 174 | hostPath: 175 | path: /var/lib/netsil/lite/user-persistence/data 176 | - name: zookeeper 177 | hostPath: 178 | path: /var/lib/netsil/lite/zookeeper/data 179 | nodeSelector: 180 | kubernetes.io/hostname: ${AOC_Server} 181 | EOF 182 | kubectl create -f aoc-server-deployment.yaml 183 | # 184 | # Check if AOC server is UP 185 | # 186 | n=1; 187 | while true 188 | do 189 | echo "INFO : Checking if netsil server is UP (Attempt $n). Recommended wait 10 attempts"; 190 | if [[ $(kubectl get deployment netsil -n netsil --output=jsonpath={.status.availableReplicas}) -eq 1 ]]; then 191 | server_running=true 192 | break 193 | else 194 | echo "INFO : Netsil server is not up yet. Sleeping 30 seconds. Press Ctrl + C to stop the loop."; 195 | sleep 30; 196 | fi 197 | n=$(( $n + 1 )); 198 | done 199 | if [[ "${server_running}" != "true" ]]; then 200 | echo "ERROR : Unable to start netsil server pod. Please check logs using: kubectl get pods -n netsil"; 201 | exit 1; 202 | fi 203 | # Install a service to expose AOC 204 | echo "INFO : Creating netsil service if it does not exist already"; 205 | cat <netsil-svc.yaml 206 | apiVersion: v1 207 | kind: Service 208 | metadata: 209 | namespace: netsil 210 | name: netsil 211 | spec: 212 | selector: 213 | app: netsil 214 | component: netsil 215 | ports: 216 | - name: http 217 | port: 80 218 | nodePort: 31000 219 | - name: https 220 | port: 443 221 | nodePort: 30443 222 | - name: port2000 223 | port: 2000 224 | - name: port2001 225 | port: 2001 226 | - name: port2003 227 | port: 2003 228 | - name: port2003udp 229 | port: 2003 230 | protocol: UDP 231 | type: NodePort 232 | EOF 233 | kubectl get svc netsil --namespace netsil > /dev/null 2>&1 || kubectl create -f netsil-svc.yaml 234 | if [[ "${SERVICE_TYPE}" == "LoadBalancer" ]]; then 235 | # Generate LB service file 236 | cat <netsil-svc-lb.yaml 237 | apiVersion: v1 238 | kind: Service 239 | metadata: 240 | namespace: netsil 241 | name: netsil-lb 242 | spec: 243 | selector: 244 | app: netsil 245 | component: netsil 246 | ports: 247 | - name: http 248 | port: 80 249 | type: LoadBalancer 250 | EOF 251 | kubectl create -f netsil-svc-lb.yaml 252 | fi 253 | } 254 | 255 | installAOCCollectors() 256 | { 257 | echo "INFO : Installing netsil collectors via DaemonSet" 258 | cat <aoc-collectors-daemonset.yaml 259 | apiVersion: extensions/v1beta1 260 | kind: DaemonSet 261 | metadata: 262 | namespace: netsil 263 | name: collector 264 | labels: 265 | app: netsil 266 | component: collector 267 | spec: 268 | EOF 269 | if [[ "${dnsPolicy}" == "ClusterFirstWithHostNet" ]]; then 270 | cat <>aoc-collectors-daemonset.yaml 271 | minReadySeconds: 0 272 | updateStrategy: 273 | type: RollingUpdate 274 | rollingUpdate: 275 | maxUnavailable: 1 276 | EOF 277 | fi 278 | cat <>aoc-collectors-daemonset.yaml 279 | template: 280 | metadata: 281 | labels: 282 | app: netsil 283 | component: collector 284 | spec: 285 | hostNetwork: true 286 | dnsPolicy: ${dnsPolicy} 287 | containers: 288 | - name: collector 289 | image: netsil/collectors:latest 290 | command: ["/bin/bash","-c","while true ; do NETSIL_SP_HOST=\$NETSIL_SERVICE_HOST /opt/netsil/collectors/start.sh ; echo Exiting, possibly to upgrade ; sleep 5 ; done"] 291 | securityContext: 292 | capabilities: 293 | add: 294 | - NET_RAW 295 | - NET_ADMIN 296 | env: 297 | - name: DEPLOY_ENV 298 | value: 'docker' 299 | - name: SAMPLINGRATE 300 | value: '100' 301 | - name: KUBERNETES 302 | value: "yes" 303 | - name: SD_BACKEND 304 | value: docker 305 | volumeMounts: 306 | - name: cgroup 307 | mountPath: /host/sys/fs/cgroup/ 308 | readOnly: true 309 | - name: proc 310 | mountPath: /host/proc/ 311 | readOnly: true 312 | - name: docker-sock 313 | mountPath: /var/run/docker.sock 314 | readOnly: true 315 | volumes: 316 | - name: cgroup 317 | hostPath: 318 | path: /sys/fs/cgroup/ 319 | - name: proc 320 | hostPath: 321 | path: /proc/ 322 | - name: docker-sock 323 | hostPath: 324 | path: /var/run/docker.sock 325 | EOF 326 | kubectl get daemonset collector -n netsil > /dev/null 2>&1 || kubectl create -f aoc-collectors-daemonset.yaml 327 | for n in {1..5} 328 | do 329 | desired=$(kubectl get daemonset collector -n netsil --output=jsonpath={.status.desiredNumberScheduled}) 330 | avail=$(kubectl get daemonset collector -n netsil --output=jsonpath={.status.numberAvailable}) 331 | if [[ $desired -ne $avail ]]; then 332 | echo "INFO : Checking if all collectors are UP (Attempt $n of 5)"; 333 | echo "INFO : Sleeping 10 seconds" 334 | sleep 10; 335 | else 336 | collector_running=true; 337 | break; 338 | fi 339 | done 340 | 341 | if [[ "${collector_running}" != "true" ]]; then 342 | echo "INFO : Collectors started successfully."; 343 | fi 344 | 345 | } 346 | 347 | displayInfo(){ 348 | 349 | if [[ "${SERVICE_TYPE}" == "LoadBalancer" ]]; then 350 | echo "INFO : Waiting for LB IP to be allocated."; 351 | while [[ "${LB_IP}" == "" ]]; 352 | do 353 | LB_IP=$(kubectl get svc netsil-lb -n netsil --output=jsonpath={.status.loadBalancer.ingress[0].ip}); 354 | echo "INFO : Sleeping 10 seconds to get LB IP."; 355 | sleep 10; 356 | done 357 | 358 | fi 359 | 360 | if [[ "${DISPLAY_VERSION_WARNING}" == "1" ]] ; then 361 | echo "=========================================================================="; 362 | echo 'Warning: Unable to set dnsPolicy to ClusterFirstWithHostNet because your kubernetes server version is too low.' 363 | echo 'Certain service integrations may be unavailable in the AOC because of this.' 364 | echo 'As a workaround, you may redeploy your collectors with the variables: ' 365 | echo 'OVERWRITE_RESOLVCONF="yes" and K8S_NAMESERVER=""' 366 | fi 367 | 368 | 369 | echo "=========================================================================="; 370 | echo "Netsil AOC and collectors are installed now."; 371 | if [[ "${SERVICE_TYPE}" == "LoadBalancer" ]]; then 372 | echo "INFO : AOC server is available on Load Balancer IP $LB_IP"; 373 | else 374 | node_ips=$(kubectl get nodes --output=jsonpath={.items..status.addresses[1].address}) 375 | echo "INFO : Node IPs ${node_ips}"; 376 | echo "INFO : AOC server is avaialble on Port 31000 [HTTP] and Port 30443 [HTTPS] of above IPs"; 377 | echo "INFO : Please ensure this port is open in firewall"; 378 | fi 379 | 380 | echo "=========================================================================="; 381 | 382 | } 383 | 384 | main(){ 385 | clear; 386 | 387 | echo "=========== Cluster Information ================="; 388 | kubectl cluster-info 389 | echo "PRE-REQS :"; 390 | echo "AOC server needs at least one node with min 4 CPU and 15GB RAM"; 391 | echo "ALL nodes need internet connectivity to pull images from Docker Hub" 392 | echo "PROMPT : Please confirm that you want to install AOC on above cluster (y/n)[y]:"; 393 | read choice; 394 | if [[ "${choice}" != "y" ]] && [[ "${choice}" != "" ]] ; then 395 | exit 1; 396 | fi 397 | 398 | # 399 | # Handling Service type by prompting user 400 | # 401 | echo "============================================================="; 402 | echo "INFO : This automation does not support ingress controller."; 403 | echo "PROMPT : Default Service type is NodePort. Type yes if you want a LoadBalancer instead (y/n)[n]:"; 404 | read choice; 405 | if [[ "${choice}" == "y" ]]; then 406 | SERVICE_TYPE=LoadBalancer 407 | else 408 | SERVICE_TYPE=NodePort 409 | fi 410 | # 411 | # Start checking for resources 412 | echo "INFO : Checking for a node that meets server requirements." 413 | checkCPURAM 414 | if [[ "${AOC_Server}" == "" ]]; then 415 | echo "ERROR: Could not find a node which has at least 4 CPU and 15GB RAM. Below are Nodes with their CPU and RAM values:"; 416 | checkCPURAM print 417 | exit 1 418 | fi 419 | checkns 420 | installAOCServer 421 | checkdnsPolicy 422 | installAOCCollectors 423 | displayInfo 424 | } 425 | 426 | if [[ "$1" == "delete" ]]; then 427 | cleanup 428 | fi 429 | main 430 | 431 | 432 | 433 | --------------------------------------------------------------------------------