├── .gitignore ├── minikube.png ├── cluster-guide ├── nifi.png ├── kubectl-logs.png ├── port-forward.png ├── kubectl-get-po.png ├── minikube-start.png ├── helm-install-zookeeper.png ├── nifi-service-externalname-2.yaml └── README.md ├── nifi-frontend ├── Dockerfile └── frontend.conf ├── nifi-service.yaml ├── nifi ├── start-patched.sh └── Dockerfile ├── .circleci └── config.yml ├── nifi.yaml ├── frontend.yaml ├── README.md ├── nifi-statefulset-aks.yaml ├── nifi-statefulset-minikube.yaml ├── nifi-statefulset-minikube-cluster.yaml └── nifi-statefulset-aks-zookeeper.yaml /.gitignore: -------------------------------------------------------------------------------- 1 | nifi-statefulset.yaml 2 | 3 | docker-compose.yaml 4 | 5 | conf -------------------------------------------------------------------------------- /minikube.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whs-dot-hk/kubernetes-nifi-refined/HEAD/minikube.png -------------------------------------------------------------------------------- /cluster-guide/nifi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whs-dot-hk/kubernetes-nifi-refined/HEAD/cluster-guide/nifi.png -------------------------------------------------------------------------------- /nifi-frontend/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nginx 2 | 3 | COPY frontend.conf /etc/nginx/conf.d/frontend.conf 4 | 5 | EXPOSE 8080 -------------------------------------------------------------------------------- /cluster-guide/kubectl-logs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whs-dot-hk/kubernetes-nifi-refined/HEAD/cluster-guide/kubectl-logs.png -------------------------------------------------------------------------------- /cluster-guide/port-forward.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whs-dot-hk/kubernetes-nifi-refined/HEAD/cluster-guide/port-forward.png -------------------------------------------------------------------------------- /cluster-guide/kubectl-get-po.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whs-dot-hk/kubernetes-nifi-refined/HEAD/cluster-guide/kubectl-get-po.png -------------------------------------------------------------------------------- /cluster-guide/minikube-start.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whs-dot-hk/kubernetes-nifi-refined/HEAD/cluster-guide/minikube-start.png -------------------------------------------------------------------------------- /cluster-guide/helm-install-zookeeper.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whs-dot-hk/kubernetes-nifi-refined/HEAD/cluster-guide/helm-install-zookeeper.png -------------------------------------------------------------------------------- /nifi-frontend/frontend.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 8080; 3 | 4 | location / { 5 | proxy_pass http://nifi:8080; 6 | proxy_set_header Host $http_host; 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /cluster-guide/nifi-service-externalname-2.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: nifi 5 | spec: 6 | type: ExternalName 7 | externalName: nifi-2.nifi-headless.default.svc.cluster.local 8 | -------------------------------------------------------------------------------- /nifi-service.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: nifi 5 | spec: 6 | selector: 7 | app: nifi 8 | tier: backend 9 | ports: 10 | - protocol: TCP 11 | port: 8080 12 | targetPort: http 13 | -------------------------------------------------------------------------------- /nifi/start-patched.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | [ ! -f conf/nifi.properties ] && cp -a nifi-1.8.0/conf . 4 | 5 | [ ! -z $KUBERNETES_HEADLESS_SERVICE_NAME ] && HOSTNAME=$HOSTNAME.$KUBERNETES_HEADLESS_SERVICE_NAME 6 | 7 | ../scripts/start.sh 8 | -------------------------------------------------------------------------------- /nifi/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM apache/nifi:latest 2 | 3 | RUN mkdir nifi-1.8.0 && cp -a conf nifi-1.8.0/conf 4 | 5 | COPY --chown=nifi:nifi start-patched.sh ../scripts 6 | 7 | RUN chmod a+x ../scripts/start-patched.sh 8 | 9 | ENTRYPOINT ../scripts/start-patched.sh 10 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build: 4 | machine: true 5 | steps: 6 | - checkout 7 | - run: docker build -t whshk/nifi-frontend ./nifi-frontend 8 | - run: docker build -t whshk/nifi ./nifi 9 | - run: docker login -u $DOCKER_USER -p $DOCKER_PASS 10 | - run: docker push whshk/nifi-frontend 11 | - run: docker push whshk/nifi 12 | 13 | -------------------------------------------------------------------------------- /nifi.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: nifi 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: nifi 9 | tier: backend 10 | track: stable 11 | replicas: 1 12 | template: 13 | metadata: 14 | labels: 15 | app: nifi 16 | tier: backend 17 | track: stable 18 | spec: 19 | containers: 20 | - name: nifi 21 | image: apache/nifi:latest 22 | ports: 23 | - name: http 24 | containerPort: 8080 25 | -------------------------------------------------------------------------------- /frontend.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: nifi-frontend 5 | spec: 6 | selector: 7 | app: nifi 8 | tier: frontend 9 | ports: 10 | - protocol: "TCP" 11 | port: 8080 12 | targetPort: 8080 13 | --- 14 | apiVersion: apps/v1 15 | kind: Deployment 16 | metadata: 17 | name: nifi-frontend 18 | spec: 19 | selector: 20 | matchLabels: 21 | app: nifi-frontend 22 | tier: frontend 23 | track: stable 24 | replicas: 1 25 | template: 26 | metadata: 27 | labels: 28 | app: nifi-frontend 29 | tier: frontend 30 | track: stable 31 | spec: 32 | containers: 33 | - name: nginx 34 | image: "whshk/nifi-frontend:latest" 35 | imagePullPolicy: Always 36 | lifecycle: 37 | preStop: 38 | exec: 39 | command: ["/usr/sbin/nginx","-s","quit"] 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![CircleCI](https://circleci.com/gh/whs-dot-hk/kubernetes-nifi-refined.svg?style=shield)](https://circleci.com/gh/whs-dot-hk/kubernetes-nifi-refined) 2 | 3 | For clusters, check out our [cluster guide](https://github.com/whs-dot-hk/kubernetes-nifi-refined/tree/master/cluster-guide). 4 | 5 | ## Standard Usage 6 | [![asciicast](https://asciinema.org/a/OYN3fdGCBOtoDqCZ4UKc5DjWK.svg)](https://asciinema.org/a/OYN3fdGCBOtoDqCZ4UKc5DjWK) 7 | 8 | ``` 9 | $ kubectl create -f https://raw.githubusercontent.com/whs-dot-hk/kubernetes-nifi-refined/master/nifi.yaml 10 | $ kubectl create -f https://raw.githubusercontent.com/whs-dot-hk/kubernetes-nifi-refined/master/nifi-service.yaml 11 | ``` 12 | 13 | ``` 14 | $ kubectl create -f https://raw.githubusercontent.com/whs-dot-hk/kubernetes-nifi-refined/master/frontend.yaml 15 | ``` 16 | 17 | ``` 18 | $ export POD_NAME=$(kubectl get pods --namespace default -l "app=nifi-frontend" -o jsonpath="{.items[0].metadata.name}") 19 | $ kubectl port-forward $POD_NAME 8080:8080 20 | ``` 21 | 22 | Visit http://localhost:8080/nifi 23 | 24 | ## Minikube StatefulSet Example 25 | ![alt text](minikube.png "Minikube") 26 | 27 | ``` 28 | $ kubectl create -f https://raw.githubusercontent.com/whs-dot-hk/kubernetes-nifi-refined/master/nifi-statefulset-minikube.yaml 29 | $ kubectl create -f https://raw.githubusercontent.com/whs-dot-hk/kubernetes-nifi-refined/master/nifi-service.yaml 30 | $ kubectl create -f https://raw.githubusercontent.com/whs-dot-hk/kubernetes-nifi-refined/master/frontend.yaml 31 | ``` 32 | 33 | ## Other Example 34 | https://bitbucket.org/whshk/kubernetes-logging/src/master 35 | -------------------------------------------------------------------------------- /cluster-guide/README.md: -------------------------------------------------------------------------------- 1 | ## Start Minikube 2 | ![alt text](minikube-start.png "Start Minikube") 3 | ``` 4 | $ minikube config set memory 10240 5 | $ minikube start 6 | ``` 7 | 8 | ## Install Zookeeper 9 | ![alt text](helm-install-zookeeper.png "Install Zookeeper") 10 | ``` 11 | $ helm init 12 | $ helm install bitnami/zookeeper \ 13 | --set replicaCount=3 14 | ``` 15 | > You may need to run `helm repo add bitnami https://charts.bitnami.com` 16 | > to add bitnami's repo. 17 | 18 | ## Deploy Nifi 19 | You will need to modify 20 | [line 43](https://github.com/whs-dot-hk/kubernetes-nifi-refined/blob/a6497b626e965b97c1ba8a73bbaf83ae55491cd2/nifi-statefulset-minikube-cluster.yaml#L43) 21 | to be your zookeeper service name. 22 | Here `singed-quoll-zookeeper` is our zookeeper service name (See 23 | screenshot above). 24 | ``` 25 | $ wget https://raw.githubusercontent.com/whs-dot-hk/kubernetes-nifi-refined/master/nifi-statefulset-minikube-cluster.yaml 26 | ``` 27 | Edit line 43. 28 | ``` 29 | $ kubectl create -f nifi-statefulset-minikube-cluster.yaml 30 | ``` 31 | ![alt text](kubectl-get-po.png "Get Pods") 32 | 33 | ## Deploy Frontend 34 | ![alt text](kubectl-logs.png "Nifi-0 Logs") 35 | Here, we run `kubectl logs nifi-0` and found the primary node `nifi-2`. 36 | So we deploy the nifi service with external name `nifi-2.nifi-headless.default.svc.cluster.local`. 37 | 38 | ``` 39 | $ kubectl create -f https://raw.githubusercontent.com/whs-dot-hk/kubernetes-nifi-refined/master/cluster-guide/nifi-service-externalname-2.yaml 40 | $ kubectl create -f https://raw.githubusercontent.com/whs-dot-hk/kubernetes-nifi-refined/master/frontend.yaml 41 | ``` 42 | 43 | ## Port Forward Frontend 44 | ![alt text](port-forward.png "Port Forward Frontend") 45 | 46 | ## Enjoy 47 | ![alt text](nifi.png "Nifi") 48 | -------------------------------------------------------------------------------- /nifi-statefulset-aks.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: nifi-headless 5 | labels: 6 | app: nifi 7 | spec: 8 | ports: 9 | - port: 8080 10 | name: http 11 | clusterIP: None 12 | selector: 13 | app: nifi 14 | --- 15 | apiVersion: apps/v1 16 | kind: StatefulSet 17 | metadata: 18 | name: nifi 19 | spec: 20 | selector: 21 | matchLabels: 22 | app: nifi 23 | tier: backend 24 | track: stable 25 | serviceName: nifi-headless 26 | replicas: 1 27 | template: 28 | metadata: 29 | labels: 30 | app: nifi 31 | tier: backend 32 | track: stable 33 | spec: 34 | containers: 35 | - name: nifi 36 | image: whshk/nifi:latest 37 | ports: 38 | - name: http 39 | containerPort: 8080 40 | volumeMounts: 41 | - name: nifi-data 42 | mountPath: /opt/nifi/nifi-current/conf 43 | subPath: conf 44 | - name: nifi-data 45 | mountPath: /opt/nifi/nifi-current/logs 46 | subPath: logs 47 | - name: nifi-data 48 | mountPath: /opt/nifi/nifi-current/database_repository 49 | subPath: database_repository 50 | - name: nifi-data 51 | mountPath: /opt/nifi/nifi-current/flowfile_repository 52 | subPath: flowfile_repository 53 | - name: nifi-data 54 | mountPath: /opt/nifi/nifi-current/content_repository 55 | subPath: content_repository 56 | - name: nifi-data 57 | mountPath: /opt/nifi/nifi-current/provenance_repository 58 | subPath: provenance_repository 59 | - name: nifi-data 60 | mountPath: /opt/nifi/nifi-current/state 61 | subPath: state 62 | securityContext: 63 | runAsUser: 1000 64 | fsGroup: 1000 65 | volumeClaimTemplates: 66 | - metadata: 67 | name: nifi-data 68 | spec: 69 | accessModes: [ "ReadWriteOnce" ] 70 | storageClassName: "default" 71 | resources: 72 | requests: 73 | storage: 10Gi 74 | -------------------------------------------------------------------------------- /nifi-statefulset-minikube.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: nifi-headless 5 | labels: 6 | app: nifi 7 | spec: 8 | ports: 9 | - port: 8080 10 | name: http 11 | clusterIP: None 12 | selector: 13 | app: nifi 14 | --- 15 | apiVersion: apps/v1 16 | kind: StatefulSet 17 | metadata: 18 | name: nifi 19 | spec: 20 | selector: 21 | matchLabels: 22 | app: nifi 23 | tier: backend 24 | track: stable 25 | serviceName: nifi-headless 26 | replicas: 1 27 | template: 28 | metadata: 29 | labels: 30 | app: nifi 31 | tier: backend 32 | track: stable 33 | spec: 34 | containers: 35 | - name: nifi 36 | image: whshk/nifi:latest 37 | ports: 38 | - name: http 39 | containerPort: 8080 40 | volumeMounts: 41 | - name: nifi-data 42 | mountPath: /opt/nifi/nifi-current/conf 43 | subPath: conf 44 | - name: nifi-data 45 | mountPath: /opt/nifi/nifi-current/logs 46 | subPath: logs 47 | - name: nifi-data 48 | mountPath: /opt/nifi/nifi-current/database_repository 49 | subPath: database_repository 50 | - name: nifi-data 51 | mountPath: /opt/nifi/nifi-current/flowfile_repository 52 | subPath: flowfile_repository 53 | - name: nifi-data 54 | mountPath: /opt/nifi/nifi-current/content_repository 55 | subPath: content_repository 56 | - name: nifi-data 57 | mountPath: /opt/nifi/nifi-current/provenance_repository 58 | subPath: provenance_repository 59 | - name: nifi-data 60 | mountPath: /opt/nifi/nifi-current/state 61 | subPath: state 62 | securityContext: 63 | runAsUser: 1000 64 | fsGroup: 1000 65 | volumeClaimTemplates: 66 | - metadata: 67 | name: nifi-data 68 | spec: 69 | accessModes: [ "ReadWriteOnce" ] 70 | storageClassName: "standard" 71 | resources: 72 | requests: 73 | storage: 10Gi 74 | -------------------------------------------------------------------------------- /nifi-statefulset-minikube-cluster.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: nifi-headless 5 | labels: 6 | app: nifi 7 | spec: 8 | ports: 9 | - port: 8080 10 | name: http 11 | clusterIP: None 12 | selector: 13 | app: nifi 14 | --- 15 | apiVersion: apps/v1 16 | kind: StatefulSet 17 | metadata: 18 | name: nifi 19 | spec: 20 | selector: 21 | matchLabels: 22 | app: nifi 23 | tier: backend 24 | track: stable 25 | serviceName: nifi-headless 26 | replicas: 3 27 | template: 28 | metadata: 29 | labels: 30 | app: nifi 31 | tier: backend 32 | track: stable 33 | spec: 34 | containers: 35 | - name: nifi 36 | image: whshk/nifi 37 | env: 38 | - name: NIFI_CLUSTER_IS_NODE 39 | value: "true" 40 | - name: NIFI_CLUSTER_NODE_PROTOCOL_PORT 41 | value: "2882" 42 | - name: NIFI_ZK_CONNECT_STRING 43 | value: singed-quoll-zookeeper 44 | - name: KUBERNETES_HEADLESS_SERVICE_NAME 45 | value: nifi-headless 46 | ports: 47 | - name: http 48 | containerPort: 8080 49 | - name: protocol 50 | containerPort: 2882 51 | volumeMounts: 52 | - name: nifi-data 53 | mountPath: /opt/nifi/nifi-current/conf 54 | subPath: conf 55 | - name: nifi-data 56 | mountPath: /opt/nifi/nifi-current/logs 57 | subPath: logs 58 | - name: nifi-data 59 | mountPath: /opt/nifi/nifi-current/database_repository 60 | subPath: database_repository 61 | - name: nifi-data 62 | mountPath: /opt/nifi/nifi-current/flowfile_repository 63 | subPath: flowfile_repository 64 | - name: nifi-data 65 | mountPath: /opt/nifi/nifi-current/content_repository 66 | subPath: content_repository 67 | - name: nifi-data 68 | mountPath: /opt/nifi/nifi-current/provenance_repository 69 | subPath: provenance_repository 70 | - name: nifi-data 71 | mountPath: /opt/nifi/nifi-current/state 72 | subPath: state 73 | securityContext: 74 | runAsUser: 1000 75 | fsGroup: 1000 76 | volumeClaimTemplates: 77 | - metadata: 78 | name: nifi-data 79 | spec: 80 | accessModes: [ "ReadWriteOnce" ] 81 | storageClassName: "standard" 82 | resources: 83 | requests: 84 | storage: 10Gi 85 | -------------------------------------------------------------------------------- /nifi-statefulset-aks-zookeeper.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: nifi-headless 5 | labels: 6 | app: nifi 7 | spec: 8 | ports: 9 | - port: 8080 10 | name: http 11 | clusterIP: None 12 | selector: 13 | app: nifi 14 | --- 15 | apiVersion: apps/v1 16 | kind: StatefulSet 17 | metadata: 18 | name: nifi 19 | spec: 20 | selector: 21 | matchLabels: 22 | app: nifi 23 | tier: backend 24 | track: stable 25 | serviceName: nifi-headless 26 | replicas: 2 27 | template: 28 | metadata: 29 | labels: 30 | app: nifi 31 | tier: backend 32 | track: stable 33 | spec: 34 | containers: 35 | - name: nifi 36 | image: whshk/nifi 37 | env: 38 | - name: NIFI_CLUSTER_IS_NODE 39 | value: "true" 40 | - name: NIFI_CLUSTER_NODE_PROTOCOL_PORT 41 | value: "2882" 42 | - name: NIFI_ZK_CONNECT_STRING 43 | value: vocal-whippet-zookeeper 44 | - name: KUBERNETES_HEADLESS_SERVICE_NAME 45 | value: nifi-headless 46 | resources: 47 | requests: 48 | memory: "1Gi" 49 | cpu: "250m" 50 | limits: 51 | memory: "1.5Gi" 52 | cpu: "500m" 53 | ports: 54 | - name: http 55 | containerPort: 8080 56 | - name: protocol 57 | containerPort: 2882 58 | volumeMounts: 59 | - name: nifi-data 60 | mountPath: /opt/nifi/nifi-current/conf 61 | subPath: conf 62 | - name: nifi-data 63 | mountPath: /opt/nifi/nifi-current/logs 64 | subPath: logs 65 | - name: nifi-data 66 | mountPath: /opt/nifi/nifi-current/database_repository 67 | subPath: database_repository 68 | - name: nifi-data 69 | mountPath: /opt/nifi/nifi-current/flowfile_repository 70 | subPath: flowfile_repository 71 | - name: nifi-data 72 | mountPath: /opt/nifi/nifi-current/content_repository 73 | subPath: content_repository 74 | - name: nifi-data 75 | mountPath: /opt/nifi/nifi-current/provenance_repository 76 | subPath: provenance_repository 77 | - name: nifi-data 78 | mountPath: /opt/nifi/nifi-current/state 79 | subPath: state 80 | securityContext: 81 | runAsUser: 1000 82 | fsGroup: 1000 83 | volumeClaimTemplates: 84 | - metadata: 85 | name: nifi-data 86 | spec: 87 | accessModes: [ "ReadWriteOnce" ] 88 | storageClassName: "default" 89 | resources: 90 | requests: 91 | storage: 10Gi 92 | --------------------------------------------------------------------------------