├── .gitignore ├── .gitlab-ci.yml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md └── compose ├── mysql ├── .gitkeep ├── docker-compose.mysql-5.7.yaml ├── mysql-connector-java-8.0.11.jar └── mysql-jdbc-input-plugin.conf ├── oracle ├── .gitkeep └── ojdbc6-11.2.0.4.jar ├── postgres ├── .gitkeep ├── docker-compose.postgres-9.6.yaml ├── postgres-jdbc-input.conf └── postgresql-42.2.8.jar └── sql-server ├── .gitkeep ├── docker-compose.sqlserver-2017-linux.yaml ├── mssql-jdbc-11.2.0.jre11.jar ├── mssql-jdbc-11.2.0.jre17.jar ├── mssql-jdbc-11.2.0.jre18.jar ├── mssql-jdbc-11.2.0.jre8.jar ├── mssql-jdbc-7.2.2.jre8.jar ├── mssql-jdbc-7.4.1.jre11.jar ├── mssql-jdbc-7.4.1.jre8.jar └── sqlserver-jdbc-input.conf /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimMaryanto93/docker-logstash-input-jdbc/35ad1402ce437d27d6f19cfaafc805410ba1333f/.gitignore -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | IMAGE_NAME: "logstash/logstash-jdbc-input-plugin" 3 | 4 | building-image: 5 | stage: deploy 6 | before_script: 7 | - 'docker login --username $DOCKER_USERNAME --password $DOCKER_PASSWORD $DOCKER_REGISTRY_HOSTED' 8 | - 'docker login --username $DOCKER_USERNAME --password $DOCKER_PASSWORD $DOCKER_REGISTRY_PUBLIC' 9 | script: 10 | - 'docker build -t $IMAGE_NAME:$CI_COMMIT_TAG .' 11 | - 'docker tag $IMAGE_NAME:$CI_COMMIT_TAG $DOCKER_REGISTRY_HOSTED/$IMAGE_NAME:$CI_COMMIT_TAG' 12 | - 'docker push $DOCKER_REGISTRY_HOSTED/$IMAGE_NAME:$CI_COMMIT_TAG' 13 | only: 14 | - /-final$/ 15 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # `7.3.2-release` 2 | 3 | - Enabled logstash plugin jdbc input for MySQL 5.7+ 4 | - Enabled logstash plugin jdbc input for PostgreSQL 9.6+ 5 | - Enabled logstash plugin jdbc input for MS SQL Server 2017 linux -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimMaryanto93/docker-logstash-input-jdbc/35ad1402ce437d27d6f19cfaafc805410ba1333f/CONTRIBUTING.md -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.elastic.co/logstash/logstash:7.3.2 2 | MAINTAINER Dimas Maryanto 3 | 4 | # install dependency 5 | RUN /usr/share/logstash/bin/logstash-plugin install logstash-input-jdbc 6 | RUN /usr/share/logstash/bin/logstash-plugin install logstash-filter-aggregate 7 | RUN /usr/share/logstash/bin/logstash-plugin install logstash-filter-jdbc_streaming 8 | RUN /usr/share/logstash/bin/logstash-plugin install logstash-filter-mutate 9 | 10 | # copy lib database jdbc jars 11 | COPY ./compose/mysql/mysql-connector-java-8.0.11.jar /usr/share/logstash/logstash-core/lib/jars/mysql-connector-java.jar 12 | COPY ./compose/sql-server/mssql-jdbc-11.2.0.jre11.jar /usr/share/logstash/logstash-core/lib/jars/mssql-jdbc.jar 13 | COPY ./compose/oracle/ojdbc6-11.2.0.4.jar /usr/share/logstash/logstash-core/lib/jars/ojdbc6.jar 14 | COPY ./compose/postgres/postgresql-42.2.8.jar /usr/share/logstash/logstash-core/lib/jars/postgresql.jar -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Dimas Maryanto 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Starting logstash 2 | 3 | Starting logstash input jdbc 4 | 5 | ```bash 6 | docker-compose -f compose/db-platform/docker-compose.version.yaml -p project_name up --build 7 | ``` 8 | 9 | Stoping all service 10 | 11 | ```bash 12 | docker-compose -f compose/db-platform/docker-compose.version.yaml -p project_name down --rmi local 13 | ``` 14 | 15 | ## Customize configuration 16 | 17 | Customize configuration for logstash 18 | 19 | - modify file `logstash.conf` in docker container 20 | 21 | ```yml 22 | input { 23 | jdbc { 24 | jdbc_driver_library => "${LOGSTASH_JDBC_DRIVER_JAR_LOCATION}" 25 | jdbc_driver_class => "${LOGSTASH_JDBC_DRIVER}" 26 | jdbc_connection_string => "${LOGSTASH_JDBC_URL}" 27 | jdbc_user => "${LOGSTASH_JDBC_USERNAME}" 28 | jdbc_password => "${LOGSTASH_JDBC_PASSWORD}" 29 | schedule => "* * * * *" 30 | statement => "select ... from table_name" 31 | } 32 | } 33 | 34 | output { 35 | stdout { codec => json_lines } 36 | } 37 | ``` 38 | 39 | Environtment example 40 | 41 | - LOGSTASH_JDBC_DRIVER_JAR_LOCATION: 42 | - for MySQL 5.7+ Database: `/usr/share/logstash/logstash-core/lib/jars/mysql-connector-java.jar` 43 | - for MS-SQL Server 2017 linux+ Database: `/usr/share/logstash/logstash-core/lib/jars/mssql-jdbc.jar` 44 | - for Oracle 11g XE Database: `/usr/share/logstash/logstash-core/lib/jars/ojdbc6.jar` 45 | - for PostgreSQL 9.6+ Database: `/usr/share/logstash-core/lib/jars/logstash/postgresql.jar` 46 | - LOGSTASH_JDBC_DRIVER: `com.mysql.jdbc.Driver` 47 | - LOGSTASH_JDBC_URL: `jdbc:mysql://[host]:[port]/[database-name]` 48 | - LOGSTASH_JDBC_USERNAME: `database_user` 49 | - LOGSTASH_JDBC_PASSWORD: `database_user_password` -------------------------------------------------------------------------------- /compose/mysql/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimMaryanto93/docker-logstash-input-jdbc/35ad1402ce437d27d6f19cfaafc805410ba1333f/compose/mysql/.gitkeep -------------------------------------------------------------------------------- /compose/mysql/docker-compose.mysql-5.7.yaml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | mysql: 4 | image: tabeldata.ip-dynamic.com:8088/mysql:5.7 5 | ports: 6 | - 3306:3306 7 | environment: 8 | - MYSQL_ROOT_PASSWORD=admin 9 | - MYSQL_DATABASE=example 10 | - MYSQL_USER=admin 11 | - MYSQL_PASSWORD=admin 12 | volumes: 13 | - mysql_data:/var/lib/mysql 14 | networks: 15 | - share-network 16 | elasticsearch: 17 | image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 18 | ports: 19 | - 9300:9300 20 | - 9200:9200 21 | environment: 22 | - http.cors.enabled=true 23 | - http.cors.allow-origin=* 24 | - http.cors.allow-methods=OPTIONS,HEAD,GET,POST,PUT,DELETE 25 | - http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization 26 | - transport.host=127.0.0.1 27 | - cluster.name=docker-cluster 28 | - discovery.type=single-node 29 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 30 | volumes: 31 | - elasticsearch_data:/usr/share/elasticsearch/data 32 | networks: 33 | - share-network 34 | kibana: 35 | image: docker.elastic.co/kibana/kibana:7.3.2 36 | ports: 37 | - 5601:5601 38 | networks: 39 | - share-network 40 | depends_on: 41 | - elasticsearch 42 | logstash: 43 | build: 44 | dockerfile: ${PWD}/Dockerfile 45 | context: ${PWD}/ 46 | environment: 47 | - LOGSTASH_JDBC_URL=jdbc:mysql://mysql:3306/example?useSSL=false 48 | - LOGSTASH_JDBC_DRIVER=com.mysql.cj.jdbc.Driver 49 | - LOGSTASH_JDBC_DRIVER_JAR_LOCATION=/usr/share/logstash/logstash-core/lib/jars/mysql-connector-java.jar 50 | - LOGSTASH_JDBC_USERNAME=admin 51 | - LOGSTASH_JDBC_PASSWORD=admin 52 | - LOGSTASH_ELASTICSEARCH_HOST=http://elasticsearch:9200 53 | volumes: 54 | - ./mysql-jdbc-input-plugin.conf:/usr/share/logstash/pipeline/logstash.conf 55 | ports: 56 | - 9600:9600 57 | - 5044:5044 58 | networks: 59 | - share-network 60 | depends_on: 61 | - elasticsearch 62 | - kibana 63 | - mysql 64 | volumes: 65 | elasticsearch_data: 66 | mysql_data: 67 | networks: 68 | share-network: -------------------------------------------------------------------------------- /compose/mysql/mysql-connector-java-8.0.11.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimMaryanto93/docker-logstash-input-jdbc/35ad1402ce437d27d6f19cfaafc805410ba1333f/compose/mysql/mysql-connector-java-8.0.11.jar -------------------------------------------------------------------------------- /compose/mysql/mysql-jdbc-input-plugin.conf: -------------------------------------------------------------------------------- 1 | input { 2 | jdbc { 3 | jdbc_driver_library => "${LOGSTASH_JDBC_DRIVER_JAR_LOCATION}" 4 | jdbc_driver_class => "${LOGSTASH_JDBC_DRIVER}" 5 | jdbc_connection_string => "${LOGSTASH_JDBC_URL}" 6 | jdbc_user => "${LOGSTASH_JDBC_USERNAME}" 7 | jdbc_password => "${LOGSTASH_JDBC_PASSWORD}" 8 | schedule => "* * * * *" 9 | statement => "select uuid() as id, 1+2 as value" 10 | } 11 | } 12 | 13 | output { 14 | elasticsearch { 15 | hosts => ["${LOGSTASH_ELASTICSEARCH_HOST}"] 16 | index => "testing" 17 | document_id => "%{id}" 18 | } 19 | stdout { codec => json_lines } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /compose/oracle/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimMaryanto93/docker-logstash-input-jdbc/35ad1402ce437d27d6f19cfaafc805410ba1333f/compose/oracle/.gitkeep -------------------------------------------------------------------------------- /compose/oracle/ojdbc6-11.2.0.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimMaryanto93/docker-logstash-input-jdbc/35ad1402ce437d27d6f19cfaafc805410ba1333f/compose/oracle/ojdbc6-11.2.0.4.jar -------------------------------------------------------------------------------- /compose/postgres/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimMaryanto93/docker-logstash-input-jdbc/35ad1402ce437d27d6f19cfaafc805410ba1333f/compose/postgres/.gitkeep -------------------------------------------------------------------------------- /compose/postgres/docker-compose.postgres-9.6.yaml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | postgres: 4 | image: tabeldata.ip-dynamic.com:8088/postgres:9.6 5 | environment: 6 | - POSTGRES_PASSWORD=passwordnyaPostgres 7 | - POSTGRES_USER=postgres 8 | - POSTGRES_DB=postgres 9 | ports: 10 | - 5432:5432 11 | volumes: 12 | - postgres_data:/var/lib/postgresql/data 13 | networks: 14 | - share-network 15 | elasticsearch: 16 | image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 17 | ports: 18 | - 9300:9300 19 | - 9200:9200 20 | environment: 21 | - http.cors.enabled=true 22 | - http.cors.allow-origin=* 23 | - http.cors.allow-methods=OPTIONS,HEAD,GET,POST,PUT,DELETE 24 | - http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization 25 | - transport.host=127.0.0.1 26 | - cluster.name=docker-cluster 27 | - discovery.type=single-node 28 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 29 | volumes: 30 | - elasticsearch_data:/usr/share/elasticsearch/data 31 | networks: 32 | - share-network 33 | kibana: 34 | image: docker.elastic.co/kibana/kibana:7.3.2 35 | ports: 36 | - 5601:5601 37 | networks: 38 | - share-network 39 | depends_on: 40 | - elasticsearch 41 | logstash: 42 | build: 43 | dockerfile: ${PWD}/Dockerfile 44 | context: ${PWD}/ 45 | environment: 46 | - LOGSTASH_JDBC_URL=jdbc:postgresql://postgres:5432/postgres 47 | - LOGSTASH_JDBC_DRIVER=org.postgresql.Driver 48 | - LOGSTASH_JDBC_DRIVER_JAR_LOCATION=/usr/share/logstash/logstash-core/lib/jars/postgresql.jar 49 | - LOGSTASH_JDBC_USERNAME=postgres 50 | - LOGSTASH_JDBC_PASSWORD=passwordnyaPostgres 51 | - LOGSTASH_ELASTICSEARCH_HOST=http://elasticsearch:9200 52 | volumes: 53 | - ./postgres-jdbc-input.conf:/usr/share/logstash/pipeline/logstash.conf 54 | ports: 55 | - 9600:9600 56 | - 5044:5044 57 | networks: 58 | - share-network 59 | depends_on: 60 | - elasticsearch 61 | - kibana 62 | - postgres 63 | volumes: 64 | elasticsearch_data: 65 | postgres_data: 66 | networks: 67 | share-network: -------------------------------------------------------------------------------- /compose/postgres/postgres-jdbc-input.conf: -------------------------------------------------------------------------------- 1 | input { 2 | jdbc { 3 | jdbc_driver_library => "${LOGSTASH_JDBC_DRIVER_JAR_LOCATION}" 4 | jdbc_driver_class => "${LOGSTASH_JDBC_DRIVER}" 5 | jdbc_connection_string => "${LOGSTASH_JDBC_URL}" 6 | jdbc_user => "${LOGSTASH_JDBC_USERNAME}" 7 | jdbc_password => "${LOGSTASH_JDBC_PASSWORD}" 8 | schedule => "* * * * *" 9 | statement => "select MD5(random()::text) as id, 1+2 as value" 10 | } 11 | } 12 | 13 | output { 14 | elasticsearch { 15 | hosts => ["${LOGSTASH_ELASTICSEARCH_HOST}"] 16 | index => "testing" 17 | document_id => "%{id}" 18 | } 19 | stdout { codec => json_lines } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /compose/postgres/postgresql-42.2.8.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimMaryanto93/docker-logstash-input-jdbc/35ad1402ce437d27d6f19cfaafc805410ba1333f/compose/postgres/postgresql-42.2.8.jar -------------------------------------------------------------------------------- /compose/sql-server/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimMaryanto93/docker-logstash-input-jdbc/35ad1402ce437d27d6f19cfaafc805410ba1333f/compose/sql-server/.gitkeep -------------------------------------------------------------------------------- /compose/sql-server/docker-compose.sqlserver-2017-linux.yaml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | sql-server: 4 | image: mcr.microsoft.com/mssql/server:2017-latest 5 | environment: 6 | - ACCEPT_EULA=Y 7 | - SA_PASSWORD=PasswordnyaSA2018 8 | - MSSQL_PID=Developer 9 | ports: 10 | - 1433:1433 11 | volumes: 12 | - sqlserver_data:/var/opt/mssql/ 13 | networks: 14 | - share-network 15 | elasticsearch: 16 | image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 17 | ports: 18 | - 9300:9300 19 | - 9200:9200 20 | environment: 21 | - http.cors.enabled=true 22 | - http.cors.allow-origin=* 23 | - http.cors.allow-methods=OPTIONS,HEAD,GET,POST,PUT,DELETE 24 | - http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization 25 | - transport.host=127.0.0.1 26 | - cluster.name=docker-cluster 27 | - discovery.type=single-node 28 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 29 | volumes: 30 | - elasticsearch_data:/usr/share/elasticsearch/data 31 | networks: 32 | - share-network 33 | kibana: 34 | image: docker.elastic.co/kibana/kibana:7.3.2 35 | ports: 36 | - 5601:5601 37 | networks: 38 | - share-network 39 | depends_on: 40 | - elasticsearch 41 | logstash: 42 | build: 43 | dockerfile: ${PWD}/Dockerfile 44 | context: ${PWD}/ 45 | environment: 46 | - LOGSTASH_JDBC_URL=jdbc:sqlserver://sql-server:1433;databaseName=master;integratedSecurity=false; 47 | - LOGSTASH_JDBC_DRIVER=com.microsoft.sqlserver.jdbc.SQLServerDriver 48 | - LOGSTASH_JDBC_DRIVER_JAR_LOCATION=/usr/share/logstash/logstash-core/lib/jars/mssql-jdbc.jar 49 | - LOGSTASH_JDBC_USERNAME=sa 50 | - LOGSTASH_JDBC_PASSWORD=PasswordnyaSA2018 51 | - LOGSTASH_ELASTICSEARCH_HOST=http://elasticsearch:9200 52 | volumes: 53 | - ./sqlserver-jdbc-input.conf:/usr/share/logstash/pipeline/logstash.conf 54 | ports: 55 | - 9600:9600 56 | - 5044:5044 57 | networks: 58 | - share-network 59 | depends_on: 60 | - elasticsearch 61 | - kibana 62 | - sql-server 63 | volumes: 64 | elasticsearch_data: 65 | sqlserver_data: 66 | networks: 67 | share-network: -------------------------------------------------------------------------------- /compose/sql-server/mssql-jdbc-11.2.0.jre11.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimMaryanto93/docker-logstash-input-jdbc/35ad1402ce437d27d6f19cfaafc805410ba1333f/compose/sql-server/mssql-jdbc-11.2.0.jre11.jar -------------------------------------------------------------------------------- /compose/sql-server/mssql-jdbc-11.2.0.jre17.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimMaryanto93/docker-logstash-input-jdbc/35ad1402ce437d27d6f19cfaafc805410ba1333f/compose/sql-server/mssql-jdbc-11.2.0.jre17.jar -------------------------------------------------------------------------------- /compose/sql-server/mssql-jdbc-11.2.0.jre18.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimMaryanto93/docker-logstash-input-jdbc/35ad1402ce437d27d6f19cfaafc805410ba1333f/compose/sql-server/mssql-jdbc-11.2.0.jre18.jar -------------------------------------------------------------------------------- /compose/sql-server/mssql-jdbc-11.2.0.jre8.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimMaryanto93/docker-logstash-input-jdbc/35ad1402ce437d27d6f19cfaafc805410ba1333f/compose/sql-server/mssql-jdbc-11.2.0.jre8.jar -------------------------------------------------------------------------------- /compose/sql-server/mssql-jdbc-7.2.2.jre8.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimMaryanto93/docker-logstash-input-jdbc/35ad1402ce437d27d6f19cfaafc805410ba1333f/compose/sql-server/mssql-jdbc-7.2.2.jre8.jar -------------------------------------------------------------------------------- /compose/sql-server/mssql-jdbc-7.4.1.jre11.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimMaryanto93/docker-logstash-input-jdbc/35ad1402ce437d27d6f19cfaafc805410ba1333f/compose/sql-server/mssql-jdbc-7.4.1.jre11.jar -------------------------------------------------------------------------------- /compose/sql-server/mssql-jdbc-7.4.1.jre8.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimMaryanto93/docker-logstash-input-jdbc/35ad1402ce437d27d6f19cfaafc805410ba1333f/compose/sql-server/mssql-jdbc-7.4.1.jre8.jar -------------------------------------------------------------------------------- /compose/sql-server/sqlserver-jdbc-input.conf: -------------------------------------------------------------------------------- 1 | input { 2 | jdbc { 3 | jdbc_driver_library => "${LOGSTASH_JDBC_DRIVER_JAR_LOCATION}" 4 | jdbc_driver_class => "${LOGSTASH_JDBC_DRIVER}" 5 | jdbc_connection_string => "${LOGSTASH_JDBC_URL}" 6 | jdbc_user => "${LOGSTASH_JDBC_USERNAME}" 7 | jdbc_password => "${LOGSTASH_JDBC_PASSWORD}" 8 | schedule => "* * * * *" 9 | statement => "select NEWID () as id, 1+2 as value" 10 | } 11 | } 12 | 13 | output { 14 | elasticsearch { 15 | hosts => ["${LOGSTASH_ELASTICSEARCH_HOST}"] 16 | index => "testing" 17 | document_id => "%{id}" 18 | } 19 | stdout { codec => json_lines } 20 | 21 | } 22 | --------------------------------------------------------------------------------