├── .devcontainer ├── dockerfiles │ ├── mrplow │ │ ├── foo │ │ ├── puthnfo.sh │ │ └── Dockerfile │ └── elasticsearch │ │ └── Dockerfile ├── icstart.sh ├── netcreate.sh ├── gethnfo.sh ├── docker-compose-postgres.yml ├── docker-compose-kibana.yml ├── docker-compose-elastic.yml └── devcontainer.json ├── .vscode ├── settings.json ├── launch.json └── tasks.json ├── .gitignore ├── Dockerfile ├── Makefile ├── .deepsource.toml ├── .github └── workflows │ ├── release.yml │ └── ci.yml ├── internal ├── config │ ├── filereader.go │ └── config.go ├── scheduler │ └── scheduler.go ├── casting │ └── converter.go ├── elastic │ └── indexer.go └── movedata │ └── move.go ├── go.mod ├── test ├── test_model.go ├── integration_common.go ├── config_test.go ├── scheduling_integration_test.go ├── upsert_integration_test.go ├── complete_config_test.go ├── insert_integration_test.go ├── insert_typed_integration_test.go ├── invalid_config_test.go ├── integration_json_test.go └── type_conversion_test.go ├── docker └── docker-compose.yml ├── cmd └── main.go ├── go.sum ├── README.md └── LICENSE /.devcontainer/dockerfiles/mrplow/foo: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.devcontainer/dockerfiles/elasticsearch/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM elasticsearch:7.16.3 2 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "cSpell.words": [ 3 | "mgroup" 4 | ] 5 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | mr-plow 3 | config.yml 4 | .devcontainer/dockerfiles/mrplow/hnfo 5 | bin/mrplow -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.16-alpine 2 | COPY ./ /mrplow 3 | RUN cd /mrplow && go build 4 | CMD ["/mrplow/mr-plow", "-config", "/mrplow/config.yml"] 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | build: 2 | go build -o bin/mrplow cmd/main.go 3 | 4 | test: 5 | go test ./... 6 | clean: 7 | @echo "Cleaning the mr-plow" 8 | @rm -fr bin/mrplow 9 | -------------------------------------------------------------------------------- /.deepsource.toml: -------------------------------------------------------------------------------- 1 | version = 1 2 | 3 | [[analyzers]] 4 | name = "go" 5 | enabled = true 6 | 7 | [analyzers.meta] 8 | import_root = "github.com/Ringloop/Mr-Plow" 9 | -------------------------------------------------------------------------------- /.devcontainer/icstart.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | WRK_PATH=$1 4 | REL_PATH=.devcontainer 5 | 6 | $WRK_PATH/$REL_PATH/gethnfo.sh $WRK_PATH 7 | $WRK_PATH/$REL_PATH/netcreate.sh $WRK_PATH -------------------------------------------------------------------------------- /.devcontainer/netcreate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if docker network ls | grep "vscode-mr-plow"; then 4 | echo "network vscode-mr-plow already exists, nothing to do" 5 | else 6 | echo "network vscode-mr-plow doesn't exist, creating ..." 7 | docker network create --driver=bridge --attachable --subnet=10.70.67.0/24 vscode-mr-plow 8 | fi -------------------------------------------------------------------------------- /.devcontainer/gethnfo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | WRK_PATH=$1 4 | REL_PATH=.devcontainer/dockerfiles/mrplow/hnfo 5 | 6 | id -u > $WRK_PATH/$REL_PATH 7 | id -g >> $WRK_PATH/$REL_PATH 8 | getent group docker | cut -d: -f3 >> $WRK_PATH/$REL_PATH 9 | echo "provisioning with host's user UID: $(id -u), GID: $(id -g), host's docker GID: $(getent group docker | cut -d: -f3)" -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Launch Package", 6 | "type": "go", 7 | "request": "launch", 8 | "mode": "auto", 9 | "program": "${workspaceFolder}", 10 | "args": ["-v", "trace", "-n", "-set", "jerico"] 11 | } 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | on: 2 | release: 3 | types: [created] 4 | 5 | jobs: 6 | release-linux-amd64: 7 | name: release linux/amd64 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | - uses: wangyoucao577/go-release-action@v1.22 12 | with: 13 | github_token: ${{ secrets.GITHUB_TOKEN }} 14 | goos: linux 15 | goarch: amd64 -------------------------------------------------------------------------------- /.devcontainer/docker-compose-postgres.yml: -------------------------------------------------------------------------------- 1 | version: '3.4' 2 | 3 | services: 4 | postgres: 5 | image: postgres:14.2 6 | 7 | environment: 8 | POSTGRES_USER: user 9 | POSTGRES_PASSWORD: pwd 10 | 11 | ports: 12 | - "5432:5432" 13 | 14 | networks: 15 | vscode-mr-plow: 16 | ipv4_address: 10.70.67.104 17 | 18 | networks: 19 | vscode-mr-plow: 20 | external: true 21 | 22 | -------------------------------------------------------------------------------- /.devcontainer/docker-compose-kibana.yml: -------------------------------------------------------------------------------- 1 | version: '3.4' 2 | 3 | services: 4 | kibana: 5 | image: docker.elastic.co/kibana/kibana:7.15.1 6 | 7 | environment: 8 | ELASTICSEARCH_URL: http://10.70.67.102:9200 9 | ELASTICSEARCH_HOSTS: '["http://10.70.67.102:9200"]' 10 | 11 | ports: 12 | - 5601:5601 13 | 14 | networks: 15 | vscode-mr-plow: 16 | ipv4_address: 10.70.67.103 17 | 18 | networks: 19 | vscode-mr-plow: 20 | external: true 21 | 22 | -------------------------------------------------------------------------------- /internal/config/filereader.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "io/ioutil" 5 | ) 6 | 7 | // Defining an interface so that functionality of 'readConfig()' can be mocked 8 | type IReader interface { 9 | ReadConfig() ([]byte, error) 10 | } 11 | 12 | type Reader struct { 13 | FileName string 14 | } 15 | 16 | // 'reader' implementing the Interface 17 | // Function to read from actual file 18 | func (r *Reader) ReadConfig() ([]byte, error) { 19 | configFile, err := ioutil.ReadFile(r.FileName) 20 | return configFile, err 21 | } 22 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Ringloop/mr-plow 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/elastic/go-elasticsearch/v7 v7.17.0 7 | github.com/go-sql-driver/mysql v1.6.0 8 | github.com/lib/pq v1.10.4 9 | github.com/stretchr/testify v1.7.0 10 | gopkg.in/yaml.v2 v2.4.0 11 | ) 12 | 13 | require ( 14 | github.com/davecgh/go-spew v1.1.0 // indirect 15 | github.com/elastic/go-elasticsearch v0.0.0 // indirect 16 | github.com/pmezard/go-difflib v1.0.0 // indirect 17 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect 18 | ) 19 | -------------------------------------------------------------------------------- /test/test_model.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import "time" 4 | 5 | //generated thanks to https://mholt.github.io/json-to-go/ 6 | 7 | type ElasticTestResponse struct { 8 | Hits struct { 9 | Total struct { 10 | Value int `json:"value"` 11 | } `json:"total"` 12 | Hits []struct { 13 | Index string `json:"_index"` 14 | Type string `json:"_type"` 15 | ID string `json:"_id"` 16 | Score interface{} `json:"_score"` 17 | Source struct { 18 | Email string `json:"email"` 19 | LastUpdate time.Time `json:"last_update"` 20 | UserID int `json:"user_id"` 21 | } `json:"_source"` 22 | } `json:"hits"` 23 | } `json:"hits"` 24 | } 25 | 26 | type FakeExitSignal struct{} 27 | 28 | func (FakeExitSignal) String() string { 29 | return "Fake exit" 30 | } 31 | 32 | func (FakeExitSignal) Signal() {} 33 | -------------------------------------------------------------------------------- /.devcontainer/docker-compose-elastic.yml: -------------------------------------------------------------------------------- 1 | version: '3.4' 2 | 3 | services: 4 | elasticsearch: 5 | image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1 6 | 7 | environment: 8 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 9 | - xpack.security.enabled=false 10 | - "discovery.type=single-node" 11 | 12 | ulimits: 13 | memlock: 14 | soft: -1 15 | hard: -1 16 | 17 | volumes: 18 | - es-data:/usr/share/elasticsearch/data 19 | 20 | ports: 21 | - 9200:9200 22 | 23 | networks: 24 | vscode-mr-plow: 25 | ipv4_address: 10.70.67.102 26 | 27 | cap_add: 28 | - SYS_PTRACE 29 | security_opt: 30 | - seccomp:unconfined 31 | 32 | volumes: 33 | es-data: 34 | driver: local 35 | 36 | networks: 37 | vscode-mr-plow: 38 | external: true 39 | 40 | -------------------------------------------------------------------------------- /.devcontainer/dockerfiles/mrplow/puthnfo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | USER_UID=$(awk 'NR==1' /hnfo) 4 | USER_GID=$(awk 'NR==2' /hnfo) 5 | DKR_GID=$(awk 'NR==3' /hnfo) 6 | VSUID=$(expr $USER_UID + 67) 7 | VSGID=$(expr $USER_GID + 67) 8 | 9 | echo putinfo.sh called with user: [$USERNAME, $USER_UID, $USER_GID, $DKR_GID] 10 | usermod --uid $VSUID vscode 11 | groupmod --gid $VSGID vscode 12 | groupadd --gid $USER_GID $USERNAME 13 | useradd --uid $USER_UID --gid $USER_GID -m -s /bin/bash $USERNAME 14 | cp /home/vscode/.bash_logout /home/$USERNAME 15 | cp /home/vscode/.bashrc /home/$USERNAME 16 | cp /home/vscode/.profile /home/$USERNAME 17 | cp /home/vscode/.zshrc /home/$USERNAME 18 | cp -R /home/vscode/.oh-my-zsh /home/$USERNAME 19 | chown -R $USERNAME:$USERNAME /home/$USERNAME 20 | echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME 21 | chmod 0440 /etc/sudoers.d/$USERNAME 22 | groupmod --gid $DKR_GID docker 23 | usermod -a -G $DKR_GID $USERNAME -------------------------------------------------------------------------------- /.devcontainer/dockerfiles/mrplow/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/vscode/devcontainers/base:0-debian-10 2 | 3 | ARG USERNAME=vscode 4 | ARG USER_UID=1000 5 | ARG USER_GID=$USER_UID 6 | 7 | ENV DEBIAN_FRONTEND=noninteractive 8 | 9 | # Configure apt and install packages 10 | RUN apt-get update \ 11 | && apt-get upgrade -y \ 12 | && apt-get -y install sudo build-essential bash-completion docker-compose \ 13 | && apt-get autoremove -y 14 | 15 | # Create the user 16 | ADD puthnfo.sh /usr/bin/ 17 | COPY foo hnfo / 18 | RUN if [ "$USERNAME" != "vscode" ]; then \ 19 | puthnfo.sh \ 20 | ; fi 21 | 22 | # Configure apt and install go 23 | RUN wget https://golang.org/dl/go1.17.7.linux-amd64.tar.gz && tar xvf go1.17.7.linux-amd64.tar.gz 24 | RUN chown -R root:root ./go && mv go /usr/local 25 | RUN echo "export GOPATH=/home/vscode/work" >> /home/vscode/.profile 26 | RUN echo "export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin" >> /home/vscode/.profile 27 | 28 | USER $USERNAME 29 | ENV DEBIAN_FRONTEND=dialog 30 | -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "label": "build mrplow", 6 | "type": "shell", 7 | "command": "make", 8 | "args": ["build"], 9 | "group": { 10 | "kind": "build", 11 | "isDefault": true 12 | }, 13 | "problemMatcher": [] 14 | }, 15 | { 16 | "label": "build test", 17 | "type": "shell", 18 | "command": "make", 19 | "args": ["test"], 20 | "group": { 21 | "kind": "build", 22 | "isDefault": true 23 | }, 24 | "problemMatcher": [] 25 | }, 26 | { 27 | "label": "clean", 28 | "type": "shell", 29 | "command": "make", 30 | "args": ["clean"], 31 | "group": { 32 | "kind": "build", 33 | "isDefault": true 34 | }, 35 | "problemMatcher": [] 36 | } 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /internal/scheduler/scheduler.go: -------------------------------------------------------------------------------- 1 | package scheduler 2 | 3 | import ( 4 | "database/sql" 5 | "log" 6 | "os" 7 | "os/signal" 8 | "syscall" 9 | "time" 10 | 11 | "github.com/Ringloop/mr-plow/internal/config" 12 | "github.com/Ringloop/mr-plow/internal/movedata" 13 | ) 14 | 15 | type Scheduler struct { 16 | Done chan os.Signal 17 | } 18 | 19 | func NewScheduler() Scheduler { 20 | s := Scheduler{make(chan os.Signal)} 21 | signal.Notify(s.Done, os.Interrupt, syscall.SIGTERM) 22 | return s 23 | } 24 | 25 | func (s *Scheduler) MoveDataUntilExit(conf *config.ImportConfig, db *sql.DB, query *config.QueryModel, finished chan bool) { 26 | ticker := time.NewTicker(time.Duration(conf.PollingSeconds * 1000000000)) 27 | defer ticker.Stop() 28 | 29 | mover := movedata.New(db, conf, query) 30 | for { 31 | select { 32 | case <-s.Done: 33 | log.Println("stopping query execution, bye...") 34 | finished <- true 35 | return 36 | default: 37 | <-ticker.C 38 | moveErr := mover.MoveData() 39 | if moveErr != nil { 40 | log.Printf("error executing query %s", moveErr) 41 | } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.2' 2 | services: 3 | postgres: 4 | container_name: postgres_container 5 | image: postgres:14.1 6 | environment: 7 | POSTGRES_USER: user 8 | POSTGRES_PASSWORD: pwd 9 | ports: 10 | - "5432:5432" 11 | es01: 12 | image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1 13 | container_name: es01 14 | environment: 15 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 16 | - xpack.security.enabled=false 17 | - "discovery.type=single-node" 18 | ulimits: 19 | memlock: 20 | soft: -1 21 | hard: -1 22 | volumes: 23 | - data01:/usr/share/elasticsearch/data 24 | ports: 25 | - 9200:9200 26 | networks: 27 | - elastic 28 | 29 | 30 | kib01: 31 | image: docker.elastic.co/kibana/kibana:7.15.1 32 | container_name: kib01 33 | ports: 34 | - 5601:5601 35 | environment: 36 | ELASTICSEARCH_URL: http://es01:9200 37 | ELASTICSEARCH_HOSTS: '["http://es01:9200"]' 38 | networks: 39 | - elastic 40 | 41 | volumes: 42 | data01: 43 | driver: local 44 | 45 | networks: 46 | elastic: 47 | driver: bridge 48 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mr-plow", 3 | "dockerFile": "./dockerfiles/mrplow/Dockerfile", 4 | 5 | /* 6 | On MS Windows you should comment both "initializeCommand" and "build" directives. 7 | */ 8 | "initializeCommand": "/bin/bash ${localWorkspaceFolder}/.devcontainer/icstart.sh ${localWorkspaceFolder}", 9 | "build": { "args": { 10 | "USERNAME": "${localEnv:USER}", 11 | }}, 12 | 13 | "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined", "--ip", "10.70.67.101", "--net", "vscode-mr-plow"], 14 | "settings": { 15 | "terminal.integrated.profiles.linux": { 16 | "bash (login)": { 17 | "path": "/bin/bash" 18 | } 19 | } 20 | }, 21 | 22 | // Add the IDs of extensions you want installed when the container is created. 23 | "extensions": [ 24 | "golang.go", 25 | "ms-azuretools.vscode-docker", 26 | ], 27 | 28 | /* 29 | On MS Windows you should set "remoteUser": "vscode" 30 | */ 31 | "remoteUser": "${localEnv:USER}", 32 | 33 | "mounts": [ 34 | "source=${localEnv:HOME}/.ssh,target=/home/${localEnv:USER}/.ssh,type=bind,consistency=cached", 35 | "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind" 36 | ] 37 | } 38 | -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "database/sql" 5 | "flag" 6 | "log" 7 | 8 | "github.com/Ringloop/mr-plow/internal/scheduler" 9 | 10 | "github.com/Ringloop/mr-plow/internal/config" 11 | _ "github.com/go-sql-driver/mysql" 12 | _ "github.com/lib/pq" 13 | ) 14 | 15 | func main() { 16 | configPath := flag.String("config", "./config.yml", "path of the configuration file") 17 | flag.Parse() 18 | 19 | ymlConfReader := config.Reader{FileName: *configPath} 20 | conf, err := config.ParseConfiguration(&ymlConfReader) 21 | if err != nil { 22 | log.Fatal("Cannot parse config file", err) 23 | } 24 | ConnectAndStart(conf) 25 | } 26 | 27 | func ConnectAndStart(conf *config.ImportConfig) { 28 | db, err := sql.Open("postgres", conf.Database) 29 | if err != nil { 30 | log.Printf("Failed to open a DB connection: %s", err) 31 | return 32 | } 33 | defer func(db *sql.DB) { 34 | err := db.Close() 35 | if err != nil { 36 | log.Printf("error in closing postgres connection: %s", err) 37 | } 38 | }(db) 39 | log.Println("Connected to postgres") 40 | 41 | finished := make(chan bool) 42 | for _, c := range conf.Queries { 43 | s := scheduler.NewScheduler() 44 | go s.MoveDataUntilExit(conf, db, &c, finished) 45 | } 46 | 47 | for i := 0; i < len(conf.Queries); i++ { 48 | <-finished 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /test/integration_common.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "database/sql" 5 | "testing" 6 | 7 | "github.com/Ringloop/mr-plow/internal/config" 8 | ) 9 | 10 | func initConfigIntegrationTest(t *testing.T, testReader config.IReader) *config.ImportConfig { 11 | conf, err := config.ParseConfiguration(testReader) 12 | if err != nil { 13 | t.Error("error reading conf", err) 14 | t.FailNow() 15 | } 16 | return conf 17 | } 18 | 19 | func initSqlDB(t *testing.T, conf *config.ImportConfig) *sql.DB { 20 | db, err := sql.Open("postgres", conf.Database) 21 | if err != nil { 22 | t.Error("error connecting to sql db", err) 23 | t.FailNow() 24 | } 25 | 26 | _, err = db.Exec(` 27 | 28 | DROP SCHEMA IF EXISTS test CASCADE; 29 | CREATE SCHEMA test; 30 | 31 | DROP TABLE IF EXISTS test.table1; 32 | CREATE TABLE test.table1 ( 33 | user_id SERIAL PRIMARY KEY, 34 | email VARCHAR ( 255 ) NOT NULL, 35 | last_update TIMESTAMP NOT NULL 36 | ) 37 | 38 | `) 39 | 40 | if err != nil { 41 | t.Error("error creating schema", err) 42 | t.FailNow() 43 | } 44 | 45 | return db 46 | } 47 | 48 | func insertData(db *sql.DB, email string, t *testing.T) { 49 | _, err := db.Exec(` 50 | INSERT INTO test.table1 (email,last_update) 51 | VALUES($1, now()) 52 | `, email) 53 | if err != nil { 54 | t.Error("Error insert temp table: ", err) 55 | t.FailNow() 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /test/config_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/Ringloop/mr-plow/internal/config" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | type readerComplexTest struct{} 11 | 12 | // 'readerTest' implementing the Interface 13 | func (*readerComplexTest) ReadConfig() ([]byte, error) { 14 | 15 | testComplexConfig := ` 16 | pollingSeconds: 5 17 | database: "databaseValue" 18 | queries: 19 | - query: "query_0_Value" 20 | index: "index_0_Value" 21 | updateDate: "test0" 22 | - query: "query_1_Value" 23 | index: "index_1_Value" 24 | updateDate: "test1" 25 | elastic: 26 | url: http://localhost:9200 27 | ` 28 | 29 | // Prepare data you want to return without reading from the file 30 | return []byte(testComplexConfig), nil 31 | } 32 | 33 | func TestGetComplexConfig(t *testing.T) { 34 | testReader := readerComplexTest{} 35 | configVal, err := config.ParseConfiguration(&testReader) 36 | if err != nil { 37 | t.Errorf("Parsing config, got error %s", err) 38 | } 39 | 40 | require.Equal(t, err, nil) 41 | require.Equal(t, configVal.Database, "databaseValue") 42 | require.Equal(t, configVal.Queries[0].Query, "query_0_Value") 43 | require.Equal(t, configVal.Queries[0].Index, "index_0_Value") 44 | require.Equal(t, configVal.Queries[0].UpdateDate, "test0") 45 | require.Equal(t, configVal.Queries[1].Query, "query_1_Value") 46 | require.Equal(t, configVal.Queries[1].Index, "index_1_Value") 47 | require.Equal(t, configVal.Queries[1].UpdateDate, "test1") 48 | } 49 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Ci Integration Test 2 | on: push 3 | 4 | jobs: 5 | container-job: 6 | runs-on: ubuntu-latest 7 | services: 8 | postgres: 9 | image: postgres:14.1 10 | env: 11 | POSTGRES_USER: user 12 | POSTGRES_PASSWORD: pwd 13 | POSTGRES_DB: postgres 14 | ports: 15 | - 5432:5432 16 | 17 | options: >- 18 | --health-cmd pg_isready 19 | --health-interval 10s 20 | --health-timeout 5s 21 | --health-retries 5 22 | 23 | steps: 24 | - name: Check out repository code 25 | uses: actions/checkout@v2 26 | 27 | #official elastic github action: https://github.com/elastic/elastic-github-actions/tree/master/elasticsearch 28 | - name: Configure sysctl limits 29 | run: | 30 | sudo swapoff -a 31 | sudo sysctl -w vm.swappiness=1 32 | sudo sysctl -w fs.file-max=262144 33 | sudo sysctl -w vm.max_map_count=262144 34 | 35 | - name: Runs Elasticsearch 36 | uses: elastic/elastic-github-actions/elasticsearch@master 37 | with: 38 | stack-version: 7.6.0 39 | 40 | - name: Set up Go 41 | uses: actions/setup-go@v2 42 | with: 43 | go-version: 1.16 44 | 45 | - name: Build 46 | run: go build -v ./... 47 | 48 | - name: Run coverage 49 | run: go test -v ./... -race -coverprofile=tmp.txt -coverpkg ./... && cat tmp.txt | grep -v starter.go | grep -v test | grep -v filereader > coverage.txt 50 | 51 | - name: Upload coverage to Codecov 52 | run: bash <(curl -s https://codecov.io/bash) 53 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/elastic/go-elasticsearch v0.0.0 h1:Pd5fqOuBxKxv83b0+xOAJDAkziWYwFinWnBO0y+TZaA= 4 | github.com/elastic/go-elasticsearch v0.0.0/go.mod h1:TkBSJBuTyFdBnrNqoPc54FN0vKf5c04IdM4zuStJ7xg= 5 | github.com/elastic/go-elasticsearch/v7 v7.15.1 h1:Wd8RLHb5D8xPBU8vGlnLXyflkso9G+rCmsXjqH8LLQQ= 6 | github.com/elastic/go-elasticsearch/v7 v7.15.1/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= 7 | github.com/elastic/go-elasticsearch/v7 v7.16.0 h1:GHsxDFXIAlhSleXun4kwA89P7kQFADRChqvgOPeYP5A= 8 | github.com/elastic/go-elasticsearch/v7 v7.16.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= 9 | github.com/elastic/go-elasticsearch/v7 v7.17.0 h1:0fcSh4qeC/i1+7QU1KXpmq2iUAdMk4l0/vmbtW1+KJM= 10 | github.com/elastic/go-elasticsearch/v7 v7.17.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= 11 | github.com/elastic/go-elasticsearch/v8 v8.0.0-20211021114623-d823a44f1eb7 h1:6/QGW6vlqpRAtbyHt8fEYN9CEFx/PNa4oqi6dUksdm4= 12 | github.com/elastic/go-elasticsearch/v8 v8.0.0-20211021114623-d823a44f1eb7/go.mod h1:xe9a/L2aeOgFKKgrO3ibQTnMdpAeL0GC+5/HpGScSa4= 13 | github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= 14 | github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 15 | github.com/lib/pq v1.10.3 h1:v9QZf2Sn6AmjXtQeFpdoq/eaNtYP6IN+7lcrygsIAtg= 16 | github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 17 | github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= 18 | github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 19 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 20 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 21 | github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= 22 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 23 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 24 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 25 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 26 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 27 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 28 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 29 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 30 | -------------------------------------------------------------------------------- /test/scheduling_integration_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | "time" 7 | 8 | "github.com/Ringloop/mr-plow/internal/scheduler" 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/Ringloop/mr-plow/internal/elastic" 12 | _ "github.com/lib/pq" 13 | ) 14 | 15 | func TestSchedulingIntegration(t *testing.T) { 16 | //given (some data on sql db) 17 | conf := initConfigIntegrationTest(t, &insertIntegrationTest{}) 18 | db := initSqlDB(t, conf) 19 | defer db.Close() 20 | repo, err := elastic.NewDefaultClient() 21 | if err != nil { 22 | t.Error("error in creating elastic connection", err) 23 | t.FailNow() 24 | } 25 | repo.Delete(conf.Queries[0].Index) 26 | 27 | insertData(db, "mario@rossi.it", t) 28 | 29 | //when (starting the scheduler) 30 | finished := make(chan bool) 31 | s := scheduler.NewScheduler() 32 | go s.MoveDataUntilExit(conf, db, &conf.Queries[0], finished) 33 | time.Sleep(2 * time.Second) 34 | 35 | indexContent1, err := repo.FindIndexContent("out_index", "last_update") 36 | defer (*indexContent1).Close() 37 | if err != nil { 38 | t.Error(err) 39 | t.FailNow() 40 | } 41 | 42 | var response1 ElasticTestResponse 43 | if err := json.NewDecoder(*indexContent1).Decode(&response1); err != nil { 44 | t.Error(err) 45 | t.FailNow() 46 | } 47 | 48 | require.Equal(t, len(response1.Hits.Hits), 1) 49 | require.Equal(t, response1.Hits.Hits[0].Source.Email, "mario@rossi.it") 50 | require.NotNil(t, response1.Hits.Hits[0].Source.LastUpdate) 51 | require.NotNil(t, response1.Hits.Hits[0].Source.UserID) 52 | 53 | //and when (inserting new data) 54 | insertData(db, "mario@rossi.it", t) 55 | 56 | // and then (the data is moved) 57 | time.Sleep(2 * time.Second) 58 | indexContent2, err := repo.FindIndexContent("out_index", "last_update") 59 | defer (*indexContent2).Close() 60 | if err != nil { 61 | t.Error(err) 62 | t.FailNow() 63 | } 64 | 65 | var response2 ElasticTestResponse 66 | if err := json.NewDecoder(*indexContent2).Decode(&response2); err != nil { 67 | t.Error(err) 68 | t.FailNow() 69 | } 70 | 71 | require.Equal(t, len(response2.Hits.Hits), 2) 72 | 73 | s.Done <- FakeExitSignal{} 74 | 75 | //and when (inserting new data again) 76 | time.Sleep(2 * time.Second) 77 | insertData(db, "mario@rossi.it", t) 78 | 79 | //and then, nothing new is extracted 80 | indexContent3, err := repo.FindIndexContent("out_index", "last_update") 81 | defer (*indexContent3).Close() 82 | if err != nil { 83 | t.Error(err) 84 | t.FailNow() 85 | } 86 | 87 | var response3 ElasticTestResponse 88 | if err := json.NewDecoder(*indexContent3).Decode(&response3); err != nil { 89 | t.Error(err) 90 | t.FailNow() 91 | } 92 | 93 | require.Equal(t, <-finished, true) 94 | require.Equal(t, len(response3.Hits.Hits), len(response2.Hits.Hits)) 95 | } 96 | -------------------------------------------------------------------------------- /internal/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "log" 7 | 8 | "gopkg.in/yaml.v2" 9 | ) 10 | 11 | type JSONField struct { 12 | FieldName string `yaml:"fieldName"` 13 | Fields []Field `yaml:"fields"` 14 | } 15 | 16 | type Field struct { 17 | Name string `yaml:"name"` 18 | Type string `yaml:"type"` 19 | } 20 | 21 | type QueryModel struct { 22 | Index string `yaml:"index"` 23 | Query string `yaml:"query"` 24 | UpdateDate string `yaml:"updateDate"` 25 | Fields []Field `yaml:"fields"` 26 | JSONFields []JSONField `yaml:"JSONFields"` 27 | Id string `yaml:"id"` 28 | } 29 | 30 | type ElasticConfig struct { 31 | Url string `yaml:"url"` 32 | User string `yaml:"user"` 33 | Password string `yaml:"password"` 34 | CaCertPath string `yaml:"caCertPath"` 35 | NumWorker int `yaml:"numWorker"` 36 | } 37 | 38 | type ImportConfig struct { 39 | PollingSeconds int `yaml:"pollingSeconds"` 40 | Database string `yaml:"database"` 41 | Queries []QueryModel `yaml:"queries"` 42 | Elastic ElasticConfig `yaml:"elastic"` 43 | } 44 | 45 | func ParseConfiguration(reader IReader) (*ImportConfig, error) { 46 | yamlFile, err := reader.ReadConfig() 47 | if err != nil { 48 | return nil, err 49 | } 50 | 51 | importConfiguration := &ImportConfig{} 52 | err = yaml.Unmarshal(yamlFile, importConfiguration) 53 | if err != nil { 54 | return &ImportConfig{}, err 55 | } 56 | 57 | if importConfiguration.PollingSeconds == 0 { 58 | return nil, errors.New("missing polling seconds url (pollingSeconds)") 59 | } 60 | 61 | if importConfiguration.Database == "" { 62 | return nil, errors.New("missing database url (database)") 63 | } 64 | 65 | if len(importConfiguration.Queries) == 0 { 66 | return nil, errors.New("missing query configuration (queries)") 67 | } 68 | 69 | err = validateQueriesConfig(importConfiguration) 70 | 71 | if importConfiguration.Elastic.Url == "" { 72 | return nil, errors.New("missing elastic url (elastic.url)") 73 | } 74 | 75 | if importConfiguration.Elastic.NumWorker < 1 { 76 | importConfiguration.Elastic.NumWorker = 1 77 | log.Println("using default worker = 1 for each elasticsearch indexer") 78 | } 79 | 80 | if err != nil { 81 | return nil, err 82 | } else { 83 | return importConfiguration, nil 84 | } 85 | } 86 | 87 | func validateQueriesConfig(importConfiguration *ImportConfig) error { 88 | for i, query := range importConfiguration.Queries { 89 | if query.Index == "" { 90 | return fmt.Errorf("missing output index for query %d (queries.index)", i) 91 | } 92 | 93 | if query.Query == "" { 94 | return fmt.Errorf("missing query for query %d (queries.query)", i) 95 | } 96 | 97 | if query.UpdateDate == "" { 98 | return fmt.Errorf("missing update date for query %d (queries.updateDate)", i) 99 | } 100 | 101 | if err := validateJsonFields(query.JSONFields, i); err != nil { 102 | return err 103 | } 104 | } 105 | return nil 106 | } 107 | 108 | func validateJsonFields(_ []JSONField, _ int) error { 109 | //TODO 110 | return nil 111 | } 112 | -------------------------------------------------------------------------------- /internal/casting/converter.go: -------------------------------------------------------------------------------- 1 | package casting 2 | 3 | import ( 4 | "reflect" 5 | "strconv" 6 | "strings" 7 | ) 8 | 9 | type Converter struct { 10 | inputTypeMap map[string]string 11 | } 12 | 13 | func NewConverter(inputTypeMap map[string]string) *Converter { 14 | converter := &Converter{ 15 | inputTypeMap: inputTypeMap} 16 | return converter 17 | } 18 | 19 | func (converter *Converter) CastSingleElement(inputName string, inputData interface{}) interface{} { 20 | if columnType, ok := converter.inputTypeMap[inputName]; ok { 21 | switch strings.ToLower(columnType) { 22 | case "string": 23 | return castToString(inputData) 24 | case "integer": 25 | return castToInt(inputData) 26 | case "float": 27 | return castToFloat(inputData) 28 | case "boolean": 29 | return castToBool(inputData) 30 | default: 31 | return inputData 32 | } 33 | } else { 34 | return inputData 35 | } 36 | } 37 | 38 | func (converter *Converter) CastArrayOfData(inputNameArray []string, inputDataArray []interface{}) []interface{} { 39 | castedColumns := make([]interface{}, len(inputDataArray)) 40 | 41 | for i := range inputDataArray { 42 | castedColumns[i] = converter.CastSingleElement(inputNameArray[i], inputDataArray[i]) 43 | } 44 | 45 | return castedColumns 46 | } 47 | 48 | func castToString(inputVar interface{}) string { 49 | switch varType := reflect.TypeOf(inputVar).String(); varType { 50 | case "bool": 51 | return strconv.FormatBool(inputVar.(bool)) 52 | case "float64": 53 | return strconv.FormatFloat(inputVar.(float64), 'f', -1, 64) 54 | case "int": 55 | return strconv.Itoa(inputVar.(int)) 56 | } 57 | return inputVar.(string) 58 | } 59 | 60 | func castToInt(inputVar interface{}) int { 61 | switch varType := reflect.TypeOf(inputVar).String(); varType { 62 | case "string": 63 | if strings.TrimSpace(inputVar.(string)) == "" { 64 | return 0 65 | } 66 | res, err := strconv.Atoi(inputVar.(string)) //have to manage this error 67 | if err != nil { 68 | return 0 69 | } 70 | return res 71 | case "bool": 72 | if inputVar.(bool) { 73 | return 1 74 | } else { 75 | return 0 76 | } 77 | case "float64": 78 | return int(inputVar.(float64)) 79 | case "int64": 80 | return int(inputVar.(int64)) 81 | } 82 | 83 | return inputVar.(int) 84 | } 85 | 86 | func castToFloat(inputVar interface{}) float64 { 87 | switch varType := reflect.TypeOf(inputVar).String(); varType { 88 | case "string": 89 | if strings.TrimSpace(inputVar.(string)) == "" { 90 | return 0. 91 | } 92 | inputVar = strings.ReplaceAll(inputVar.(string), ",", "") 93 | res, err := strconv.ParseFloat(inputVar.(string), 64) //have to manage this error 94 | if err != nil { 95 | return 0. 96 | } 97 | return res 98 | case "bool": 99 | if inputVar.(bool) { 100 | return 1. 101 | } else { 102 | return 0. 103 | } 104 | case "int": 105 | return float64(inputVar.(int)) 106 | } 107 | return inputVar.(float64) 108 | } 109 | 110 | func castToBool(inputVar interface{}) bool { 111 | switch varType := reflect.TypeOf(inputVar).String(); varType { 112 | case "string": 113 | if strings.EqualFold("true", inputVar.(string)) { 114 | return true 115 | } else { 116 | return false 117 | } 118 | case "float64": 119 | if inputVar.(float64) == 0. { 120 | return false 121 | } else { 122 | return true 123 | } 124 | case "int": 125 | if inputVar.(int) == 0 { 126 | return false 127 | } else { 128 | return true 129 | } 130 | } 131 | return inputVar.(bool) 132 | } 133 | -------------------------------------------------------------------------------- /test/upsert_integration_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "testing" 7 | 8 | "github.com/Ringloop/mr-plow/internal/elastic" 9 | "github.com/Ringloop/mr-plow/internal/movedata" 10 | _ "github.com/lib/pq" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | type upsertIntegrationTest struct{} 15 | 16 | // test case config scenario 17 | func (*upsertIntegrationTest) ReadConfig() ([]byte, error) { 18 | 19 | testComplexConfig := ` 20 | pollingSeconds: 5 21 | database: "postgres://user:pwd@localhost:5432/postgres?sslmode=disable" 22 | queries: 23 | - query: "select * from test.table1 where last_update > $1" 24 | index: "out_index" 25 | updateDate: "last_update" 26 | id: "email" 27 | elastic: 28 | url: http://localhost:9200 29 | ` 30 | 31 | // Prepare data you want to return without reading from the file 32 | return []byte(testComplexConfig), nil 33 | } 34 | 35 | func TestUpsertIntegration(t *testing.T) { 36 | //given (some data on sql db) 37 | conf := initConfigIntegrationTest(t, &upsertIntegrationTest{}) 38 | db := initSqlDB(t, conf) 39 | defer db.Close() 40 | repo, err := elastic.NewDefaultClient() 41 | if err != nil { 42 | t.Error("error in creating elastic connection", err) 43 | t.FailNow() 44 | } 45 | repo.Delete(conf.Queries[0].Index) 46 | 47 | insertData(db, "mario@rossi.it", t) 48 | originalLastDate, err := repo.FindLastUpdateOrEpochDate(conf.Queries[0].Index, conf.Queries[0].UpdateDate) 49 | if err != nil { 50 | t.Error("error in getting last date", err) 51 | t.FailNow() 52 | } 53 | 54 | //when (moving data to elastic) 55 | mover := movedata.New(db, conf, &conf.Queries[0]) 56 | err = mover.MoveData() 57 | if err != nil { 58 | t.Error("error data moving", err) 59 | t.FailNow() 60 | } 61 | 62 | //then (last date on elastic should be updated) 63 | lastImportedDate, err := repo.FindLastUpdateOrEpochDate(conf.Queries[0].Index, conf.Queries[0].UpdateDate) 64 | if err != nil { 65 | t.Error("error in getting last date", err) 66 | t.FailNow() 67 | } 68 | 69 | log.Println("original date", originalLastDate) 70 | log.Println("date after import", lastImportedDate) 71 | 72 | if !lastImportedDate.After(*originalLastDate) { 73 | t.Error("error date not incremented!") 74 | t.FailNow() 75 | } 76 | 77 | indexContent1, err := repo.FindIndexContent("out_index", "last_update") 78 | defer (*indexContent1).Close() 79 | if err != nil { 80 | t.Error(err) 81 | t.FailNow() 82 | } 83 | 84 | var response1 ElasticTestResponse 85 | if err := json.NewDecoder(*indexContent1).Decode(&response1); err != nil { 86 | t.Error(err) 87 | t.FailNow() 88 | } 89 | 90 | require.Equal(t, len(response1.Hits.Hits), 1, "Test should extract exactly ONE result from Elastic") 91 | require.Equal(t, response1.Hits.Hits[0].Source.Email, "mario@rossi.it", "Email not valid") 92 | require.NotNil(t, response1.Hits.Hits[0].Source.LastUpdate, "Last Update should not be NIL") 93 | require.NotNil(t, response1.Hits.Hits[0].Source.UserID, "UserID should not be null") 94 | require.Equal(t, response1.Hits.Hits[0].ID, "mario@rossi.it") 95 | 96 | //and when (inserting new data) 97 | insertData(db, "mario@rossi.it", t) 98 | 99 | // and then (the data is moved) 100 | err = mover.MoveData() 101 | if err != nil { 102 | t.Error("error data moving", err) 103 | t.FailNow() 104 | } 105 | 106 | indexContent2, err := repo.FindIndexContent("out_index", "last_update") 107 | defer (*indexContent2).Close() 108 | if err != nil { 109 | t.Error(err) 110 | t.FailNow() 111 | } 112 | 113 | var response2 ElasticTestResponse 114 | if err := json.NewDecoder(*indexContent2).Decode(&response2); err != nil { 115 | t.Error(err) 116 | t.FailNow() 117 | } 118 | 119 | require.Equal(t, len(response2.Hits.Hits), 1) 120 | } 121 | -------------------------------------------------------------------------------- /test/complete_config_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/Ringloop/mr-plow/internal/config" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | type readerCompleteTest struct{} 11 | 12 | // 'readerTest' implementing the Interface 13 | func (*readerCompleteTest) ReadConfig() ([]byte, error) { 14 | 15 | testCompleteConfig := ` 16 | pollingSeconds: 5 17 | database: databaseValue 18 | queries: 19 | - index: index_1 20 | query: select * from table_1 21 | updateDate: test01 22 | fields: 23 | - name: name 24 | type: String 25 | - name: last_update 26 | type: Date 27 | JSONFields: 28 | - fieldName: dataField_1 29 | fields: 30 | - name: attribute_1_Name 31 | type: attribute_1_Type 32 | - fieldName: dataField_2 33 | fields: 34 | - name: attribute_2_Name 35 | type: attribute_2_Type 36 | - name: attribute_2_1_Name 37 | type: attribute_2_1_Type 38 | id: MyId_1 39 | - index: index_2 40 | query: select * from table_2 41 | updateDate: test02 42 | JSONFields: 43 | - fieldName: dataField_2 44 | fields: 45 | - name: attribute_1_Name_2 46 | type: attribute_1_Type_2 47 | id: MyId_2 48 | elastic: 49 | url: http://localhost:9200 50 | ` 51 | 52 | // Prepare data you want to return without reading from the file 53 | return []byte(testCompleteConfig), nil 54 | } 55 | 56 | func TestGetCompleteConfig(t *testing.T) { 57 | testReader := readerCompleteTest{} 58 | configVal, err := config.ParseConfiguration(&testReader) 59 | if err != nil { 60 | t.Errorf("Parsing config, got error %s", err) 61 | } 62 | 63 | require.NoError(t, err) 64 | require.Equal(t, configVal.Database, "databaseValue") 65 | queries := configVal.Queries 66 | require.Equal(t, len(queries), 2) 67 | //test queries[0] 68 | require.Equal(t, queries[0].Index, "index_1") 69 | require.Equal(t, queries[0].Query, "select * from table_1") 70 | jsonFields1 := queries[0].JSONFields 71 | require.Equal(t, len(jsonFields1), 2) 72 | require.Equal(t, jsonFields1[0].FieldName, "dataField_1") 73 | require.Equal(t, queries[0].UpdateDate, "test01") 74 | require.Equal(t, queries[0].Id, "MyId_1") 75 | queryFields := queries[0].Fields 76 | require.Equal(t, queryFields[0].Name, "name") 77 | require.Equal(t, queryFields[0].Type, "String") 78 | require.Equal(t, queryFields[1].Name, "last_update") 79 | require.Equal(t, queryFields[1].Type, "Date") 80 | 81 | jsonFields1 = queries[0].JSONFields 82 | require.Equal(t, len(jsonFields1), 2) 83 | require.Equal(t, jsonFields1[0].FieldName, "dataField_1") 84 | attribute1JsonFields1 := jsonFields1[0] 85 | require.Equal(t, attribute1JsonFields1.Fields[0].Name, "attribute_1_Name") 86 | require.Equal(t, attribute1JsonFields1.Fields[0].Type, "attribute_1_Type") 87 | require.Equal(t, jsonFields1[1].FieldName, "dataField_2") 88 | attribute1JsonFields2 := jsonFields1[1] 89 | require.Equal(t, attribute1JsonFields2.Fields[0].Type, "attribute_2_Type") 90 | require.Equal(t, attribute1JsonFields2.Fields[0].Name, "attribute_2_Name") 91 | require.Equal(t, attribute1JsonFields2.Fields[1].Name, "attribute_2_1_Name") 92 | require.Equal(t, attribute1JsonFields2.Fields[1].Type, "attribute_2_1_Type") 93 | 94 | //test queries[1] 95 | require.Equal(t, queries[1].Index, "index_2") 96 | require.Equal(t, queries[1].Query, "select * from table_2") 97 | require.Equal(t, queries[1].UpdateDate, "test02") 98 | require.Equal(t, queries[1].Id, "MyId_2") 99 | jsonFields2 := queries[1].JSONFields 100 | require.Equal(t, len(jsonFields2), 1) 101 | require.Equal(t, jsonFields2[0].FieldName, "dataField_2") 102 | attribute2JsonFields1 := jsonFields2[0].Fields 103 | require.Equal(t, attribute2JsonFields1[0].Name, "attribute_1_Name_2") 104 | require.Equal(t, attribute2JsonFields1[0].Type, "attribute_1_Type_2") 105 | } 106 | -------------------------------------------------------------------------------- /test/insert_integration_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "sync" 7 | "testing" 8 | 9 | "github.com/Ringloop/mr-plow/internal/elastic" 10 | "github.com/Ringloop/mr-plow/internal/movedata" 11 | _ "github.com/lib/pq" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | type insertIntegrationTest struct{} 16 | 17 | // test case config scenario 18 | func (*insertIntegrationTest) ReadConfig() ([]byte, error) { 19 | 20 | testComplexConfig := ` 21 | pollingSeconds: 1 22 | database: "postgres://user:pwd@localhost:5432/postgres?sslmode=disable" 23 | queries: 24 | - query: "select * from test.table1 where last_update > $1" 25 | index: "out_index" 26 | updateDate: "last_update" 27 | elastic: 28 | url: http://localhost:9200 29 | ` 30 | 31 | // Prepare data you want to return without reading from the file 32 | return []byte(testComplexConfig), nil 33 | } 34 | 35 | func TestInsertIntegration(t *testing.T) { 36 | //given (some data on sql db) 37 | conf := initConfigIntegrationTest(t, &insertIntegrationTest{}) 38 | db := initSqlDB(t, conf) 39 | defer db.Close() 40 | repo, err := elastic.NewDefaultClient() 41 | if err != nil { 42 | t.Error("error in creating elastic connection", err) 43 | t.FailNow() 44 | } 45 | repo.Delete(conf.Queries[0].Index) 46 | 47 | insertData(db, "mario@rossi.it", t) 48 | originalLastDate, err := repo.FindLastUpdateOrEpochDate(conf.Queries[0].Index, conf.Queries[0].UpdateDate) 49 | if err != nil { 50 | t.Error("error in getting last date", err) 51 | t.FailNow() 52 | } 53 | 54 | //when (moving data to elastic 55 | var allMovesDone sync.WaitGroup 56 | allMovesDone.Add(5) 57 | mover := movedata.New(db, conf, &conf.Queries[0]) 58 | doMove := func() { 59 | defer allMovesDone.Done() 60 | errRoutine := mover.MoveData() 61 | if errRoutine != nil { 62 | t.Error("error data moving", err) 63 | t.FailNow() 64 | } 65 | } 66 | 67 | //(testing also long-running execution by executing the function as separate go routine)) 68 | for i := 0; i < 5; i++ { 69 | go doMove() 70 | } 71 | allMovesDone.Wait() 72 | 73 | //then (last date on elastic should be updated) 74 | lastImportedDate, err := repo.FindLastUpdateOrEpochDate(conf.Queries[0].Index, conf.Queries[0].UpdateDate) 75 | if err != nil { 76 | t.Error("error in getting last date", err) 77 | t.FailNow() 78 | } 79 | 80 | log.Println("original date", originalLastDate) 81 | log.Println("date after import", lastImportedDate) 82 | 83 | if !lastImportedDate.After(*originalLastDate) { 84 | t.Error("error date not incremented!") 85 | t.FailNow() 86 | } 87 | 88 | indexContent1, err := repo.FindIndexContent("out_index", "last_update") 89 | defer (*indexContent1).Close() 90 | if err != nil { 91 | t.Error(err) 92 | t.FailNow() 93 | } 94 | 95 | var response1 ElasticTestResponse 96 | if err := json.NewDecoder(*indexContent1).Decode(&response1); err != nil { 97 | t.Error(err) 98 | t.FailNow() 99 | } 100 | 101 | require.Equal(t, len(response1.Hits.Hits), 1) 102 | require.Equal(t, response1.Hits.Hits[0].Source.Email, "mario@rossi.it") 103 | require.NotNil(t, response1.Hits.Hits[0].Source.LastUpdate) 104 | require.NotNil(t, response1.Hits.Hits[0].Source.UserID) 105 | 106 | //and when (inserting new data) 107 | insertData(db, "mario@rossi.it", t) 108 | 109 | // and then (the data is moved) 110 | err = mover.MoveData() 111 | if err != nil { 112 | t.Error("error data moving", err) 113 | t.FailNow() 114 | } 115 | 116 | indexContent2, err := repo.FindIndexContent("out_index", "last_update") 117 | defer (*indexContent2).Close() 118 | if err != nil { 119 | t.Error(err) 120 | t.FailNow() 121 | } 122 | 123 | var response2 ElasticTestResponse 124 | if err := json.NewDecoder(*indexContent2).Decode(&response2); err != nil { 125 | t.Error(err) 126 | t.FailNow() 127 | } 128 | 129 | require.Equal(t, len(response2.Hits.Hits), 2) 130 | } 131 | -------------------------------------------------------------------------------- /test/insert_typed_integration_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "sync" 7 | "testing" 8 | 9 | "github.com/Ringloop/mr-plow/internal/elastic" 10 | "github.com/Ringloop/mr-plow/internal/movedata" 11 | _ "github.com/lib/pq" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | type insertTypedIntegrationTest struct{} 16 | 17 | // test case config scenario 18 | func (*insertTypedIntegrationTest) ReadConfig() ([]byte, error) { 19 | 20 | return []byte(` 21 | pollingSeconds: 1 22 | database: "postgres://user:pwd@localhost:5432/postgres?sslmode=disable" 23 | queries: 24 | - query: "select * from test.table1 where last_update > $1" 25 | index: "out_index" 26 | updateDate: "last_update" 27 | fields: 28 | - name: email 29 | type: String 30 | - name: user_id 31 | type: Integer 32 | elastic: 33 | url: http://localhost:9200 34 | `), nil 35 | } 36 | 37 | func TestInsertTypedIntegration(t *testing.T) { 38 | //given (some data on sql db) 39 | conf := initConfigIntegrationTest(t, &insertTypedIntegrationTest{}) 40 | db := initSqlDB(t, conf) 41 | defer db.Close() 42 | repo, err := elastic.NewDefaultClient() 43 | if err != nil { 44 | t.Error("error in creating elastic connection", err) 45 | t.FailNow() 46 | } 47 | repo.Delete(conf.Queries[0].Index) 48 | 49 | insertData(db, "mario@rossi.it", t) 50 | originalLastDate, err := repo.FindLastUpdateOrEpochDate(conf.Queries[0].Index, conf.Queries[0].UpdateDate) 51 | if err != nil { 52 | t.Error("error in getting last date", err) 53 | t.FailNow() 54 | } 55 | 56 | //when (moving data to elastic 57 | var allMovesDone sync.WaitGroup 58 | allMovesDone.Add(5) 59 | mover := movedata.New(db, conf, &conf.Queries[0]) 60 | doMove := func() { 61 | defer allMovesDone.Done() 62 | errRoutine := mover.MoveData() 63 | if errRoutine != nil { 64 | t.Error("error data moving", err) 65 | t.FailNow() 66 | } 67 | } 68 | 69 | //(testing also long-running execution by executing the function as separate go routine)) 70 | for i := 0; i < 5; i++ { 71 | go doMove() 72 | } 73 | allMovesDone.Wait() 74 | 75 | //then (last date on elastic should be updated) 76 | lastImportedDate, err := repo.FindLastUpdateOrEpochDate(conf.Queries[0].Index, conf.Queries[0].UpdateDate) 77 | if err != nil { 78 | t.Error("error in getting last date", err) 79 | t.FailNow() 80 | } 81 | 82 | log.Println("original date", originalLastDate) 83 | log.Println("date after import", lastImportedDate) 84 | 85 | if !lastImportedDate.After(*originalLastDate) { 86 | t.Error("error date not incremented!") 87 | t.FailNow() 88 | } 89 | 90 | indexContent1, err := repo.FindIndexContent("out_index", "last_update") 91 | defer (*indexContent1).Close() 92 | if err != nil { 93 | t.Error(err) 94 | t.FailNow() 95 | } 96 | 97 | var response1 ElasticTestResponse 98 | if err := json.NewDecoder(*indexContent1).Decode(&response1); err != nil { 99 | t.Error(err) 100 | t.FailNow() 101 | } 102 | 103 | require.Equal(t, len(response1.Hits.Hits), 1) 104 | require.Equal(t, response1.Hits.Hits[0].Source.Email, "mario@rossi.it") 105 | require.NotNil(t, response1.Hits.Hits[0].Source.LastUpdate) 106 | require.NotNil(t, response1.Hits.Hits[0].Source.UserID) 107 | 108 | //and when (inserting new data) 109 | insertData(db, "mario@rossi.it", t) 110 | 111 | // and then (the data is moved) 112 | err = mover.MoveData() 113 | if err != nil { 114 | t.Error("error data moving", err) 115 | t.FailNow() 116 | } 117 | 118 | indexContent2, err := repo.FindIndexContent("out_index", "last_update") 119 | defer (*indexContent2).Close() 120 | if err != nil { 121 | t.Error(err) 122 | t.FailNow() 123 | } 124 | 125 | var response2 ElasticTestResponse 126 | if err := json.NewDecoder(*indexContent2).Decode(&response2); err != nil { 127 | t.Error(err) 128 | t.FailNow() 129 | } 130 | 131 | require.Equal(t, len(response2.Hits.Hits), 2) 132 | } 133 | -------------------------------------------------------------------------------- /test/invalid_config_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/Ringloop/mr-plow/internal/config" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | type invalidConf1 struct{} 11 | 12 | func (*invalidConf1) ReadConfig() ([]byte, error) { 13 | yml := ` 14 | #pollingSeconds: 5 missing polling... 15 | database: "databaseValue" 16 | queries: 17 | - query: "query_0_Value" 18 | index: "index_0_Value" 19 | updateDate: "test0" 20 | - query: "query_1_Value" 21 | index: "index_1_Value" 22 | updateDate: "test1" 23 | elastic: 24 | url: http://localhost:9200 25 | ` 26 | return []byte(yml), nil 27 | } 28 | 29 | type invalidConf2 struct{} 30 | 31 | func (*invalidConf2) ReadConfig() ([]byte, error) { 32 | yml := ` 33 | pollingSeconds: 5 34 | #database: "databaseValue" missing db... 35 | queries: 36 | - query: "query_0_Value" 37 | index: "index_0_Value" 38 | updateDate: "test0" 39 | - query: "query_1_Value" 40 | index: "index_1_Value" 41 | updateDate: "test1" 42 | elastic: 43 | url: http://localhost:9200 44 | ` 45 | return []byte(yml), nil 46 | } 47 | 48 | type invalidConf3 struct{} 49 | 50 | func (*invalidConf3) ReadConfig() ([]byte, error) { 51 | yml := ` 52 | pollingSeconds: 5 53 | database: "databaseValue" 54 | elastic: 55 | url: http://localhost:9200 56 | ` 57 | return []byte(yml), nil 58 | } 59 | 60 | type invalidConf4 struct{} 61 | 62 | func (*invalidConf4) ReadConfig() ([]byte, error) { 63 | yml := ` 64 | pollingSeconds: 5 65 | database: "databaseValue" 66 | queries: 67 | - query: "query_0_Value" 68 | index: "index_0_Value" 69 | updateDate: "test0" 70 | - query: "query_1_Value" 71 | index: "index_1_Value" 72 | updateDate: "test1" 73 | ` 74 | return []byte(yml), nil 75 | } 76 | 77 | type invalidConf5 struct{} 78 | 79 | func (*invalidConf5) ReadConfig() ([]byte, error) { 80 | yml := ` 81 | pollingSeconds: 5 82 | database: "databaseValue" 83 | queries: 84 | - query: "query_0_Value" 85 | updateDate: "test0" 86 | - query: "query_1_Value" 87 | index: "index_1_Value" 88 | updateDate: "test1" 89 | elastic: 90 | url: http://localhost:9200 91 | ` 92 | return []byte(yml), nil 93 | } 94 | 95 | type invalidConf6 struct{} 96 | 97 | func (*invalidConf6) ReadConfig() ([]byte, error) { 98 | yml := ` 99 | pollingSeconds: 5 100 | database: "databaseValue" 101 | queries: 102 | - index: "index_0_Value" 103 | updateDate: "test0" 104 | - query: "query_1_Value" 105 | index: "index_1_Value" 106 | updateDate: "test1" 107 | elastic: 108 | url: http://localhost:9200 109 | ` 110 | return []byte(yml), nil 111 | } 112 | 113 | type invalidConf7 struct{} 114 | 115 | func (*invalidConf7) ReadConfig() ([]byte, error) { 116 | yml := ` 117 | pollingSeconds: 5 118 | database: "databaseValue" 119 | queries: 120 | - query: "query_0_Value" 121 | index: "index_0_Value" 122 | - query: "query_1_Value" 123 | index: "index_1_Value" 124 | updateDate: "test1" 125 | elastic: 126 | url: http://localhost:9200 127 | ` 128 | return []byte(yml), nil 129 | } 130 | 131 | func TestInvalidConfig(t *testing.T) { 132 | _, err := config.ParseConfiguration(&invalidConf1{}) 133 | require.Equal(t, err.Error(), "missing polling seconds url (pollingSeconds)") 134 | 135 | _, err = config.ParseConfiguration(&invalidConf2{}) 136 | require.Equal(t, err.Error(), "missing database url (database)") 137 | 138 | _, err = config.ParseConfiguration(&invalidConf3{}) 139 | require.Equal(t, err.Error(), "missing query configuration (queries)") 140 | 141 | _, err = config.ParseConfiguration(&invalidConf4{}) 142 | require.Equal(t, err.Error(), "missing elastic url (elastic.url)") 143 | 144 | _, err = config.ParseConfiguration(&invalidConf5{}) 145 | require.NotNil(t, err) 146 | _, err = config.ParseConfiguration(&invalidConf6{}) 147 | require.NotNil(t, err) 148 | _, err = config.ParseConfiguration(&invalidConf7{}) 149 | require.NotNil(t, err) 150 | 151 | } 152 | -------------------------------------------------------------------------------- /test/integration_json_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "log" 7 | "testing" 8 | 9 | "github.com/Ringloop/mr-plow/internal/config" 10 | "github.com/Ringloop/mr-plow/internal/elastic" 11 | "github.com/Ringloop/mr-plow/internal/movedata" 12 | _ "github.com/lib/pq" 13 | ) 14 | 15 | func initSqlDB_local(t *testing.T, conf *config.ImportConfig) *sql.DB { 16 | db, err := sql.Open("postgres", conf.Database) 17 | if err != nil { 18 | t.Error("error connecting to sql db", err) 19 | t.FailNow() 20 | } 21 | 22 | _, err = db.Exec(` 23 | 24 | DROP SCHEMA IF EXISTS test CASCADE; 25 | CREATE SCHEMA test; 26 | 27 | DROP TABLE IF EXISTS test.table1; 28 | CREATE TABLE test.table1 ( 29 | user_id SERIAL PRIMARY KEY, 30 | email VARCHAR ( 255 ) UNIQUE NOT NULL, 31 | additional_info JSON, 32 | optional_info JSON, 33 | last_update TIMESTAMP NOT NULL 34 | ) 35 | 36 | `) 37 | 38 | if err != nil { 39 | t.Error("error creating schema", err) 40 | t.FailNow() 41 | } 42 | 43 | return db 44 | } 45 | 46 | func TestIntegrationWithJSON(t *testing.T) { 47 | //given (some data on sql db) 48 | conf := initConfigIntegrationTestWithJson(t) 49 | db := initSqlDB_local(t, conf) 50 | defer db.Close() 51 | 52 | repo, err := elastic.NewDefaultClient() 53 | if err != nil { 54 | t.Error("error in creating elastic connection", err) 55 | t.FailNow() 56 | } 57 | repo.Delete(conf.Queries[0].Index) 58 | 59 | email := "mario@rossi.it" 60 | data_json := ` 61 | { 62 | "str_col": "String Data", 63 | "int_col": "4237", 64 | "bool_col": "true", 65 | "float_col": "48.94065780742467", 66 | "optional_string": "Optional Value" 67 | }` 68 | opt_json := ` 69 | { 70 | "opt_str": "Optional String", 71 | "opt_int": 8978, 72 | "opt_bool": true, 73 | "opt_float": 32.547545 74 | }` 75 | insertDataWithJSON(db, email, data_json, opt_json, t) 76 | originalLastDate, err := repo.FindLastUpdateOrEpochDate(conf.Queries[0].Index, conf.Queries[0].UpdateDate) 77 | if err != nil { 78 | t.Error("error in getting last date", err) 79 | t.FailNow() 80 | } 81 | 82 | //when (moving data to elastic) 83 | mover := movedata.New(db, conf, &conf.Queries[0]) 84 | err = mover.MoveData() 85 | if err != nil { 86 | t.Error("error data moving", err) 87 | t.FailNow() 88 | } 89 | 90 | //then (last date on elastic should be updated) 91 | lastImportedDate, err := repo.FindLastUpdateOrEpochDate(conf.Queries[0].Index, conf.Queries[0].UpdateDate) 92 | if err != nil { 93 | t.Error("error in getting last date", err) 94 | t.FailNow() 95 | } 96 | 97 | log.Println("JSON_TEST: original date", originalLastDate) 98 | log.Println("JSON_TEST: date after import", lastImportedDate) 99 | 100 | if !lastImportedDate.After(*originalLastDate) { 101 | t.Error("error date not incremented!") 102 | t.FailNow() 103 | } 104 | 105 | } 106 | 107 | type readerIntegrationTestWithJson struct{} 108 | 109 | // 'readerTest' implementing the Interface 110 | func (*readerIntegrationTestWithJson) ReadConfig() ([]byte, error) { 111 | 112 | configIntegrationWithJson := ` 113 | pollingSeconds: 5 114 | database: "postgres://user:pwd@localhost:5432/postgres?sslmode=disable" 115 | queries: 116 | - index: "out_index" 117 | query: "select * from test.table1 where last_update > $1" 118 | updateDate: "last_update" 119 | JSONFields: 120 | - fieldName: additional_info 121 | fields: 122 | - name: str_col 123 | type: string 124 | - name: int_col 125 | type: integer 126 | - name: bool_col 127 | type: boolean 128 | - name: float_col 129 | type: float 130 | elastic: 131 | url: http://localhost:9200 132 | ` 133 | 134 | // Prepare data you want to return without reading from the file 135 | return []byte(configIntegrationWithJson), nil 136 | } 137 | 138 | func initConfigIntegrationTestWithJson(t *testing.T) *config.ImportConfig { 139 | testReader := readerIntegrationTestWithJson{} 140 | conf, err := config.ParseConfiguration(&testReader) 141 | if err != nil { 142 | t.Error("error reading conf", err) 143 | t.FailNow() 144 | } 145 | return conf 146 | } 147 | 148 | func insertDataWithJSON(db *sql.DB, email, info_json, opt_json string, t *testing.T) { 149 | sql_statement := fmt.Sprintf(` 150 | INSERT INTO test.table1 (email, additional_info, optional_info, last_update) 151 | VALUES ('%s', '%s', '%s', now()); 152 | `, email, info_json, opt_json) 153 | _, err := db.Exec(sql_statement) 154 | if err != nil { 155 | t.Error("Error insert temp table: ", err) 156 | t.FailNow() 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /internal/elastic/indexer.go: -------------------------------------------------------------------------------- 1 | package elastic 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "log" 10 | "strings" 11 | "time" 12 | 13 | "github.com/Ringloop/mr-plow/internal/config" 14 | 15 | "github.com/elastic/go-elasticsearch/v7" 16 | "github.com/elastic/go-elasticsearch/v7/esapi" 17 | "github.com/elastic/go-elasticsearch/v7/esutil" 18 | ) 19 | 20 | type Repository struct { 21 | es *elasticsearch.Client 22 | numWorkers int 23 | flushBytes int 24 | flushInterval time.Duration 25 | } 26 | 27 | func NewDefaultClient() (*Repository, error) { 28 | if es, err := elasticsearch.NewDefaultClient(); err != nil { 29 | return &Repository{}, err 30 | } else { 31 | return &Repository{ 32 | es: es, 33 | numWorkers: 1, 34 | flushBytes: 100000, 35 | flushInterval: 30 * time.Second}, nil 36 | } 37 | } 38 | 39 | func NewClient(config *config.ImportConfig) (*Repository, error) { 40 | cfg := elasticsearch.Config{ 41 | Addresses: []string{ 42 | config.Elastic.Url, 43 | }, 44 | } 45 | 46 | if config.Elastic.User != "" { 47 | cfg.Username = config.Elastic.User 48 | cfg.Password = config.Elastic.Password 49 | } 50 | 51 | if config.Elastic.CaCertPath != "" { 52 | cert, err := ioutil.ReadFile(config.Elastic.CaCertPath) 53 | if err != nil { 54 | return nil, err 55 | } 56 | cfg.CACert = cert 57 | } 58 | 59 | if es, err := elasticsearch.NewClient(cfg); err != nil { 60 | return &Repository{}, err 61 | } else { 62 | return &Repository{ 63 | es: es, 64 | numWorkers: config.Elastic.NumWorker, 65 | flushBytes: 100000, 66 | flushInterval: 30 * time.Second}, nil 67 | } 68 | } 69 | 70 | func (repo *Repository) GetBulkIndexer(index string) (esutil.BulkIndexer, error) { 71 | bi, err := esutil.NewBulkIndexer(esutil.BulkIndexerConfig{ 72 | Index: index, 73 | Client: repo.es, 74 | NumWorkers: repo.numWorkers, 75 | FlushBytes: repo.flushBytes, 76 | FlushInterval: repo.flushInterval, 77 | }) 78 | if err != nil { 79 | return nil, fmt.Errorf("error getting bulkIndexer: %s", err) 80 | } 81 | return bi, nil 82 | } 83 | 84 | func (repo *Repository) FindLastUpdateOrEpochDate(index, sortingField string) (*time.Time, error) { 85 | lastDate, err := repo.FindLastUpdate(index, sortingField) 86 | if err != nil { 87 | return nil, err 88 | } 89 | 90 | if lastDate == nil { 91 | log.Printf("cannot find old values for %s", sortingField) 92 | var defaultDate time.Time 93 | defaultDate, err = time.Parse(time.RFC3339, "1970-01-01T00:00:00+00:00") 94 | lastDate = &defaultDate 95 | } 96 | 97 | return lastDate, err 98 | } 99 | 100 | func (repo *Repository) FindLastUpdate(index, sortingField string) (*time.Time, error) { 101 | err := repo.Refresh(index) 102 | if err != nil { 103 | return nil, err 104 | } 105 | var query = ` 106 | { 107 | "sort": [ 108 | { 109 | "$order": { 110 | "order": "desc" 111 | } 112 | } 113 | ], 114 | "size": 1, 115 | "_source": [ 116 | "$order" 117 | ] 118 | } 119 | ` 120 | query = replaceOrderByField(query, sortingField) 121 | 122 | var mapResp map[string]interface{} 123 | 124 | res, err := repo.es.Search( 125 | repo.es.Search.WithContext(context.Background()), 126 | repo.es.Search.WithIndex(index), 127 | repo.es.Search.WithBody(strings.NewReader(query)), 128 | ) 129 | 130 | if err != nil { 131 | return nil, err 132 | } else { 133 | defer res.Body.Close() 134 | err := json.NewDecoder(res.Body).Decode(&mapResp) 135 | if err != nil { 136 | return nil, err 137 | } 138 | 139 | if mapResp["hits"] == nil { 140 | return nil, nil //index non existing 141 | } 142 | 143 | hits := mapResp["hits"].(map[string]interface{}) 144 | hitsList := hits["hits"].([]interface{}) 145 | if len(hitsList) == 0 { 146 | return nil, nil //no data in the index 147 | } 148 | 149 | data := hitsList[0].(map[string]interface{})["_source"].(map[string]interface{}) 150 | last_update := data[sortingField].(string) 151 | log.Println("found old data:", last_update) 152 | 153 | parsed_last_date, err := time.Parse(time.RFC3339, last_update) 154 | return &parsed_last_date, err 155 | 156 | } 157 | 158 | } 159 | 160 | func (repo *Repository) FindIndexContent(index, sortingField string) (*io.ReadCloser, error) { 161 | err := repo.Refresh(index) 162 | if err != nil { 163 | return nil, err 164 | } 165 | var query = ` 166 | { 167 | "sort": [ 168 | { 169 | "$order": { 170 | "order": "desc" 171 | } 172 | } 173 | ], 174 | "size": 1000 175 | } 176 | ` 177 | query = replaceOrderByField(query, sortingField) 178 | 179 | res, err := repo.es.Search( 180 | repo.es.Search.WithContext(context.Background()), 181 | repo.es.Search.WithIndex(index), 182 | repo.es.Search.WithBody(strings.NewReader(query)), 183 | ) 184 | 185 | if err != nil { 186 | return nil, err 187 | } else { 188 | return &res.Body, nil 189 | } 190 | } 191 | 192 | func replaceOrderByField(query, sortingField string) string { 193 | query = strings.Replace(query, "$order", sortingField, 2) 194 | return query 195 | } 196 | 197 | func (repo *Repository) Refresh(index string) error { 198 | r := esapi.IndicesRefreshRequest{ 199 | Index: []string{index}, 200 | } 201 | _, err := r.Do(context.Background(), repo.es) 202 | return err 203 | } 204 | 205 | func (repo *Repository) Delete(index string) error { 206 | _, err := repo.es.Indices.Delete([]string{index}) 207 | return err 208 | } 209 | -------------------------------------------------------------------------------- /internal/movedata/move.go: -------------------------------------------------------------------------------- 1 | package movedata 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "database/sql" 7 | "encoding/json" 8 | "fmt" 9 | "log" 10 | "time" 11 | 12 | "github.com/Ringloop/mr-plow/internal/casting" 13 | "github.com/Ringloop/mr-plow/internal/config" 14 | "github.com/Ringloop/mr-plow/internal/elastic" 15 | 16 | "github.com/elastic/go-elasticsearch/v7/esutil" 17 | _ "github.com/lib/pq" 18 | ) 19 | 20 | type Mover struct { 21 | lastDate *time.Time 22 | db *sql.DB 23 | globalConfig *config.ImportConfig 24 | queryConf *config.QueryModel 25 | columnsMap map[string]string 26 | jsonColsMap map[string]map[string]string 27 | canExec chan bool 28 | } 29 | 30 | func New(db *sql.DB, globalConfig *config.ImportConfig, queryConf *config.QueryModel) *Mover { 31 | columnsMap, jsonColsMap := getColumnsConfiguration(queryConf) 32 | 33 | mover := &Mover{ 34 | db: db, 35 | globalConfig: globalConfig, 36 | queryConf: queryConf, 37 | columnsMap: columnsMap, 38 | jsonColsMap: jsonColsMap, 39 | canExec: make(chan bool, 1)} 40 | mover.canExec <- true 41 | return mover 42 | } 43 | 44 | func getColumnsConfiguration(queryConf *config.QueryModel) (map[string]string, map[string]map[string]string) { 45 | //create the map for the native fields of the query 46 | columnsMap := make(map[string]string) 47 | for _, colConfig := range queryConf.Fields { 48 | columnsMap[colConfig.Name] = colConfig.Type 49 | } 50 | 51 | //create a nested map for the JSON fields (any JSON contains a set of fields) 52 | var jsonColsMap = make(map[string]map[string]string) 53 | for _, jsonColConfig := range queryConf.JSONFields { 54 | for _, colConfig := range jsonColConfig.Fields { 55 | if jsonColsMap[jsonColConfig.FieldName] == nil { 56 | jsonColsMap[jsonColConfig.FieldName] = make(map[string]string) 57 | } 58 | jsonColsMap[jsonColConfig.FieldName][colConfig.Name] = colConfig.Type 59 | } 60 | } 61 | return columnsMap, jsonColsMap 62 | } 63 | 64 | func (mover *Mover) MoveData() error { 65 | select { 66 | case <-mover.canExec: 67 | default: 68 | log.Printf("Skipping execution of %s, since the previous one is still executing", mover.queryConf.Query) 69 | return nil 70 | } 71 | 72 | defer func() { 73 | mover.canExec <- true 74 | }() 75 | 76 | repo, err := elastic.NewClient(mover.globalConfig) 77 | if err != nil { 78 | return err 79 | } 80 | 81 | lastDate, err := mover.getLastDate(repo) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | elasticBulk, err := repo.GetBulkIndexer(mover.queryConf.Index) 87 | if err != nil { 88 | return err 89 | } 90 | defer elasticBulk.Close(context.Background()) 91 | 92 | log.Printf("going to execute query %s with param %s", mover.queryConf.Query, lastDate) 93 | rows, err := mover.db.Query(mover.queryConf.Query, lastDate) 94 | if err != nil { 95 | return err 96 | } 97 | cols, err := rows.Columns() 98 | if err != nil { 99 | return err 100 | } 101 | 102 | for rows.Next() { 103 | columns := make([](interface{}), len(cols)) 104 | for i := range columns { 105 | columns[i] = &columns[i] 106 | } 107 | 108 | if err := rows.Scan(columns...); err != nil { 109 | return err 110 | } 111 | converter := casting.NewConverter(mover.columnsMap) 112 | convertedArrayOfData := converter.CastArrayOfData(cols, columns) 113 | document := make(map[string]interface{}) 114 | for i, colName := range cols { 115 | document[colName] = convertedArrayOfData[i] 116 | } 117 | for _, jsonfield := range mover.queryConf.JSONFields { 118 | var jsonData map[string]interface{} 119 | 120 | data, ok := document[jsonfield.FieldName] 121 | if !ok { 122 | return fmt.Errorf("error getting ....: %s", err) 123 | } 124 | byteData := data.([]byte) 125 | 126 | err := json.Unmarshal(byteData, &jsonData) 127 | if err != nil { 128 | return err 129 | } 130 | for _, field := range jsonfield.Fields { 131 | jsonConverter := casting.NewConverter(mover.jsonColsMap[jsonfield.FieldName]) 132 | jsonData[field.Name] = jsonConverter.CastSingleElement(field.Name, jsonData[field.Name]) 133 | } 134 | document[jsonfield.FieldName] = jsonData 135 | } 136 | 137 | documentToSend, err := json.Marshal(document) 138 | if err != nil { 139 | return err 140 | } 141 | 142 | bulkItem := esutil.BulkIndexerItem{ 143 | Action: "index", 144 | Body: bytes.NewBuffer(documentToSend), 145 | OnFailure: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem, err error) { 146 | if err != nil { 147 | log.Printf("ERROR: %s", err) 148 | } else { 149 | log.Printf("ERROR: %s: %s", res.Error.Type, res.Error.Reason) 150 | } 151 | }, 152 | } 153 | addDocumentId(mover.queryConf, document, &bulkItem) 154 | err = mover.updateLastUpdate(mover.queryConf, document) 155 | if err != nil { 156 | return err 157 | } 158 | 159 | err = elasticBulk.Add(context.Background(), bulkItem) 160 | if err != nil { 161 | return err 162 | } 163 | } 164 | return nil 165 | } 166 | 167 | func (mover *Mover) getLastDate(repo *elastic.Repository) (*time.Time, error) { 168 | if mover.lastDate != nil { 169 | return mover.lastDate, nil 170 | } 171 | 172 | lastDate, err := repo.FindLastUpdateOrEpochDate(mover.queryConf.Index, mover.queryConf.UpdateDate) 173 | if err != nil { 174 | return nil, err 175 | } 176 | log.Print("found last date ", lastDate) 177 | return lastDate, nil 178 | } 179 | 180 | func (mover *Mover) updateLastUpdate(conf *config.QueryModel, document map[string]interface{}) error { 181 | date, ok := document[conf.UpdateDate] 182 | if !ok { 183 | return fmt.Errorf("cannot find %s in results set of %s", conf.UpdateDate, conf.Query) 184 | } 185 | dateParsed, ok := date.(time.Time) 186 | if !ok { 187 | return fmt.Errorf("cannot cast to date %s in results set of %s", date, conf.Query) 188 | } 189 | mover.lastDate = &dateParsed 190 | return nil 191 | } 192 | 193 | func addDocumentId(queryConf *config.QueryModel, document map[string]interface{}, bulkItem *esutil.BulkIndexerItem) { 194 | if queryConf.Id != "" { 195 | id, present := document[queryConf.Id] 196 | if present { 197 | idAsString := fmt.Sprint(id) 198 | bulkItem.DocumentID = idAsString 199 | } 200 | } 201 | } 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![example workflow](https://github.com/Ringloop/mr-plow/actions/workflows/ci.yml/badge.svg) 2 | [![codecov](https://codecov.io/gh/Ringloop/mr-plow/branch/main/graph/badge.svg?token=PE53PJ8HHR)](https://codecov.io/gh/Ringloop/mr-plow) 3 | 4 | # Mr-Plow 5 | Tiny and minimal tool to export data from relational db (postgres or mysql) to elasticsearch. 6 | 7 | The tool does not implement all the logstash features, but its goal is to be an alternative to logstash when keeping in-sync elastic and a relational database. 8 | 9 | ### Goal 10 | **Low memory usage**: (~15 MB when idle, great to be deployed on cloud environments). 11 | 12 | **Stateless**: a timestamp/date column is used in order to filter inserted/update data and to avoid fetching already seen data. 13 | During the startup Mr-Plow checks the data inserted into elasticsearch to check the last timestamp/date of the transferred data, and so it does not require a local state. 14 | 15 | ![image](https://user-images.githubusercontent.com/7256185/141697554-4e6f86d8-06e4-4c22-aea5-30145e40fc41.png ) 16 | 17 | ### Usage: 18 | Mr-Plow essentially executes queries on a relational database and writes these data to ElasticSearch. 19 | The configured queries are run in parallel, and data are written incrementally, it's only sufficient to specify a timestamp/date column in the queries in order to get only newly updated/inserted data. 20 | 21 | This is a basic configuration template example, where we only specify two queries and the endpoint configuration (one Postgres database and one ElasticSearch cluster: 22 | ```yaml 23 | # example of config.yml 24 | pollingSeconds: 5 #database polling interval 25 | database: "postgres://user:pwd@localhost:5432/postgres?sslmode=disable" #specify here the db connection 26 | queries: #put here one of more queries (each one will be executed in parallel): 27 | - query: "select * from my_table1 where last_update > $1" #please add a filter on an incrementing date/ts column using the $1 value as param 28 | index: "table1_index" #name of the elastic output index 29 | updateDate: "last_update" #name of the incrementing date column 30 | id: "customer_id" #optional, column to use as elasticsearch id 31 | - query: "select * from my_table2 where ts > $1" 32 | index: "table2_index" 33 | updateDate: "ts" 34 | elastic: 35 | url: "http://localhost:9200" 36 | user: "elastic_user" #optional 37 | password: "my_secret" #optional 38 | numWorker: 10 #optional, number of worker for indexing each query 39 | caCertPath: "my/path/ca" #optional, path of custom CA file (it may be needed in some HTTPS connection..) 40 | ``` 41 | 42 | Anyway, Mr Plow has also additional features, for example interacting with a database like Postgres, supporting JSON columns, we can specify JSON fields, in order to create a complex (nested) object to be created in Elastic. In the following, we show an example where in the Employee table we store two dynamic JSON fields, one containing the Payment Data and another one containing additional informations for the employee: 43 | 44 | ```yaml 45 | pollingSeconds: 5 46 | database: databaseValue 47 | queries: 48 | - index: index_1 49 | query: select * from employees 50 | updateDate: last_update 51 | JSONFields: 52 | - fieldName: payment_data 53 | - fieldName: additional_infos 54 | id: MyId_1 55 | ``` 56 | 57 | And additionally, we can specify the type expected for some specific fields. Please note hat field type is optional and if not specified, the field is casted as String. 58 | 59 | Actually supported type are: String, Integer, Float and Boolean 60 | 61 | ```yaml 62 | pollingSeconds: 5 63 | database: databaseValue 64 | queries: 65 | - index: index_1 66 | query: select * from employees 67 | updateDate: last_update 68 | fields: # Optional config, casting standard sql columns to specific data type 69 | - name: name 70 | type: String 71 | - name: working_hours 72 | type: Integer 73 | ``` 74 | Merging the previous two examples, we can apply the type casting also to inner JSON fields, here is a complete example of configuration: 75 | 76 | ```yaml 77 | pollingSeconds: 5 78 | database: databaseValue 79 | queries: 80 | - index: index_1 81 | query: select * from employees 82 | updateDate: last_update 83 | fields: # Optional, casting standard sql columns to specific data type 84 | - name: name 85 | type: String 86 | - name: working_hours 87 | type: Integer 88 | JSONFields: 89 | - fieldName: payment_info 90 | fields: # Optional, casting json fields to specific data type 91 | - name: bank_account 92 | type: String 93 | - name: validated 94 | type: Boolean 95 | id: MyId_1 96 | ``` 97 | 98 | Download or build the binary (docker images will be released soon): 99 | ```bash 100 | make 101 | ``` 102 | 103 | Run the tool: 104 | ```bash 105 | ./bin/mr-plow -config /path/to/my/config.yml 106 | ``` 107 | 108 | To build as docker image, create a `config.yml` and put into the root folder of the project. Then run: 109 | ```bash 110 | docker build . 111 | ``` 112 | 113 | ### Mr-Plow development with Visual Studio Code 114 | 115 | **Requirements** 116 | 117 | - [Docker](https://docs.docker.com/) 118 | - [Visual Studio Code](https://code.visualstudio.com/) 119 | 120 | Linux and Mac users should ensure to have the following programs installed on their system: 121 | 122 | - `bash`, `id`, `getent` 123 | 124 | Windows users must be aware that they should clone Mr-Plow repository over the `WSL` filesystem. 125 | This is the recommended way because mounting a `NTFS` filesystem inside a container exposes the overall user's experience to major issues. 126 | Windows users should also patch the `.devcontainer/devcontainer.json` as indicated in the comments inside the file. 127 | 128 | **Steps** 129 | 130 | 1. Clone the project to a local folder 131 | 2. VSCode -> `><` (left bottom button) -> Open Folder in Container... 132 | 133 | **Instructions** 134 | 135 | Users can develop and test Mr-Plow inside a docker container without being forced to install or configure anything on your own machine. 136 | Visual Studio Code can take care of automatically download and build the developer's docker image. 137 | For Linux and Mac users, an especial care has been devoted to make sure the host's user will match `UID` and `GID` with the user inside the container. 138 | This ensures that every modification from inside the container will be completely transparent from the host's perspective. 139 | Moreover, host's user `~.ssh` directory will be mounted on the container's user `~.ssh` directory. This is especially convenient if an ssh authentication type is configured to work with GitHub. 140 | From inside the container, users will be able to access the host's docker engine as if they were just in a regular host's shell. 141 | This capability allows users to launch the predefined `docker-compose` images directly from Visual Studio Code. 142 | Users can simply access to the task menu pressing: `ctrl + shift + p` and select `Docker: Compose Up`. 143 | Therefore, they can choose to spawn up the following services: 144 | 145 | - `docker-compose-elastic.yml`: ElasticSearch 146 | - `docker-compose-kibana.yml`: Kibana 147 | - `docker-compose-postgres.yml`: Postgres 148 | 149 | -------------------------------------------------------------------------------- /test/type_conversion_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/Ringloop/mr-plow/internal/casting" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | //func CastSingleElement(inputTypeMap map[string]string, inputName string, inputData interface{}) interface{} { 11 | func prepareInputType() map[string]string { 12 | 13 | inputTypeMap := make(map[string]string) 14 | inputTypeMap["stringElement"] = "string" 15 | inputTypeMap["intElement"] = "integer" 16 | inputTypeMap["boolElement"] = "boolean" 17 | inputTypeMap["floatElement"] = "float" 18 | 19 | return inputTypeMap 20 | } 21 | 22 | func TestStringConvertion(t *testing.T) { 23 | var intElement int = 5 24 | var stringElement string = "StringValue" 25 | var stringEmptyElement string = "" 26 | var boolElement bool = true 27 | var floatElement float64 = 4.54 28 | 29 | var ok bool 30 | 31 | inputMap := prepareInputType() 32 | 33 | converter := casting.NewConverter(inputMap) 34 | var convertedString = converter.CastSingleElement("stringElement", intElement) 35 | _, ok = convertedString.(string) 36 | require.True(t, ok) 37 | require.Equal(t, convertedString, "5") 38 | convertedString = converter.CastSingleElement("stringElement", stringEmptyElement) 39 | _, ok = convertedString.(string) 40 | require.True(t, ok) 41 | require.Equal(t, convertedString, "") 42 | convertedString = converter.CastSingleElement("stringElement", stringElement) 43 | _, ok = convertedString.(string) 44 | require.True(t, ok) 45 | require.Equal(t, convertedString, "StringValue") 46 | convertedString = converter.CastSingleElement("stringElement", boolElement) 47 | _, ok = convertedString.(string) 48 | require.True(t, ok) 49 | require.Equal(t, convertedString, "true") 50 | convertedString = converter.CastSingleElement("stringElement", floatElement) 51 | _, ok = convertedString.(string) 52 | require.True(t, ok) 53 | require.Equal(t, convertedString, "4.54") 54 | 55 | } 56 | 57 | func TestIntegerConvertion(t *testing.T) { 58 | var intElement int = 5 59 | var stringElement string = "5" 60 | var stringEmptyElement string = "" 61 | var stringSpacesElement string = " " 62 | var stringAlphanumericElement string = "sglhsdg8478" 63 | var boolElement bool = true 64 | var floatElement float64 = 5. 65 | 66 | var ok bool 67 | 68 | inputMap := prepareInputType() 69 | 70 | converter := casting.NewConverter(inputMap) 71 | var convertedInt = converter.CastSingleElement("intElement", intElement) 72 | _, ok = convertedInt.(int) 73 | require.True(t, ok) 74 | require.Equal(t, convertedInt, 5) 75 | convertedInt = converter.CastSingleElement("intElement", stringEmptyElement) 76 | _, ok = convertedInt.(int) 77 | require.True(t, ok) 78 | require.Equal(t, convertedInt, 0) 79 | convertedInt = converter.CastSingleElement("intElement", stringSpacesElement) 80 | _, ok = convertedInt.(int) 81 | require.True(t, ok) 82 | require.Equal(t, convertedInt, 0) 83 | convertedInt = converter.CastSingleElement("intElement", stringAlphanumericElement) 84 | _, ok = convertedInt.(int) 85 | require.True(t, ok) 86 | require.Equal(t, convertedInt, 0) 87 | convertedInt = converter.CastSingleElement("intElement", stringElement) 88 | _, ok = convertedInt.(int) 89 | require.True(t, ok) 90 | require.Equal(t, convertedInt, 5) 91 | convertedInt = converter.CastSingleElement("intElement", boolElement) 92 | _, ok = convertedInt.(int) 93 | require.True(t, ok) 94 | require.Equal(t, convertedInt, 1) 95 | convertedInt = converter.CastSingleElement("intElement", floatElement) 96 | _, ok = convertedInt.(int) 97 | require.True(t, ok) 98 | require.Equal(t, convertedInt, 5) 99 | 100 | } 101 | 102 | func TestFloatConvertion(t *testing.T) { 103 | var intElement int = 5 104 | var stringElement string = "5" 105 | var stringElementComma string = "1,024,543.22" 106 | var stringEmptyElement string = "" 107 | var stringSpacesElement string = "" 108 | var stringAlphanumericElement string = "sdgsogdiso94,g09" 109 | var boolElement bool = true 110 | var floatElement float64 = 5. 111 | 112 | var ok bool 113 | 114 | inputMap := prepareInputType() 115 | converter := casting.NewConverter(inputMap) 116 | 117 | var convertedFloat = converter.CastSingleElement("floatElement", intElement) 118 | _, ok = convertedFloat.(float64) 119 | require.True(t, ok) 120 | require.Equal(t, convertedFloat, 5.) 121 | convertedFloat = converter.CastSingleElement("floatElement", stringEmptyElement) 122 | _, ok = convertedFloat.(float64) 123 | require.True(t, ok) 124 | require.Equal(t, convertedFloat, 0.) 125 | convertedFloat = converter.CastSingleElement("floatElement", stringSpacesElement) 126 | _, ok = convertedFloat.(float64) 127 | require.True(t, ok) 128 | require.Equal(t, convertedFloat, 0.) 129 | convertedFloat = converter.CastSingleElement("floatElement", stringAlphanumericElement) 130 | _, ok = convertedFloat.(float64) 131 | require.True(t, ok) 132 | require.Equal(t, convertedFloat, 0.) 133 | convertedFloat = converter.CastSingleElement("floatElement", stringElement) 134 | _, ok = convertedFloat.(float64) 135 | require.True(t, ok) 136 | require.Equal(t, convertedFloat, 5.) 137 | convertedFloat = converter.CastSingleElement("floatElement", stringElementComma) 138 | _, ok = convertedFloat.(float64) 139 | require.True(t, ok) 140 | require.Equal(t, convertedFloat, 1024543.22) 141 | convertedFloat = converter.CastSingleElement("floatElement", boolElement) 142 | _, ok = convertedFloat.(float64) 143 | require.True(t, ok) 144 | require.Equal(t, convertedFloat, 1.) 145 | convertedFloat = converter.CastSingleElement("floatElement", floatElement) 146 | _, ok = convertedFloat.(float64) 147 | require.True(t, ok) 148 | require.Equal(t, convertedFloat, 5.) 149 | 150 | } 151 | 152 | func TestBooleanConvertion(t *testing.T) { 153 | var intElementFalse int = 0 154 | var intElementDefault int = 4 155 | var stringEmptyElement string = "" 156 | var stringSpacesElement string = " " 157 | var stringAlphanumericElement string = "sglhsdg8478" 158 | var stringElementTrue string = "true" 159 | var stringElementDefault string = "AnyOtherValue" 160 | var boolElement bool = true 161 | var floatElementFalse float64 = 0. 162 | var floatElementDefault float64 = 5. 163 | 164 | var ok bool 165 | 166 | inputMap := prepareInputType() 167 | converter := casting.NewConverter(inputMap) 168 | 169 | var convertedBool = converter.CastSingleElement("boolElement", intElementFalse) 170 | _, ok = convertedBool.(bool) 171 | require.True(t, ok) 172 | require.Equal(t, convertedBool, false) 173 | convertedBool = converter.CastSingleElement("boolElement", intElementDefault) 174 | _, ok = convertedBool.(bool) 175 | require.True(t, ok) 176 | require.Equal(t, convertedBool, true) 177 | convertedBool = converter.CastSingleElement("boolElement", stringEmptyElement) 178 | _, ok = convertedBool.(bool) 179 | require.True(t, ok) 180 | require.Equal(t, convertedBool, false) 181 | convertedBool = converter.CastSingleElement("boolElement", stringSpacesElement) 182 | _, ok = convertedBool.(bool) 183 | require.True(t, ok) 184 | require.Equal(t, convertedBool, false) 185 | convertedBool = converter.CastSingleElement("boolElement", stringAlphanumericElement) 186 | _, ok = convertedBool.(bool) 187 | require.True(t, ok) 188 | require.Equal(t, convertedBool, false) 189 | convertedBool = converter.CastSingleElement("boolElement", stringElementTrue) 190 | _, ok = convertedBool.(bool) 191 | require.True(t, ok) 192 | require.Equal(t, convertedBool, true) 193 | convertedBool = converter.CastSingleElement("boolElement", stringElementDefault) 194 | _, ok = convertedBool.(bool) 195 | require.True(t, ok) 196 | require.Equal(t, convertedBool, false) 197 | convertedBool = converter.CastSingleElement("boolElement", boolElement) 198 | _, ok = convertedBool.(bool) 199 | require.True(t, ok) 200 | require.Equal(t, convertedBool, true) 201 | convertedBool = converter.CastSingleElement("boolElement", floatElementFalse) 202 | _, ok = convertedBool.(bool) 203 | require.True(t, ok) 204 | require.Equal(t, convertedBool, false) 205 | convertedBool = converter.CastSingleElement("boolElement", floatElementDefault) 206 | _, ok = convertedBool.(bool) 207 | require.True(t, ok) 208 | require.Equal(t, convertedBool, true) 209 | 210 | } 211 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------