├── .dockerignore ├── .github └── workflows │ ├── pr.yaml │ └── release.yaml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── _examples └── client │ └── py │ ├── client.py │ └── weather.csv ├── _scripts ├── fix_pb_import.py ├── gen_session_env.go ├── go_benchmark.py ├── install-cudf-env.sh ├── py_benchmark.py └── travis.go ├── api └── api.go ├── backends ├── backends.go ├── backends_test.go ├── csv │ ├── backend.go │ └── backend_test.go ├── kv │ ├── backend.go │ ├── inferschema.go │ ├── inferschema_test.go │ ├── reader.go │ ├── writer.go │ └── writer_test.go ├── stream │ ├── backend.go │ ├── reader.go │ └── writer.go ├── tsdb │ ├── backend.go │ ├── reader.go │ └── writer.go └── utils │ ├── history_server.go │ ├── utils.go │ └── utils_test.go ├── benchmark_test.go ├── builder.go ├── builder_test.go ├── client.go ├── clients └── py │ ├── .gitignore │ ├── LICENSE.txt │ ├── MANIFEST.in │ ├── Makefile │ ├── README.md │ ├── conda.recipe │ ├── build.bat │ ├── build.sh │ └── meta.yaml │ ├── dev-requirements.txt │ ├── docs │ ├── .gitignore │ ├── Makefile │ ├── api.rst │ ├── conf.py │ ├── index.rst │ ├── make.bat │ └── requirements.txt │ ├── environment-cudf.yml │ ├── environment.yml │ ├── pypi_upload.py │ ├── requirements.txt │ ├── set-version.py │ ├── setup.py │ ├── tests │ ├── conftest.py │ ├── pip_docker.py │ ├── test_benchmark.py │ ├── test_client.py │ ├── test_concurrent.py │ ├── test_cudf.py │ ├── test_http.py │ ├── test_integration.py │ ├── test_pbutils.py │ ├── test_pdutils.py │ ├── test_pip_docker.py │ ├── test_v3io_frames.py │ └── weather.csv │ └── v3io_frames │ ├── __init__.py │ ├── client.py │ ├── dtypes.py │ ├── errors.py │ ├── frames_pb2.py │ ├── frames_pb2_grpc.py │ ├── grpc.py │ ├── http.py │ ├── pbutils.py │ └── pdutils.py ├── cmd ├── framesd │ ├── Dockerfile │ └── framesd.go └── framulate │ ├── Dockerfile │ ├── README.md │ └── framulate.go ├── column.go ├── column_test.go ├── colutils.go ├── config.go ├── config_example.yaml ├── doc.go ├── frame.go ├── frame_test.go ├── frames.proto ├── framulate ├── config.go ├── framulate.go ├── scenario.go └── writeverify.go ├── go.mod ├── go.sum ├── grpc ├── client.go ├── end2end_test.go └── server.go ├── http ├── client.go ├── client_example_test.go ├── client_test.go ├── end2end_test.go ├── grafana.go ├── prof.go ├── server.go ├── server_example_test.go └── server_test.go ├── integration_test.go ├── log.go ├── log_test.go ├── marshal.go ├── pb ├── frames.pb.go ├── frames_grpc.pb.go └── methods.go ├── repeatingtask ├── pool.go ├── pool_test.go ├── task.go ├── taskgroup.go └── worker.go ├── rowiter.go ├── rowiter_test.go ├── server.go ├── sql.go ├── sql_test.go ├── test ├── basic_suite_integration_test.go ├── csv_integration_test.go ├── fileContentIterator_integration_test.go ├── kv_integration_test.go ├── kv_save_mode_integration_test.go ├── stream_integration_test.go ├── test_utils.go └── tsdb_integration_test.go ├── testdata └── weather.csv ├── types.go ├── types_test.go └── v3ioutils ├── FileContentIterator.go ├── FileContentLineIterator.go ├── asynciter.go ├── container.go ├── listDirAsyncIter.go ├── schema.go └── schema_test.go /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | -------------------------------------------------------------------------------- /.github/workflows/pr.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | name: CI 16 | 17 | on: 18 | pull_request_target: 19 | branches: 20 | - development 21 | - 'integ_[0-9]+\.[0-9]+' 22 | 23 | jobs: 24 | test-go: 25 | name: Lint & test Go code 26 | runs-on: [ self-hosted, Linux ] 27 | container: 28 | image: golang:1.24-alpine 29 | 30 | steps: 31 | - name: Dump github context 32 | run: echo "$GITHUB_CONTEXT" 33 | env: 34 | GITHUB_CONTEXT: ${{ toJson(github) }} 35 | 36 | - name: Dump runner context 37 | run: echo "$RUNNER_CONTEXT" 38 | env: 39 | RUNNER_CONTEXT: ${{ toJson(runner) }} 40 | 41 | - name: Dump github ref 42 | run: echo "$GITHUB_REF" 43 | 44 | - uses: actions/checkout@v4 45 | with: 46 | ref: refs/pull/${{ github.event.number }}/merge 47 | 48 | - name: Install Make 49 | run: apk add make 50 | 51 | - name: Lint 52 | run: make lint 53 | 54 | - name: Run Go tests 55 | run: make test-go 56 | env: 57 | V3IO_API: ${{ secrets.V3IO_API }} 58 | V3IO_ACCESS_KEY: ${{ secrets.V3IO_ACCESS_KEY }} 59 | V3IO_SESSION: container=bigdata,user=admin 60 | 61 | test-py: 62 | name: Lint & test Python code 63 | runs-on: [ self-hosted, Linux ] 64 | container: 65 | image: python:3.9.18 66 | 67 | steps: 68 | - name: Dump github context 69 | run: echo "$GITHUB_CONTEXT" 70 | env: 71 | GITHUB_CONTEXT: ${{ toJson(github) }} 72 | 73 | - name: Dump runner context 74 | run: echo "$RUNNER_CONTEXT" 75 | env: 76 | RUNNER_CONTEXT: ${{ toJson(runner) }} 77 | 78 | - name: Dump github ref 79 | run: echo "$GITHUB_REF" 80 | 81 | - uses: actions/checkout@v4 82 | with: 83 | ref: refs/pull/${{ github.event.number }}/merge 84 | 85 | - name: Install dependencies 86 | run: make python-deps 87 | 88 | - name: Lint Python code 89 | run: make flake8 90 | 91 | - uses: actions/setup-go@v3 92 | with: 93 | go-version: "^1.24.3" 94 | 95 | - name: Run Python tests 96 | run: make test-py 97 | env: 98 | V3IO_API: ${{ secrets.V3IO_API }} 99 | V3IO_ACCESS_KEY: ${{ secrets.V3IO_ACCESS_KEY }} 100 | V3IO_SESSION: container=bigdata 101 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### Go template 3 | # Binaries for programs and plugins 4 | *.exe 5 | *.exe~ 6 | *.dll 7 | *.so 8 | *.dylib 9 | 10 | # Test binary, built with `go test -c` 11 | *.test 12 | 13 | # Output of the go coverage tool, specifically when used with LiteIDE 14 | *.out 15 | 16 | # Dependency directories (remove the comment below to include it) 17 | # vendor/ 18 | 19 | ### Python template 20 | # Byte-compiled / optimized / DLL files 21 | __pycache__/ 22 | *.py[cod] 23 | *$py.class 24 | 25 | # C extensions 26 | 27 | # Distribution / packaging 28 | .Python 29 | build/ 30 | develop-eggs/ 31 | dist/ 32 | downloads/ 33 | eggs/ 34 | .eggs/ 35 | lib/ 36 | lib64/ 37 | parts/ 38 | sdist/ 39 | var/ 40 | wheels/ 41 | pip-wheel-metadata/ 42 | share/python-wheels/ 43 | *.egg-info/ 44 | .installed.cfg 45 | *.egg 46 | MANIFEST 47 | 48 | # PyInstaller 49 | # Usually these files are written by a python script from a template 50 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 51 | *.manifest 52 | *.spec 53 | 54 | # Installer logs 55 | pip-log.txt 56 | pip-delete-this-directory.txt 57 | 58 | # Unit test / coverage reports 59 | htmlcov/ 60 | .tox/ 61 | .nox/ 62 | .coverage 63 | .coverage.* 64 | .cache 65 | nosetests.xml 66 | coverage.xml 67 | *.cover 68 | *.py,cover 69 | .hypothesis/ 70 | .pytest_cache/ 71 | 72 | # Translations 73 | *.mo 74 | *.pot 75 | 76 | # Django stuff: 77 | *.log 78 | local_settings.py 79 | db.sqlite3 80 | db.sqlite3-journal 81 | 82 | # Flask stuff: 83 | instance/ 84 | .webassets-cache 85 | 86 | # Scrapy stuff: 87 | .scrapy 88 | 89 | # Sphinx documentation 90 | docs/_build/ 91 | 92 | # PyBuilder 93 | target/ 94 | 95 | # Jupyter Notebook 96 | .ipynb_checkpoints 97 | 98 | # IPython 99 | profile_default/ 100 | ipython_config.py 101 | 102 | # pyenv 103 | .python-version 104 | 105 | # pipenv 106 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 107 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 108 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 109 | # install all needed dependencies. 110 | #Pipfile.lock 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Jetbrains project settings 132 | .idea/ 133 | 134 | # Spyder project settings 135 | .spyderproject 136 | .spyproject 137 | 138 | # Rope project settings 139 | .ropeproject 140 | 141 | # mkdocs documentation 142 | /site 143 | 144 | # mypy 145 | .mypy_cache/ 146 | .dmypy.json 147 | dmypy.json 148 | 149 | # Pyre type checker 150 | .pyre/ 151 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | FRAMES_TAG ?= latest 16 | FRAMES_REPOSITORY ?= iguazio/ 17 | FRAMES_PATH ?= src/github.com/v3io/frames 18 | FRAMES_BUILD_COMMAND ?= GO111MODULE=on go build -o framesd-$(FRAMES_TAG)-$(GOOS)-$(GOARCH) -ldflags "-X main.Version=$(FRAMES_TAG)" ./cmd/framesd 19 | 20 | GOPATH ?= ~/go 21 | 22 | .PHONY: build 23 | build: 24 | docker build \ 25 | --build-arg FRAMES_VERSION=$(FRAMES_TAG) \ 26 | --file cmd/framesd/Dockerfile \ 27 | --tag $(FRAMES_REPOSITORY)frames:$(FRAMES_TAG) \ 28 | . 29 | 30 | build-framulate: 31 | docker build \ 32 | --build-arg FRAMES_VERSION=$(FRAMES_TAG) \ 33 | --file cmd/framulate/Dockerfile \ 34 | --tag $(FRAMES_REPOSITORY)framulate:$(FRAMES_TAG) \ 35 | . 36 | 37 | .PHONY: flake8 38 | flake8: 39 | cd clients/py && make flake8 40 | 41 | .PHONY: test 42 | test: test-go test-py 43 | 44 | .PHONY: test-go 45 | test-go: 46 | GO111MODULE=on go test -v -timeout 20m ./... 47 | 48 | .PHONY: test-py 49 | test-py: 50 | cd clients/py && $(MAKE) test 51 | 52 | .PHONY: wheel 53 | wheel: 54 | cd clients/py && python setup.py bdist_wheel 55 | 56 | .PHONY: python-dist 57 | python-dist: python-deps 58 | cd clients/py && $(MAKE) dist 59 | 60 | .PHONY: set-version 61 | set-version: 62 | cd clients/py && $(MAKE) set-version 63 | 64 | .PHONY: grpc 65 | grpc: grpc-go grpc-py 66 | 67 | .PHONY: grpc-go 68 | grpc-go: 69 | protoc frames.proto --go_out=pb --go-grpc_out=pb --go-grpc_opt=require_unimplemented_servers=false 70 | 71 | .PHONY: grpc-py 72 | grpc-py: 73 | cd clients/py && \ 74 | python -m grpc_tools.protoc \ 75 | -I../.. --python_out=v3io_frames\ 76 | --grpc_python_out=v3io_frames \ 77 | ../../frames.proto 78 | python _scripts/fix_pb_import.py \ 79 | clients/py/v3io_frames/frames_pb2_grpc.py 80 | 81 | .PHONY: pypi 82 | pypi: 83 | cd clients/py && make upload 84 | 85 | .PHONY: cloc 86 | cloc: 87 | cloc \ 88 | --exclude-dir=_t,.ipynb_checkpoints,_examples,_build \ 89 | . 90 | 91 | .PHONY: update-deps 92 | update-deps: update-go-deps update-py-deps update-tsdb-deps 93 | 94 | .PHONY: update-go-deps 95 | update-go-deps: 96 | go mod tidy 97 | git add go.mod go.sum 98 | @echo "Don't forget to test & commit" 99 | 100 | .PHONY: python-deps 101 | python-deps: 102 | cd clients/py && $(MAKE) sync-deps 103 | 104 | .PHONY: bench 105 | bench: 106 | @echo Go 107 | $(MAKE) bench-go 108 | @echo Python 109 | $(MAKE) bench-py 110 | 111 | .PHONY: bench-go 112 | bench-go: 113 | ./_scripts/go_benchmark.py 114 | 115 | .PHONY: bench-py 116 | bench-py: 117 | ./_scripts/py_benchmark.py 118 | 119 | .PHONY: frames-bin 120 | frames-bin: 121 | $(FRAMES_BUILD_COMMAND) 122 | 123 | .PHONY: frames 124 | frames: 125 | docker run \ 126 | --volume $(shell pwd):/go/$(FRAMES_PATH) \ 127 | --volume $(shell pwd):/go/bin \ 128 | --workdir /go/$(FRAMES_PATH) \ 129 | --env GOOS=$(GOOS) \ 130 | --env GOARCH=$(GOARCH) \ 131 | --env FRAMES_TAG=$(FRAMES_TAG) \ 132 | --platform=linux/amd64 133 | golang:1.24-alpine \ 134 | make frames-bin 135 | 136 | PHONY: gofmt 137 | gofmt: 138 | ifeq ($(shell gofmt -l .),) 139 | # gofmt OK 140 | else 141 | $(error Please run `go fmt ./...` to format the code) 142 | endif 143 | 144 | .PHONY: lint 145 | lint: gofmt 146 | -------------------------------------------------------------------------------- /_examples/client/py/client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from os.path import abspath, dirname 16 | from datetime import datetime 17 | from getpass import getuser 18 | 19 | import pandas as pd 20 | 21 | import v3io_frames as v3f 22 | 23 | here = abspath(dirname(__file__)) 24 | csv_file = '{}/weather.csv'.format(here) 25 | 26 | table = '{:%Y-%m-%dT%H:%M:%S}-{}-weather.csv'.format(datetime.now(), getuser()) 27 | 28 | print('Table: {}'.format(table)) 29 | 30 | size = 1000 31 | df = pd.read_csv(csv_file, parse_dates=['DATE']) 32 | # Example how we work with iterable of dfs, you can pass the original df 33 | # "as-is" to write 34 | dfs = [df[i*size:i*size+size] for i in range((len(df)//size)+1)] 35 | 36 | client = v3f.Client('localhost:8081') 37 | 38 | print('Writing') 39 | out = client.write('weather', table, dfs) 40 | print('Result: {}'.format(out)) 41 | 42 | print('Reading') 43 | num_dfs = num_rows = 0 44 | for df in client.read(backend='weather', table=table, max_in_message=size): 45 | print(df) 46 | num_dfs += 1 47 | num_rows += len(df) 48 | 49 | print('\nnum_dfs = {}, num_rows = {}'.format(num_dfs, num_rows)) 50 | 51 | # If you'd like to get single big DataFrame use 52 | # df = pd.concat(client.read(table=csv_file, max_in_message=1000)) 53 | -------------------------------------------------------------------------------- /_scripts/fix_pb_import.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2018 Iguazio 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | from argparse import ArgumentParser, FileType 18 | 19 | parser = ArgumentParser(description='fix gRPC import') 20 | parser.add_argument('file', help='file to fix', type=FileType('rt')) 21 | args = parser.parse_args() 22 | 23 | old_import = 'import frames_pb2 as frames__pb2' 24 | new_import = 'from . import frames_pb2 as frames__pb2' 25 | 26 | lines = [] 27 | for line in args.file: 28 | if line.startswith(old_import): 29 | lines.append(new_import + '\n') 30 | else: 31 | lines.append(line) 32 | args.file.close() 33 | 34 | with open(args.file.name, 'wt') as out: 35 | out.write(''.join(lines)) 36 | -------------------------------------------------------------------------------- /_scripts/gen_session_env.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | // Generate session information environment variable 22 | // $ export V3IO_SESSION=$(go run _scripts/gen_session_env.go \ 23 | // -user iguazio \ 24 | // -password t0ps3cr3t \ 25 | // -address backend353.iguazio.com:8081 \ 26 | // -container bigdata) 27 | 28 | package main 29 | 30 | import ( 31 | "encoding/json" 32 | "flag" 33 | "log" 34 | "os" 35 | "path" 36 | 37 | "github.com/v3io/frames/pb" 38 | ) 39 | 40 | func main() { 41 | var session pb.Session 42 | 43 | flag.StringVar(&session.Url, "url", "", "web API url") 44 | flag.StringVar(&session.Container, "container", "", "container name") 45 | flag.StringVar(&session.Password, "password", "", "password") 46 | flag.StringVar(&session.Path, "path", "", "path in container") 47 | flag.StringVar(&session.Token, "token", "", "authentication token") 48 | flag.StringVar(&session.User, "user", "", "login user") 49 | flag.Parse() 50 | 51 | log.SetFlags(0) // Remove time ... prefix 52 | if flag.NArg() != 0 { 53 | log.Fatalf("error: %s takes no arguments", path.Base(os.Args[0])) 54 | } 55 | 56 | json.NewEncoder(os.Stdout).Encode(session) 57 | } 58 | -------------------------------------------------------------------------------- /_scripts/go_benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2018 Iguazio 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from subprocess import run, PIPE 17 | import re 18 | 19 | 20 | with open('testdata/weather.csv') as fp: 21 | read_rows = sum(1 for _ in fp) - 1 22 | 23 | # See benchmark_test.go 24 | write_rows = 1982 25 | 26 | 27 | out = run(['go', 'test', '-run', '^$', '-bench', '.'], stdout=PIPE) 28 | if out.returncode != 0: 29 | raise SystemExit(1) 30 | 31 | for line in out.stdout.decode('utf-8').splitlines(): 32 | match = re.match(r'Benchmark(Read|Write)_([a-zA-Z]+).* (\d+) ns/op', line) 33 | if not match: 34 | continue 35 | op, proto, ns = match.groups() 36 | op, proto = op.lower(), proto.lower() 37 | us = int(ns) / 1000 38 | nrows = read_rows if op == 'read' else write_rows 39 | usl = us/nrows 40 | print(f'{proto:<5} {op:<5} {usl:7.3f}µs/row') 41 | -------------------------------------------------------------------------------- /_scripts/install-cudf-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Install minconda + cudf 0.5 + go SDK on Linux machine 3 | # On AWS we use NVIDIA Volta Deep Learning AMI 18.11 AMI 4 | # All that needed to run frames tests 5 | 6 | set -x 7 | set -e 8 | 9 | miniconda_sh=Miniconda3-latest-Linux-x86_64.sh 10 | miniconda_url="https://repo.anaconda.com/miniconda/${miniconda_sh}" 11 | go_tar=go1.11.5.linux-amd64.tar.gz 12 | go_url="https://dl.google.com/go/${go_tar}" 13 | 14 | # Install miniconda 15 | curl -LO ${miniconda_url} 16 | bash ${miniconda_sh} -b 17 | echo 'export PATH=${HOME}/miniconda3/bin:${PATH}' >> ~/.bashrc 18 | 19 | # Install Go SDK 20 | curl -LO ${go_url} 21 | tar xzf ${go_tar} 22 | mv go goroot 23 | echo 'export GOROOT=${HOME}/goroot' >> ~/.bashrc 24 | echo 'export PATH=${GOROOT}/bin:${PATH}' >> ~/.bashrc 25 | 26 | CONDA_INSTALL="${HOME}/miniconda3/bin/conda install -y" 27 | 28 | # Install cudf 29 | ${CONDA_INSTALL} \ 30 | -c nvidia -c rapidsai -c pytorch -c numba \ 31 | -c conda-forge -c defaults \ 32 | cudf=0.5 cuml=0.5 python=3.6 33 | ${CONDA_INSTALL} cudatoolkit=9.2 34 | 35 | # Install testing 36 | ${CONDA_INSTALL} pytest pyyaml ipython 37 | 38 | # Get frames code 39 | git clone https://github.com/v3io/frames.git 40 | 41 | # Install frames dependencies 42 | ${CONDA_INSTALL} grpcio-tools=1.16.1 protobuf=3.6.1 requests=2.21.0 43 | -------------------------------------------------------------------------------- /_scripts/py_benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2018 Iguazio 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from subprocess import run, PIPE 17 | import json 18 | from os import remove, path 19 | import re 20 | 21 | # From clients/py/Makefile 22 | bench_json = '/tmp/framesd-py-bench.json' 23 | 24 | with open('testdata/weather.csv') as fp: 25 | read_rows = sum(1 for _ in fp) - 1 26 | 27 | # See clients/py/tests/test_benchmark.py 28 | write_rows = 1982 29 | 30 | if path.exists(bench_json): 31 | remove(bench_json) 32 | 33 | 34 | out = run(['make', 'bench'], cwd='clients/py', stdout=PIPE) 35 | if out.returncode != 0: 36 | raise SystemExit(out.stdout.decode('utf-8')) 37 | 38 | 39 | with open(bench_json) as fp: 40 | data = json.load(fp) 41 | 42 | for bench in data['benchmarks']: 43 | # test_read[http-csv] 44 | match = re.match(r'test_(\w+)\[([a-z]+)', bench['name']) 45 | if not match: 46 | raise SystemExit('error: bad test name - {}'.format(bench['name'])) 47 | op, proto = match.groups() 48 | time_us = bench['stats']['mean'] * 1e6 49 | nrows = read_rows if op == 'read' else write_rows 50 | usl = time_us / nrows 51 | print(f'{proto:<5} {op:<5} {usl:7.3f}µs/row') 52 | -------------------------------------------------------------------------------- /backends/backends.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package backends 22 | 23 | import ( 24 | "fmt" 25 | "reflect" 26 | "strings" 27 | "sync" 28 | "unicode" 29 | 30 | "github.com/nuclio/logger" 31 | "github.com/pkg/errors" 32 | "github.com/v3io/frames" 33 | "github.com/v3io/frames/pb" 34 | v3io "github.com/v3io/v3io-go/pkg/dataplane" 35 | ) 36 | 37 | var ( 38 | factories map[string]Factory 39 | lock sync.RWMutex 40 | normalizeType = strings.ToLower 41 | ) 42 | 43 | // Factory is a backend factory 44 | type Factory func(logger.Logger, v3io.Context, *frames.BackendConfig, *frames.Config) (frames.DataBackend, error) 45 | 46 | // Register registers a backend factory for a type 47 | func Register(typ string, factory Factory) error { 48 | lock.Lock() 49 | defer lock.Unlock() 50 | 51 | if factories == nil { 52 | factories = make(map[string]Factory) 53 | } 54 | 55 | typ = normalizeType(typ) 56 | if _, ok := factories[typ]; ok { 57 | return fmt.Errorf("backend %q already registered", typ) 58 | } 59 | 60 | factories[typ] = factory 61 | return nil 62 | } 63 | 64 | // GetFactory returns factory for a backend 65 | func GetFactory(typ string) Factory { 66 | lock.RLock() 67 | defer lock.RUnlock() 68 | 69 | return factories[normalizeType(typ)] 70 | } 71 | 72 | var globalRequestFieldsByRequestType = map[reflect.Type]map[string]bool{ 73 | reflect.TypeOf(pb.ReadRequest{}): { 74 | "Session": true, 75 | "Backend": true, 76 | "Schema": true, 77 | "DataFormat": true, 78 | "RowLayout": true, 79 | "Table": true, 80 | "Columns": true, 81 | "Filter": true, 82 | "Join": true, 83 | "Limit": true, 84 | "MessageLimit": true, 85 | "Marker": true, 86 | "ResetIndex": true, 87 | }, 88 | reflect.TypeOf(frames.WriteRequest{}): { 89 | "Session": true, 90 | "Password": true, 91 | "Token": true, 92 | "Backend": true, 93 | "Table": true, 94 | "ImmidiateData": true, 95 | }, 96 | reflect.TypeOf(pb.DeleteRequest{}): { 97 | "Session": true, 98 | "Backend": true, 99 | "Table": true, 100 | "Filter": true, 101 | "IfMissing": true, 102 | }, 103 | } 104 | 105 | func ValidateRequest(backend string, request interface{}, allowedFields map[string]bool) error { 106 | 107 | reftype := reflect.TypeOf(request).Elem() 108 | for i := 0; i < reftype.NumField(); i++ { 109 | field := reftype.Field(i) 110 | fieldName := field.Name 111 | if unicode.IsLower(rune(fieldName[0])) { 112 | continue 113 | } 114 | fieldValue := reflect.ValueOf(request).Elem().FieldByName(fieldName).Interface() 115 | zeroValue := reflect.Zero(field.Type).Interface() 116 | if !globalRequestFieldsByRequestType[reftype][fieldName] && !allowedFields[fieldName] && !reflect.DeepEqual(fieldValue, zeroValue) { 117 | return errors.Errorf("%s cannot be used as an argument to a %s to %s backend", fieldName, reftype.Name(), backend) 118 | } 119 | } 120 | return nil 121 | } 122 | -------------------------------------------------------------------------------- /backends/backends_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package backends 22 | 23 | import ( 24 | "fmt" 25 | "strings" 26 | "testing" 27 | 28 | "github.com/nuclio/logger" 29 | "github.com/stretchr/testify/suite" 30 | "github.com/v3io/frames" 31 | "github.com/v3io/frames/pb" 32 | v3io "github.com/v3io/v3io-go/pkg/dataplane" 33 | ) 34 | 35 | type BackendsTestSuite struct { 36 | suite.Suite 37 | } 38 | 39 | // Special error return from testFactory so we can see it's this function 40 | var errorBackendsTest = fmt.Errorf("backends test") 41 | 42 | func testFactory(logger.Logger, v3io.Context, *frames.BackendConfig, *frames.Config) (frames.DataBackend, error) { 43 | return nil, errorBackendsTest 44 | } 45 | 46 | func (suite *BackendsTestSuite) TestBackends() { 47 | typ := "testBackend" 48 | err := Register(typ, testFactory) 49 | suite.Require().NoError(err) 50 | 51 | err = Register(typ, testFactory) 52 | suite.Require().Error(err) 53 | 54 | capsType := strings.ToUpper(typ) 55 | factory := GetFactory(capsType) 56 | suite.Require().NotNil(factory) 57 | 58 | _, err = factory(nil, nil, nil, nil) 59 | suite.Require().Equal(errorBackendsTest, err) 60 | } 61 | 62 | func (suite *BackendsTestSuite) TestValidateEmptyReadRequest() { 63 | err := ValidateRequest("mybackend", &pb.ReadRequest{}, nil) 64 | suite.NoError(err) 65 | } 66 | 67 | func (suite *BackendsTestSuite) TestValidateSimpleReadRequest() { 68 | err := ValidateRequest("kv", &pb.ReadRequest{Segments: []int64{5}}, map[string]bool{"Segments": true}) 69 | suite.Require().NoError(err) 70 | } 71 | 72 | func (suite *BackendsTestSuite) TestValidateBadReadRequest() { 73 | err := ValidateRequest("tsdb", &pb.ReadRequest{Segments: []int64{5}}, nil) 74 | suite.Require().Error(err) 75 | expected := "Segments cannot be used as an argument to a ReadRequest to tsdb backend" 76 | suite.Require().Equal(expected, err.Error()) 77 | } 78 | 79 | func (suite *BackendsTestSuite) TestValidateWriteRequest() { 80 | request := &frames.WriteRequest{Session: &frames.Session{}, PartitionKeys: []string{"mykey"}} 81 | err := ValidateRequest("kv", request, map[string]bool{"PartitionKeys": true}) 82 | suite.Require().NoError(err) 83 | } 84 | 85 | func (suite *BackendsTestSuite) TestValidateBadWriteRequest() { 86 | request := &frames.WriteRequest{Session: &frames.Session{}, PartitionKeys: []string{"mykey"}} 87 | err := ValidateRequest("tsdb", request, nil) 88 | suite.Require().Error(err) 89 | } 90 | 91 | func TestBackendsTestSuite(t *testing.T) { 92 | suite.Run(t, new(BackendsTestSuite)) 93 | } 94 | -------------------------------------------------------------------------------- /backends/csv/backend_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package csv 22 | 23 | import ( 24 | "os" 25 | "path" 26 | "testing" 27 | 28 | "github.com/v3io/frames" 29 | "github.com/v3io/frames/pb" 30 | ) 31 | 32 | var ( 33 | csvData = []byte(`STATION,DATE,PRCP,SNWD,SNOW,TMAX,TMIN,AWND,WDF2,WDF5,WSF2,WSF5,PGTM,FMTM 34 | GHCND:USW00094728,2000-01-01,0,-9999,0,100,11,26,250,230,72,94,1337,1337.7 35 | GHCND:USW00094728,2000-01-02,0,-9999,0,156,61,21,260,260,72,112,2313,2314.7 36 | GHCND:USW00094728,2000-01-03,0,-9999,0,178,106,30,260,250,67,94,320,321.7 37 | GHCND:USW00094728,2000-01-04,178,-9999,0,156,78,35,320,350,67,107,1819,1840.7 38 | GHCND:USW00094728,2000-01-05,0,-9999,0,83,-17,51,330,340,107,143,843,844.7 39 | GHCND:USW00094728,2000-01-06,0,-9999,0,56,-22,30,220,250,67,98,1833,1834.7 40 | GHCND:USW00094728,2000-01-07,0,-9999,0,94,17,42,300,310,103,156,1521,1601.7 41 | GHCND:USW00094728,2000-01-09,5,-9999,0,106,28,26,270,270,63,89,22,601.7 42 | GHCND:USW00094728,2000-01-10,213,-9999,0,144,67,41,280,260,94,139,1736,1758.7 43 | GHCND:USW00094728,2000-01-11,0,-9999,0,111,44,49,300,310,112,174,1203,1203.7 44 | GHCND:USW00094728,2000-01-12,0,-9999,0,83,39,39,330,330,94,161,536,610.7 45 | GHCND:USW00094728,2000-01-13,13,-9999,0,39,-78,51,90,10,103,143,1539,843.7 46 | `) 47 | numCSVRows = 12 48 | numCSVCols = 14 49 | colTypes = map[string]frames.DType{ 50 | "STATION": frames.StringType, 51 | "DATE": frames.TimeType, 52 | "PRCP": frames.IntType, 53 | "SNWD": frames.IntType, 54 | "FMTM": frames.FloatType, 55 | } 56 | ) 57 | 58 | func TestCSV(t *testing.T) { 59 | req := &frames.ReadRequest{Proto: &pb.ReadRequest{}} 60 | result := loadTempCSV(t, req) 61 | 62 | nRows := totalRows(result) 63 | if nRows != numCSVRows { 64 | t.Fatalf("# rows mismatch %d != %d", nRows, numCSVRows) 65 | } 66 | 67 | for _, frame := range result { 68 | if len(frame.Names()) != numCSVCols { 69 | t.Fatalf("# columns mismatch %d != %d", len(frame.Names()), numCSVRows) 70 | } 71 | 72 | for name, dtype := range colTypes { 73 | col, err := frame.Column(name) 74 | if err != nil { 75 | t.Fatalf("can't find column %q", name) 76 | } 77 | 78 | if col.DType() != dtype { 79 | t.Fatalf("dype mismatch %d != %d", dtype, col.DType()) 80 | } 81 | } 82 | } 83 | 84 | } 85 | 86 | func TestLimit(t *testing.T) { 87 | limit := numCSVRows - 3 88 | 89 | req := &frames.ReadRequest{Proto: &pb.ReadRequest{}} 90 | req.Proto.Limit = int64(limit) 91 | 92 | result := loadTempCSV(t, req) 93 | if nRows := totalRows(result); nRows != limit { 94 | t.Fatalf("got %d rows, expected %d", nRows, limit) 95 | } 96 | } 97 | 98 | func TestMaxInMessage(t *testing.T) { 99 | frameLimit := numCSVRows / 3 100 | 101 | req := &frames.ReadRequest{Proto: &pb.ReadRequest{}} 102 | req.Proto.MessageLimit = int64(frameLimit) 103 | 104 | result := loadTempCSV(t, req) 105 | if nRows := totalRows(result); nRows != numCSVRows { 106 | t.Fatalf("got %d rows, expected %d", nRows, numCSVRows) 107 | } 108 | 109 | for _, frame := range result { 110 | if frame.Len() > frameLimit { 111 | t.Fatalf("frame too big (%d > %d)", frame.Len(), frameLimit) 112 | } 113 | } 114 | } 115 | 116 | func totalRows(result []frames.Frame) int { 117 | total := 0 118 | for _, frame := range result { 119 | total += frame.Len() 120 | } 121 | 122 | return total 123 | } 124 | 125 | func loadTempCSV(t *testing.T, req *frames.ReadRequest) []frames.Frame { 126 | logger, err := frames.NewLogger("debug") 127 | if err != nil { 128 | t.Fatalf("can't create logger - %s", err) 129 | } 130 | 131 | csvPath, err := tmpCSV() 132 | if err != nil { 133 | t.Fatal(err) 134 | } 135 | 136 | cfg := &frames.BackendConfig{ 137 | Name: "testCsv", 138 | Type: "csv", 139 | RootDir: path.Dir(csvPath), 140 | } 141 | 142 | backend, err := NewBackend(logger, nil, cfg, nil) 143 | if err != nil { 144 | t.Fatal(err) 145 | } 146 | 147 | req.Proto.Table = path.Base(csvPath) 148 | it, err := backend.Read(req) 149 | if err != nil { 150 | t.Fatal(err) 151 | } 152 | 153 | var result []frames.Frame 154 | for it.Next() { 155 | result = append(result, it.At()) 156 | } 157 | 158 | if err := it.Err(); err != nil { 159 | t.Fatal(err) 160 | } 161 | 162 | return result 163 | } 164 | 165 | func tmpCSV() (string, error) { 166 | tmp, err := os.CreateTemp("", "csv-test") 167 | if err != nil { 168 | return "", err 169 | } 170 | 171 | _, err = tmp.Write(csvData) 172 | if err != nil { 173 | return "", nil 174 | } 175 | 176 | if err := tmp.Sync(); err != nil { 177 | return "", err 178 | } 179 | 180 | return tmp.Name(), nil 181 | } 182 | -------------------------------------------------------------------------------- /backends/stream/writer.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package stream 22 | 23 | import ( 24 | "encoding/json" 25 | "time" 26 | 27 | "github.com/nuclio/logger" 28 | "github.com/pkg/errors" 29 | "github.com/v3io/frames" 30 | "github.com/v3io/frames/backends" 31 | "github.com/v3io/v3io-go/pkg/dataplane" 32 | ) 33 | 34 | var allowedWriteRequestFields = map[string]bool{ 35 | "HaveMore": true, 36 | } 37 | 38 | func (b *Backend) Write(request *frames.WriteRequest) (frames.FrameAppender, error) { 39 | 40 | err := backends.ValidateRequest("stream", request, allowedWriteRequestFields) 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | container, tablePath, err := b.newConnection(request.Session, request.Password.Get(), request.Token.Get(), request.Table, true) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | appender := streamAppender{ 51 | request: request, 52 | container: container, 53 | tablePath: tablePath, 54 | responseChan: make(chan *v3io.Response, 1000), 55 | commChan: make(chan int, 2), 56 | logger: b.logger, 57 | } 58 | 59 | if request.ImmidiateData != nil { 60 | err := appender.Add(request.ImmidiateData) 61 | if err != nil { 62 | appender.Close() 63 | return nil, err 64 | } 65 | } 66 | 67 | return &appender, nil 68 | } 69 | 70 | type streamAppender struct { 71 | request *frames.WriteRequest 72 | container v3io.Container 73 | tablePath string 74 | responseChan chan *v3io.Response 75 | commChan chan int 76 | logger logger.Logger 77 | closed bool 78 | } 79 | 80 | // TODO: make it async 81 | func (a *streamAppender) Add(frame frames.Frame) error { 82 | if a.closed { 83 | err := errors.New("Adding on a closed stream appender") 84 | a.logger.Error(err) 85 | return err 86 | } 87 | records := make([]*v3io.StreamRecord, 0, frame.Len()) 88 | iter := frame.IterRows(true) 89 | for iter.Next() { 90 | 91 | body, err := json.Marshal(iter.Row()) 92 | if err != nil { 93 | return err 94 | } 95 | records = append(records, &v3io.StreamRecord{Data: body}) 96 | } 97 | 98 | if err := iter.Err(); err != nil { 99 | return errors.Wrap(err, "row iteration error") 100 | } 101 | 102 | _, err := a.container.PutRecordsSync(&v3io.PutRecordsInput{ 103 | Path: a.tablePath, Records: records}) 104 | 105 | return err 106 | } 107 | 108 | func (a *streamAppender) WaitForComplete(timeout time.Duration) error { 109 | return nil 110 | } 111 | 112 | func (a *streamAppender) Close() { 113 | a.closed = true 114 | } 115 | -------------------------------------------------------------------------------- /backends/utils/utils_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package utils 22 | 23 | import ( 24 | "fmt" 25 | "math" 26 | "reflect" 27 | "testing" 28 | 29 | "github.com/v3io/frames" 30 | ) 31 | 32 | func TestAppendValue(t *testing.T) { 33 | data := []int64{1, 2, 3} 34 | out, err := AppendValue(data, int64(4)) 35 | if err != nil { 36 | t.Fatal(err) 37 | } 38 | 39 | expected := []int64{1, 2, 3, 4} 40 | if !reflect.DeepEqual(out, expected) { 41 | t.Fatalf("bad append %v != %v", out, expected) 42 | } 43 | 44 | _, err = AppendValue(data, "4") 45 | if err == nil { 46 | t.Fatal("no error on type mismatch") 47 | } 48 | 49 | _, err = AppendValue([]bool{true}, false) 50 | if err == nil { 51 | t.Fatal("no error on unknown mismatch") 52 | } 53 | } 54 | 55 | func TestNewColumn(t *testing.T) { 56 | i := int64(7) 57 | out, err := NewColumn(i, 3) 58 | if err != nil { 59 | t.Fatal(err) 60 | } 61 | 62 | expected := []int64{0, 0, 0} 63 | if !reflect.DeepEqual(out, expected) { 64 | t.Fatalf("bad new column %v != %v", out, expected) 65 | } 66 | 67 | _, err = NewColumn(uint(7), 2) 68 | if err == nil { 69 | t.Fatal("no error on unknown type") 70 | } 71 | } 72 | 73 | func TestAppendNil(t *testing.T) { 74 | size := 10 75 | data, err := NewColumn(1.0, size) 76 | if err != nil { 77 | t.Fatalf("can't create data - %s", err) 78 | } 79 | 80 | col, err := frames.NewSliceColumn("col1", data) 81 | if err != nil { 82 | t.Fatalf("can't create new slice column - %s", err) 83 | } 84 | 85 | if err := AppendNil(col); err != nil { 86 | t.Fatalf("can't append nil - %s", err) 87 | } 88 | 89 | if newSize := col.Len(); newSize != size+1 { 90 | t.Fatalf("bad size change - %d != %d", newSize, size+1) 91 | } 92 | 93 | val, _ := col.FloatAt(size) 94 | if !math.IsNaN(val) { 95 | t.Fatalf("AppendNil didn't add NaN to floats (got %v)", val) 96 | } 97 | } 98 | 99 | func TestRemoveColumn(t *testing.T) { 100 | colName := func(i int) string { 101 | return fmt.Sprintf("col-%d", i) 102 | } 103 | 104 | size := 7 105 | columns := make([]frames.Column, size) 106 | for i := 0; i < size; i++ { 107 | col, err := frames.NewSliceColumn(colName(i), []int64{}) 108 | if err != nil { 109 | t.Fatalf("can't create column - %s", err) 110 | } 111 | columns[i] = col 112 | } 113 | 114 | name := colName(4) 115 | newCols := RemoveColumn(name, columns) 116 | if newSize := len(newCols); newSize != size-1 { 117 | t.Fatalf("size after remove %d (wanted %d)", newSize, size-1) 118 | } 119 | 120 | for _, col := range newCols { 121 | if col.Name() == name { 122 | t.Fatalf("column %q found after removal", name) 123 | } 124 | } 125 | 126 | newCols = RemoveColumn("no-such-column", columns) 127 | if len(newCols) != len(columns) { 128 | t.Fatal("no existing column removed") 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /benchmark_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package frames_test 22 | 23 | import ( 24 | "testing" 25 | "time" 26 | 27 | "github.com/v3io/frames" 28 | "github.com/v3io/frames/grpc" 29 | "github.com/v3io/frames/http" 30 | "github.com/v3io/frames/pb" 31 | ) 32 | 33 | var ( 34 | rreq = &pb.ReadRequest{ 35 | Backend: "csv", 36 | Table: "weather.csv", 37 | } 38 | wreq = &frames.WriteRequest{ 39 | Backend: "csv", 40 | Table: "write-bench.csv", 41 | } 42 | wRows = 1982 43 | ) 44 | 45 | // FIXME: We're measuring the speed of CSV parsing here as well 46 | func read(c frames.Client, b *testing.B) { 47 | it, err := c.Read(rreq) 48 | if err != nil { 49 | b.Fatal(err) 50 | } 51 | 52 | for it.Next() { 53 | frame := it.At() 54 | if frame.Len() == 0 { 55 | b.Fatal("empty frame") 56 | } 57 | } 58 | 59 | if err := it.Err(); err != nil { 60 | b.Fatal(err) 61 | } 62 | } 63 | 64 | func BenchmarkRead_gRPC(b *testing.B) { 65 | b.StopTimer() 66 | info := setupTest(b) 67 | defer info.killProcess() 68 | c, err := grpc.NewClient(info.grpcAddr, nil, nil) 69 | if err != nil { 70 | b.Fatal(err) 71 | } 72 | b.StartTimer() 73 | 74 | for i := 0; i < b.N; i++ { 75 | read(c, b) 76 | } 77 | } 78 | 79 | func BenchmarkRead_HTTP(b *testing.B) { 80 | b.StopTimer() 81 | info := setupTest(b) 82 | defer info.killProcess() 83 | c, err := http.NewClient(info.httpAddr, nil, nil) 84 | if err != nil { 85 | b.Fatal(err) 86 | } 87 | b.StartTimer() 88 | 89 | for i := 0; i < b.N; i++ { 90 | read(c, b) 91 | } 92 | } 93 | 94 | func write(c frames.Client, req *frames.WriteRequest, frame frames.Frame, b *testing.B) { 95 | fa, err := c.Write(req) 96 | if err != nil { 97 | b.Fatal(err) 98 | } 99 | 100 | if err := fa.Add(frame); err != nil { 101 | b.Fatal(err) 102 | } 103 | 104 | if err := fa.WaitForComplete(time.Second); err != nil { 105 | b.Fatal(err) 106 | } 107 | } 108 | 109 | func BenchmarkWrite_gRPC(b *testing.B) { 110 | b.StopTimer() 111 | info := setupTest(b) 112 | defer info.killProcess() 113 | c, err := grpc.NewClient(info.grpcAddr, nil, nil) 114 | if err != nil { 115 | b.Fatal(err) 116 | } 117 | 118 | frame := csvFrame(b, wRows) 119 | b.StartTimer() 120 | for i := 0; i < b.N; i++ { 121 | write(c, wreq, frame, b) 122 | } 123 | } 124 | 125 | func BenchmarkWrite_HTTP(b *testing.B) { 126 | b.StopTimer() 127 | info := setupTest(b) 128 | defer info.killProcess() 129 | c, err := http.NewClient(info.httpAddr, nil, nil) 130 | if err != nil { 131 | b.Fatal(err) 132 | } 133 | 134 | frame := csvFrame(b, wRows) 135 | b.StartTimer() 136 | for i := 0; i < b.N; i++ { 137 | write(c, wreq, frame, b) 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /builder_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package frames 22 | 23 | import ( 24 | "testing" 25 | ) 26 | 27 | func TestSliceBuilder(t *testing.T) { 28 | name := "intCol" 29 | dtype := IntType 30 | size := 10 31 | b := NewSliceColumnBuilder(name, dtype, size/3) 32 | for i := 0; i < size; i++ { 33 | if err := b.Set(i, i); err != nil { 34 | t.Fatal(err) 35 | } 36 | } 37 | 38 | col := b.Finish() 39 | if col.Len() != size { 40 | t.Fatalf("bad size %d != %d", col.Len(), size) 41 | } 42 | 43 | if col.DType() != dtype { 44 | t.Fatalf("bad dtype %d != %d", col.DType(), dtype) 45 | } 46 | 47 | vals, err := col.Ints() 48 | if err != nil { 49 | t.Fatal(err) 50 | } 51 | t.Logf("vals: %v", vals) 52 | 53 | for i, val := range vals { 54 | if int64(i) != val { 55 | t.Fatalf("%d: %d != %d", i, val, i) 56 | } 57 | } 58 | } 59 | 60 | func TestSliceBuilderEmpty(t *testing.T) { 61 | name := "fCol" 62 | dtype := FloatType 63 | b := NewSliceColumnBuilder(name, dtype, 0) 64 | size := 0 65 | for i := 0.7; i < 3.1; i += 0.62 { 66 | if err := b.Append(i); err != nil { 67 | t.Fatal(err) 68 | } 69 | size++ 70 | 71 | } 72 | 73 | col := b.Finish() 74 | if col.Len() != size { 75 | t.Fatalf("wrong len - %d != %d", col.Len(), size) 76 | } 77 | } 78 | 79 | func TestLabelBuilder(t *testing.T) { 80 | name := "bCol" 81 | dtype := BoolType 82 | size := 10 83 | b := NewLabelColumnBuilder(name, dtype, size/3) 84 | val := true 85 | for i := 0; i < size; i++ { 86 | if err := b.Set(i, val); err != nil { 87 | t.Fatal(err) 88 | } 89 | } 90 | 91 | col := b.Finish() 92 | if col.Len() != size { 93 | t.Fatalf("bad size %d != %d", col.Len(), size) 94 | } 95 | 96 | err := b.Set(0, false) 97 | if err == nil { 98 | t.Fatal("changed value in label column") 99 | } 100 | } 101 | 102 | func TestBuilderDelete(t *testing.T) { 103 | size := 10 104 | b := NewSliceColumnBuilder("a", IntType, size) 105 | for i := 0; i < size; i++ { 106 | _ = b.Set(i, i) 107 | } 108 | 109 | deleted := map[int]bool{ 110 | 1: true, 111 | 3: true, 112 | 9: true, 113 | } 114 | 115 | for v := range deleted { 116 | if err := b.Delete(v); err != nil { 117 | t.Fatalf("can't delete %d - %s", v, err) 118 | } 119 | } 120 | 121 | col := b.Finish() 122 | if cs := col.Len(); cs != size-len(deleted) { 123 | t.Fatalf("wrong size %d != %d", col.Len(), size-len(deleted)) 124 | } 125 | 126 | for i := 0; i < col.Len(); i++ { 127 | v, err := col.IntAt(i) 128 | if err != nil { 129 | t.Fatal(err) 130 | } 131 | 132 | if deleted[int(v)] { 133 | t.Fatalf("found deleted value - %d", v) 134 | } 135 | 136 | } 137 | } 138 | 139 | func TestBuilderDeleteFirst(t *testing.T) { 140 | b := NewSliceColumnBuilder("a", IntType, 1) 141 | _ = b.Set(0, 1) 142 | 143 | if err := b.Delete(0); err != nil { 144 | if err != nil { 145 | t.Fatal(err) 146 | } 147 | } 148 | 149 | col := b.Finish() 150 | if col.Len() != 0 { 151 | t.Fatalf("bad size: %d != 0", col.Len()) 152 | } 153 | } 154 | 155 | func TestBuilderAppend(t *testing.T) { 156 | b := NewSliceColumnBuilder("a", IntType, 1) 157 | size := 5 158 | for i := 0; i < size; i++ { 159 | if err := b.Append(i); err != nil { 160 | t.Fatal(err) 161 | } 162 | } 163 | 164 | col := b.Finish() 165 | if col.Len() != size { 166 | t.Fatalf("bad size: %d != %d", col.Len(), size) 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package frames 22 | 23 | import ( 24 | "encoding/json" 25 | "os" 26 | "strings" 27 | 28 | "github.com/pkg/errors" 29 | "github.com/v3io/frames/pb" 30 | ) 31 | 32 | // Client interface 33 | type Client interface { 34 | // Read reads data from server 35 | Read(request *pb.ReadRequest) (FrameIterator, error) 36 | // Write writes data to server 37 | Write(request *WriteRequest) (FrameAppender, error) 38 | // Create creates a table 39 | Create(request *pb.CreateRequest) error 40 | // Delete deletes data or table 41 | Delete(request *pb.DeleteRequest) error 42 | // Exec executes a command on the backend 43 | Exec(request *pb.ExecRequest) (Frame, error) 44 | } 45 | 46 | // SessionFromEnv return a session from V3IO_SESSION environment variable (JSON encoded) 47 | func SessionFromEnv() (*pb.Session, error) { 48 | session := &pb.Session{} 49 | envKey := "V3IO_SESSION" 50 | 51 | data := os.Getenv(envKey) 52 | if data == "" { 53 | return session, nil 54 | } 55 | 56 | // Support var1=val1,var2=val2,... format 57 | data = strings.TrimSpace(data) 58 | if !strings.HasPrefix(data, "{") { 59 | parts := strings.Split(data, ",") 60 | newData := map[string]string{} 61 | for _, part := range parts { 62 | pair := strings.SplitN(part, "=", 2) 63 | if len(pair) != 2 { 64 | return nil, errors.Errorf("%s was not recognized as either a JSON dictionary or comma-separated value pairs", envKey) 65 | } 66 | newData[pair[0]] = pair[1] 67 | } 68 | bytes, err := json.Marshal(newData) 69 | data = string(bytes) 70 | if err != nil { 71 | return nil, err 72 | } 73 | } 74 | 75 | dec := json.NewDecoder(strings.NewReader(data)) 76 | if err := dec.Decode(session); err != nil { 77 | return nil, errors.Wrapf(err, "can't read JSON from %s environment", envKey) 78 | } 79 | 80 | if session.Url == "" { 81 | session.Url = os.Getenv("V3IO_API") 82 | } 83 | if session.Token == "" { 84 | session.Token = os.Getenv("V3IO_ACCESS_KEY") 85 | } 86 | 87 | return session, nil 88 | } 89 | -------------------------------------------------------------------------------- /clients/py/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | build 3 | dist 4 | v3io_frames.egg-info 5 | -------------------------------------------------------------------------------- /clients/py/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md LICENSE.txt 2 | include Pipfile Pipfile.lock 3 | recursive-include tests *.py *.csv 4 | -------------------------------------------------------------------------------- /clients/py/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | .PHONY: all 16 | all: 17 | $(error please pick a target) 18 | 19 | .PHONY: upload 20 | upload: 21 | python pypi_upload.py --force 22 | 23 | .PHONY: dist 24 | dist: sync-deps 25 | python -m build --sdist --wheel --outdir dist/ . 26 | 27 | .PHONY: set-version 28 | set-version: 29 | python set-version.py 30 | 31 | .PHONY: clean_pyc 32 | clean_pyc: 33 | find . -name '*.pyc' -exec rm {} \; 34 | 35 | .PHONY: flake8 36 | flake8: 37 | # --ignore=E121,E123,E126,E226,E24,E704 is the default. 38 | # Additionally, E501 ignores long lines. 39 | python -m flake8 \ 40 | --ignore=E121,E123,E126,E226,E24,E704,E501 \ 41 | --exclude 'frames_pb2*.py' \ 42 | v3io_frames tests 43 | 44 | .PHONY: test 45 | test: clean_pyc flake8 46 | python -m pytest -v \ 47 | --disable-warnings \ 48 | --benchmark-disable \ 49 | tests 50 | 51 | README.html: README.md 52 | kramdown -i GFM $< > $@ 53 | 54 | .PHONY: sync-deps 55 | sync-deps: 56 | pip install -r requirements.txt -r dev-requirements.txt 57 | 58 | .PHONY: bench 59 | bench: 60 | python -m pytest \ 61 | --disable-warnings \ 62 | --benchmark-json /tmp/framesd-py-bench.json \ 63 | tests/test_benchmark.py 64 | -------------------------------------------------------------------------------- /clients/py/README.md: -------------------------------------------------------------------------------- 1 | # v3io_frames - Streaming Data Client for v3io 2 | 3 | [![Build Status](https://travis-ci.org/v3io/frames.svg?branch=master)](https://travis-ci.org/v3io/frames) 4 | [![Documentation](https://readthedocs.org/projects/v3io_frames/badge/?version=latest)](https://v3io-frames.readthedocs.io/en/latest/) 5 | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 6 | 7 | Streaming data from client [nuclio](http://nuclio.io/) handler to a pandas DataFrame. 8 | 9 | You will need a `framesd` server running, see [here](https://github.com/v3io/frames). 10 | 11 | ```python 12 | import v3io_frames as v3f 13 | 14 | client = v3f.Client(address='localhost:8081') 15 | num_dfs = num_rows = 0 16 | size = 1000 17 | for df in client.read(backend='weather', table='table', max_in_message=size): 18 | print(df) 19 | num_dfs += 1 20 | num_rows += len(df) 21 | 22 | print('\nnum_dfs = {}, num_rows = {}'.format(num_dfs, num_rows)) 23 | ``` 24 | 25 | 26 | ## License 27 | 28 | Apache License Version 2.0, see [LICENSE.txt](LICENSE.txt) 29 | -------------------------------------------------------------------------------- /clients/py/conda.recipe/build.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | %PYTHON% setup.py install 3 | -------------------------------------------------------------------------------- /clients/py/conda.recipe/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | $PYTHON setup.py install 3 | -------------------------------------------------------------------------------- /clients/py/conda.recipe/meta.yaml: -------------------------------------------------------------------------------- 1 | package: 2 | name: v3io_frames 3 | version: "0.1.0" 4 | 5 | build: 6 | number: 1 7 | 8 | source: 9 | path: .. 10 | 11 | requirements: 12 | build: 13 | - pandas 14 | - setuptools 15 | run: 16 | - python 17 | 18 | test: 19 | requires: 20 | - pytest 21 | 22 | about: 23 | home: https://github.com/v3io/frames 24 | license: Apache 25 | -------------------------------------------------------------------------------- /clients/py/dev-requirements.txt: -------------------------------------------------------------------------------- 1 | build 2 | # https://stackoverflow.com/questions/73929564/entrypoints-object-has-no-attribute-get-digital-ocean 3 | importlib-metadata<5 4 | flake8 5 | ipython 6 | numpydoc 7 | pytest>=4.2,<6 8 | pyyaml 9 | sphinx 10 | twine~=1.12 11 | pytest-benchmark 12 | -------------------------------------------------------------------------------- /clients/py/docs/.gitignore: -------------------------------------------------------------------------------- 1 | _build 2 | -------------------------------------------------------------------------------- /clients/py/docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = . 8 | BUILDDIR = _build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /clients/py/docs/api.rst: -------------------------------------------------------------------------------- 1 | API 2 | === 3 | 4 | .. module:: v3io_frames 5 | 6 | .. autofunction:: Client 7 | -------------------------------------------------------------------------------- /clients/py/docs/index.rst: -------------------------------------------------------------------------------- 1 | .. v3io_frames documentation master file, created by 2 | sphinx-quickstart on Sun Nov 18 14:58:04 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to v3io_frames's documentation! 7 | ======================================= 8 | 9 | ``v3io_frames`` is a fast streaming of ``pandas.DataFrame`` to and from various 10 | nuclio_ databases. 11 | 12 | 13 | .. _nuclio: https://nuclio.io/ 14 | 15 | 16 | .. code-block:: python 17 | 18 | import v3io_frames as v3f 19 | 20 | client = v3f.Client(address='localhost:8080') 21 | num_dfs = num_rows = 0 22 | size = 1000 23 | for df in client.read(backend='weather', table='table', max_in_message=size): 24 | print(df) 25 | num_dfs += 1 26 | num_rows += len(df) 27 | 28 | print('\nnum_dfs = {}, num_rows = {}'.format(num_dfs, num_rows)) 29 | 30 | 31 | 32 | .. toctree:: 33 | :maxdepth: 2 34 | :caption: Contents: 35 | 36 | api 37 | 38 | Indices and tables 39 | ================== 40 | 41 | * :ref:`genindex` 42 | * :ref:`modindex` 43 | * :ref:`search` 44 | -------------------------------------------------------------------------------- /clients/py/docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /clients/py/docs/requirements.txt: -------------------------------------------------------------------------------- 1 | googleapis-common-protos>=1.5.3 2 | grpcio-tools>=1.16.0 3 | numpydoc==0.8.0 4 | pandas>=0.23.* 5 | requests>=2.19.1 6 | -------------------------------------------------------------------------------- /clients/py/environment-cudf.yml: -------------------------------------------------------------------------------- 1 | channels: 2 | - conda-forge 3 | - defaults 4 | - numba 5 | - nvidia 6 | - pytorch 7 | - rapidsai 8 | dependencies: 9 | - cudf=0.5 10 | - cuml=0.5 11 | - python=3.6 12 | - cudatoolkit=9.2 13 | -------------------------------------------------------------------------------- /clients/py/environment.yml: -------------------------------------------------------------------------------- 1 | channels: 2 | - defaults 3 | dependencies: 4 | - grpcio-tools=1.16.1 5 | - protobuf=3.6.1 6 | - requests=2.21.0 7 | - pandas>=0.23.* 8 | -------------------------------------------------------------------------------- /clients/py/pypi_upload.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2018 Iguazio 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Upload packages to PyPI""" 17 | 18 | from argparse import ArgumentParser 19 | from glob import glob 20 | from os import environ, path 21 | from shutil import rmtree 22 | from subprocess import run 23 | from sys import executable 24 | 25 | 26 | def should_upload(): 27 | repo = environ.get('TRAVIS_REPO_SLUG') 28 | tag = environ.get('TRAVIS_TAG') 29 | 30 | return repo == 'v3io/frames' and tag 31 | 32 | 33 | def git_sha(): 34 | return environ.get('TRAVIS_COMMIT', '')[:7] 35 | 36 | 37 | def set_version(): 38 | version = environ.get('TRAVIS_TAG') 39 | assert version, 'no tag' 40 | 41 | if version[0] == 'v': 42 | version = version[1:] 43 | 44 | if version.endswith('.py'): 45 | version = version[:-3] 46 | 47 | lines = [] 48 | init_py = 'v3io_frames/__init__.py' 49 | with open(init_py) as fp: 50 | for line in fp: 51 | # __version__ = '0.3.1' 52 | if '__version__' in line: 53 | line = "__version__ = '{}'\n".format(version) 54 | lines.append(line) 55 | 56 | with open(init_py, 'w') as out: 57 | out.write(''.join(lines)) 58 | 59 | 60 | if __name__ == '__main__': 61 | parser = ArgumentParser(description=__doc__) 62 | parser.add_argument( 63 | '--force', '-f', help='force upload', action='store_true') 64 | parser.add_argument( 65 | '--user', '-u', help='pypi user (or V3IO_PYPI_USER)', default='') 66 | parser.add_argument( 67 | '--password', '-p', help='pypi password (or V3IO_PYPI_PASSWORD)', 68 | default='') 69 | args = parser.parse_args() 70 | 71 | ok = args.force or should_upload() 72 | if not ok: 73 | raise SystemExit('error: wrong branch or repo (try with --force)') 74 | 75 | if path.exists('dist'): 76 | rmtree('dist') 77 | 78 | set_version() 79 | for dist in ('sdist', 'bdist_wheel'): 80 | out = run([executable, 'setup.py', dist]) 81 | if out.returncode != 0: 82 | raise SystemExit('error: cannot build {}'.format(dist)) 83 | 84 | user = args.user or environ.get('V3IO_PYPI_USER') 85 | passwd = args.password or environ.get('V3IO_PYPI_PASSWORD') 86 | 87 | if not (user and passwd): 88 | print('warning: missing login information - skipping upload') 89 | raise SystemExit() 90 | 91 | cmd = [ 92 | 'twine', 'upload', 93 | '--user', user, 94 | '--password', passwd, 95 | ] + glob('dist/v3io_frames-*') 96 | out = run(cmd) 97 | if out.returncode != 0: 98 | raise SystemExit('error: cannot upload to pypi') 99 | -------------------------------------------------------------------------------- /clients/py/requirements.txt: -------------------------------------------------------------------------------- 1 | googleapis-common-protos>=1.5.3 2 | grpcio-tools>=1.49 3 | protobuf~=4.0 4 | pandas>=0.23.4 5 | requests>=2.19.1 6 | -------------------------------------------------------------------------------- /clients/py/set-version.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | from os import environ 16 | 17 | 18 | def set_version(): 19 | version = environ.get("GITHUB_REF") 20 | assert version, "GITHUB_REF is not defined" 21 | 22 | version = version.replace("refs/tags/v", "") 23 | 24 | lines = [] 25 | init_py = "v3io_frames/__init__.py" 26 | with open(init_py) as fp: 27 | for line in fp: 28 | if "__version__" in line: 29 | line = f"__version__ = '{version}'\n" 30 | lines.append(line) 31 | 32 | with open(init_py, "w") as out: 33 | out.write("".join(lines)) 34 | 35 | 36 | set_version() 37 | -------------------------------------------------------------------------------- /clients/py/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | try: 16 | from setuptools import setup 17 | except ImportError: 18 | from distutils.core import setup 19 | 20 | 21 | def version(): 22 | with open('v3io_frames/__init__.py') as fp: 23 | for line in fp: 24 | if '__version__' in line: 25 | _, version = line.split('=') 26 | return version.replace("'", '').strip() 27 | 28 | def is_ignored(line): 29 | line = line.strip() 30 | return (not line) or (line[0] == "#") 31 | 32 | def load_deps(path): 33 | """Load dependencies from requirements file""" 34 | with open(path) as fp: 35 | deps = [] 36 | for line in fp: 37 | if is_ignored(line): 38 | continue 39 | line = line.strip() 40 | 41 | # e.g.: git+https://github.com/nuclio/nuclio-jupyter.git@some-branch#egg=nuclio-jupyter 42 | if "#egg=" in line: 43 | _, package = line.split("#egg=") 44 | deps.append(f"{package} @ {line}") 45 | continue 46 | 47 | # append package 48 | deps.append(line) 49 | return deps 50 | 51 | 52 | install_requires = load_deps('requirements.txt') 53 | tests_require = load_deps('dev-requirements.txt') 54 | 55 | with open('README.md') as fp: 56 | long_desc = fp.read() 57 | 58 | 59 | setup( 60 | name='v3io_frames', 61 | version=version(), 62 | description='Unified multi-module DataFrames client for the Iguazio Data Science Platform', 63 | long_description=long_desc, 64 | long_description_content_type='text/markdown', 65 | author='Miki Tebeka', 66 | author_email='miki@353solutions.com', 67 | license='MIT', 68 | url='https://github.com/v3io/frames', 69 | packages=['v3io_frames'], 70 | install_requires=install_requires, 71 | classifiers=[ 72 | 'Development Status :: 4 - Beta', 73 | 'Intended Audience :: Developers', 74 | 'License :: OSI Approved :: Apache Software License', 75 | 'Operating System :: POSIX :: Linux', 76 | 'Operating System :: Microsoft :: Windows', 77 | 'Operating System :: MacOS', 78 | 'Programming Language :: Python :: 2', 79 | 'Programming Language :: Python :: 2.7', 80 | 'Programming Language :: Python :: 3', 81 | 'Programming Language :: Python :: 3.6', 82 | 'Programming Language :: Python :: 3.7', 83 | 'Programming Language :: Python', 84 | 'Topic :: Software Development :: Libraries :: Python Modules', 85 | 'Topic :: Software Development :: Libraries', 86 | ], 87 | tests_require=tests_require, 88 | ) 89 | -------------------------------------------------------------------------------- /clients/py/tests/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import re 16 | from collections import namedtuple 17 | from os import environ, makedirs, path 18 | from shutil import rmtree, copyfile 19 | from socket import error as SocketError 20 | from socket import socket 21 | from subprocess import Popen, call, check_output 22 | from time import sleep, time 23 | from uuid import uuid4 24 | 25 | import pytest 26 | import v3io_frames as v3f 27 | import yaml 28 | 29 | has_session = v3f.SESSION_ENV_KEY in environ 30 | is_travis = 'TRAVIS' in environ 31 | test_id = uuid4().hex 32 | here = path.dirname(path.abspath(__file__)) 33 | 34 | csv_file = '{}/weather.csv'.format(here) 35 | git_root = path.abspath('{}/../../..'.format(here)) 36 | grpc_port = 8766 37 | http_port = 8765 38 | protocols = ['grpc', 'http'] 39 | root_dir = '/tmp/test-integration-root-{}'.format(test_id) 40 | server_timeout = 30 # seconds 41 | 42 | extra_backends = [ 43 | {'type': 'kv'}, 44 | {'type': 'stream'}, 45 | {'type': 'tsdb', 'workers': 16}, 46 | ] 47 | 48 | test_backends = ['csv'] 49 | if has_session: 50 | test_backends.extend(backend['type'] for backend in extra_backends) 51 | 52 | 53 | def has_working_go(): 54 | """Check we have go version >= 1.11""" 55 | try: 56 | out = check_output(['go', 'version']).decode('utf-8') 57 | match = re.search(r'(\d+)\.(\d+)', out) 58 | if not match: 59 | print('warning: cannot find version in {!r}'.format(out)) 60 | return False 61 | 62 | major, minor = int(match.group(1)), int(match.group(2)) 63 | return (major, minor) >= (1, 11) 64 | except FileNotFoundError: 65 | return False 66 | 67 | 68 | has_go = has_working_go() 69 | 70 | 71 | def wait_for_server(port, timeout): 72 | start = time() 73 | while time() - start <= timeout: 74 | with socket() as sock: 75 | try: 76 | sock.connect(('localhost', port)) 77 | return True 78 | except SocketError: 79 | sleep(0.1) 80 | 81 | return False 82 | 83 | 84 | Framesd = namedtuple('Framesd', 'grpc_addr http_addr config') 85 | 86 | 87 | def initialize_csv_root(): 88 | if path.exists(root_dir): 89 | rmtree(root_dir) 90 | makedirs(root_dir) 91 | 92 | dest = '{}/{}'.format(root_dir, path.basename(csv_file)) 93 | copyfile(csv_file, dest) 94 | 95 | 96 | @pytest.fixture(scope='module') 97 | def framesd(): 98 | initialize_csv_root() 99 | 100 | config = { 101 | 'log': { 102 | 'level': 'debug', 103 | }, 104 | 'backends': [ 105 | { 106 | 'type': 'csv', 107 | 'rootDir': root_dir, 108 | }, 109 | ] 110 | } 111 | 112 | if has_session: 113 | # We assume we have backends 114 | config['backends'].extend(extra_backends) 115 | 116 | cfg_file = '{}/config.yaml'.format(root_dir) 117 | with open(cfg_file, 'wt') as out: 118 | yaml.dump(config, out, default_flow_style=False) 119 | 120 | server_exe = '/tmp/test-framesd-{}'.format(test_id) 121 | cmd = [ 122 | 'go', 'build', 123 | '-o', server_exe, 124 | '{}/cmd/framesd/framesd.go'.format(git_root), 125 | ] 126 | assert call(cmd) == 0, 'cannot build server' 127 | 128 | log_file = open('/tmp/framesd-integration.log', 'at') 129 | cmd = [ 130 | server_exe, 131 | '-httpAddr', ':{}'.format(http_port), 132 | '-grpcAddr', ':{}'.format(grpc_port), 133 | '-config', cfg_file, 134 | ] 135 | pipe = Popen(cmd, stdout=log_file, stderr=log_file) 136 | assert wait_for_server(http_port, server_timeout), 'server did not start' 137 | try: 138 | yield Framesd( 139 | grpc_addr='grpc://localhost:{}'.format(grpc_port), 140 | http_addr='http://localhost:{}'.format(http_port), 141 | config=config, 142 | ) 143 | finally: 144 | pipe.kill() 145 | log_file.close() 146 | 147 | 148 | @pytest.fixture(scope='module') 149 | def session(): 150 | """Return session parameters fit for v3f.Client arguments""" 151 | obj = v3f.session_from_env() 152 | session = {desc.name: value for desc, value in obj.ListFields()} 153 | if 'url' in session: 154 | session['data_url'] = session.pop('url') 155 | return session 156 | -------------------------------------------------------------------------------- /clients/py/tests/pip_docker.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Code used by test_pip_docker 16 | 17 | from argparse import ArgumentParser 18 | 19 | import v3io_frames as v3f 20 | 21 | parser = ArgumentParser() 22 | parser.add_argument('--grpc-port', default='8081') 23 | parser.add_argument('--http-port', default='8080') 24 | args = parser.parse_args() 25 | 26 | client = v3f.Client('localhost:{}'.format(args.grpc_port)) 27 | df = client.read('csv', table='weather.csv') 28 | assert len(df) > 0, 'empty df' 29 | -------------------------------------------------------------------------------- /clients/py/tests/test_benchmark.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import v3io_frames as v3f 4 | from conftest import has_go 5 | 6 | from test_integration import integ_params, csv_df 7 | 8 | wdf = csv_df(1982) 9 | 10 | 11 | def read_benchmark(client): 12 | for df in client.read('csv', 'weather.csv'): 13 | assert len(df), 'empty df' 14 | 15 | 16 | def write_benchmark(client, df): 17 | client.write('csv', 'write-bench.csv', df) 18 | 19 | 20 | @pytest.mark.skipif(not has_go, reason='Go SDK not found') 21 | @pytest.mark.parametrize('protocol,backend', integ_params) 22 | def test_read(benchmark, framesd, protocol, backend): 23 | addr = getattr(framesd, '{}_addr'.format(protocol)) 24 | client = v3f.Client(addr) 25 | benchmark(read_benchmark, client) 26 | 27 | 28 | @pytest.mark.skipif(not has_go, reason='Go SDK not found') 29 | @pytest.mark.parametrize('protocol,backend', integ_params) 30 | def test_write(benchmark, framesd, protocol, backend): 31 | addr = getattr(framesd, '{}_addr'.format(protocol)) 32 | client = v3f.Client(addr) 33 | benchmark(write_benchmark, client, wdf) 34 | -------------------------------------------------------------------------------- /clients/py/tests/test_client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | import v3io_frames as v3f 17 | 18 | 19 | test_client_params = [ 20 | ('grpc', v3f.gRPCClient), 21 | ('http', v3f.HTTPClient), 22 | ] 23 | 24 | 25 | @pytest.mark.parametrize('proto,cls', test_client_params) 26 | def test_client(proto, cls): 27 | address = '{}://localhost:8080'.format(proto) 28 | session_params = { 29 | 'data_url': 'http://iguazio.com', 30 | 'container': 'large one', 31 | 'user': 'bugs', 32 | 'password': 'duck season', 33 | } 34 | 35 | client = v3f.Client(address, should_check_version=False, **session_params) 36 | assert client.__class__ is cls, 'wrong class' 37 | for key, value in session_params.items(): 38 | key = 'url' if key == 'data_url' else key 39 | assert getattr(client.session, key) == value, \ 40 | 'bad session value for {}'.format(key) 41 | 42 | 43 | @pytest.mark.parametrize('proto,cls', test_client_params) 44 | def test_client_wrong_params(proto, cls): 45 | address = '{}://localhost:8080'.format(proto) 46 | session_params = { 47 | 'data_url': 'http://iguazio.com', 48 | 'container': 'large one', 49 | 'user': 'bugs', 50 | 'password': 'duck season', 51 | 'token': 'a quarter', 52 | } 53 | 54 | try: 55 | v3f.Client(address, should_check_version=False, **session_params) 56 | raise ValueError('expected fail but finished successfully') 57 | except ValueError: 58 | return 59 | 60 | 61 | def test_partition_keys(): 62 | class Proto(v3f.client.ClientBase): 63 | def _write(self, request, dfs, labels, index_cols): 64 | self.request = request 65 | 66 | c = Proto('localhost:8081', None) 67 | keys = ['pk1', 'pk2'] 68 | c.write('backend', 'table', None, partition_keys=keys) 69 | assert c.request.partition_keys == keys, 'bad partition keys' 70 | -------------------------------------------------------------------------------- /clients/py/tests/test_concurrent.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from concurrent.futures import ThreadPoolExecutor 16 | 17 | import pandas as pd 18 | import pytest 19 | 20 | import v3io_frames as v3f 21 | from conftest import protocols, csv_file, has_go 22 | from time import sleep, monotonic 23 | from random import random 24 | 25 | 26 | def csv_info(): 27 | with open(csv_file) as fp: 28 | columns = fp.readline().strip().split(',') 29 | size = sum(1 for _ in fp) 30 | return columns, size 31 | 32 | 33 | columns, size = csv_info() 34 | 35 | 36 | def reader(id, n, c): 37 | for i in range(n): 38 | df = pd.concat(c.read('weather', 'weather.csv')) 39 | assert set(df.columns) == set(columns), 'columns mismatch' 40 | assert len(df) == size, 'bad size' 41 | sleep(random() / 10) 42 | 43 | 44 | @pytest.mark.skipif(not has_go, reason='Go SDK not found') 45 | @pytest.mark.parametrize('protocol', protocols) 46 | def test_concurrent(framesd, protocol): 47 | addr = getattr(framesd, '{}_addr'.format(protocol)) 48 | c = v3f.Client(addr) 49 | start = monotonic() 50 | with ThreadPoolExecutor() as pool: 51 | for i in range(7): 52 | pool.submit(reader, i, 5, c) 53 | duration = monotonic() - start 54 | print('duration: {:.3f}sec'.format(duration)) 55 | -------------------------------------------------------------------------------- /clients/py/tests/test_cudf.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from time import sleep, time 16 | 17 | import pandas as pd 18 | import pytest 19 | import v3io_frames as v3f 20 | from conftest import has_go 21 | from conftest import test_backends 22 | 23 | try: 24 | import cudf 25 | 26 | has_cudf = True 27 | except ImportError: 28 | has_cudf = False 29 | 30 | 31 | @pytest.mark.skipif(not has_cudf, reason='cudf not found') 32 | @pytest.mark.skipif(not has_go, reason='Go SDK not found') 33 | def test_cudf(framesd, session): 34 | df = cudf.DataFrame({ 35 | 'a': [1, 2, 3], 36 | 'b': [1.1, 2.2, 3.3], 37 | }) 38 | 39 | c = v3f.Client(framesd.grpc_addr, frame_factory=cudf.DataFrame) 40 | backend = 'csv' 41 | table = 'cudf-{}'.format(int(time())) 42 | print('table = {}'.format(table)) 43 | 44 | c.write(backend, table, [df]) 45 | sleep(1) # Let db flush 46 | rdf = c.read(backend, table=table) 47 | assert isinstance(rdf, cudf.DataFrame), 'not a cudf.DataFrame' 48 | assert len(rdf) == len(df), 'wrong frame size' 49 | assert set(rdf.columns) == set(df.columns), 'columns mismatch' 50 | 51 | 52 | @pytest.mark.skipif(not has_cudf, reason='cudf not found') 53 | def test_concat_categorical(): 54 | df1 = cudf.DataFrame({'a': range(10, 13), 'b': range(50, 53)}) 55 | df1['c'] = pd.Series(['a'] * 3, dtype='category') 56 | 57 | df2 = cudf.DataFrame({'a': range(20, 23), 'b': range(60, 63)}) 58 | df2['c'] = pd.Series(['b'] * 3, dtype='category') 59 | 60 | for backend in test_backends: 61 | df = v3f.pdutils.concat_dfs([df1, df2], backend, cudf.DataFrame, cudf.concat, False) 62 | assert len(df) == len(df1) + len(df2), 'bad concat size' 63 | dtype = df['c'].dtype 64 | assert v3f.pdutils.is_categorical_dtype(dtype), 'result not categorical' 65 | 66 | 67 | @pytest.mark.skipif(not has_cudf, reason='cudf not found') 68 | def test_concat_categorical_with_multi_index(): 69 | df1 = cudf.DataFrame({'a': range(10, 13), 'b': range(50, 53)}) 70 | df1['c'] = pd.Series(['a'] * 3, dtype='category') 71 | 72 | df2 = cudf.DataFrame({'a': range(20, 23), 'b': range(60, 63)}) 73 | df2['c'] = pd.Series(['b'] * 3, dtype='category') 74 | 75 | for backend in test_backends: 76 | df = v3f.pdutils.concat_dfs([df1, df2], backend, cudf.DataFrame, cudf.concat, True) 77 | assert len(df) == len(df1) + len(df2), 'bad concat size' 78 | dtype = df['c'].dtype 79 | assert v3f.pdutils.is_categorical_dtype(dtype), 'result not categorical' 80 | -------------------------------------------------------------------------------- /clients/py/tests/test_http.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import re 16 | import struct 17 | from datetime import datetime 18 | from io import BytesIO 19 | from os.path import abspath, dirname 20 | 21 | import pandas as pd 22 | import pytz 23 | 24 | import v3io_frames as v3f 25 | from v3io_frames import frames_pb2 as fpb 26 | from v3io_frames import http 27 | from v3io_frames.pbutils import df2msg 28 | 29 | here = dirname(abspath(__file__)) 30 | 31 | 32 | class RequestSessionMock(object): 33 | 34 | def __init__(self, data=None): 35 | self.requests = [] 36 | self.data = [] if data is None else data 37 | self.write_request = None 38 | self.write_frames = [] 39 | 40 | def post(self, *args, **kw): 41 | self.requests.append((args, kw)) 42 | if args[0].endswith('/read'): 43 | return self._read(*args, **kw) 44 | elif args[0].endswith('/write'): 45 | return self._write(*args, **kw) 46 | elif args[0].endswith('/version'): 47 | return self._version() 48 | 49 | def _read(self, *args, **kw): 50 | io = BytesIO() 51 | for df in self.data: 52 | data = df2msg(df, None).SerializeToString() 53 | io.write(struct.pack(http.header_fmt, len(data))) 54 | io.write(data) 55 | 56 | io.seek(0, 0) 57 | 58 | class Response: 59 | raw = io 60 | ok = True 61 | 62 | return Response 63 | 64 | def _write(self, *args, **kw): 65 | it = iter(kw.get('data', [])) 66 | data = next(it) 67 | self.write_request = fpb.InitialWriteRequest.FromString(data) 68 | for chunk in it: 69 | msg = fpb.Frame.FromString(chunk) 70 | self.write_frames.append(msg) 71 | 72 | class Response: 73 | ok = True 74 | 75 | @staticmethod 76 | def json(): 77 | # TODO: Real values 78 | return { 79 | 'num_frames': -1, 80 | 'num_rows': -1, 81 | } 82 | 83 | return Response 84 | 85 | def _version(self): 86 | class Response: 87 | ok = True 88 | 89 | @staticmethod 90 | def json(): 91 | return { 92 | 'version': 'test_version', 93 | } 94 | 95 | return Response 96 | 97 | def close(self): 98 | pass 99 | 100 | 101 | def test_read(): 102 | address = 'https://nuclio.io' 103 | query = 'SELECT 1' 104 | data = [ 105 | pd.DataFrame({ 106 | 'x': [1, 2, 3], 107 | 'y': [4., 5., 6.], 108 | }), 109 | pd.DataFrame({ 110 | 'x': [10, 20, 30], 111 | 'y': [40., 50., 60.], 112 | }), 113 | ] 114 | 115 | client = new_test_client(address=address) 116 | 117 | # mock the session 118 | client._session = RequestSessionMock(data) 119 | dfs = client.read( 120 | backend='backend', table='table', query=query, iterator=True) 121 | 122 | assert len(client._session.requests) == 1 123 | 124 | args, kw = client._session.requests[0] 125 | assert args == (address + '/read',) 126 | 127 | df = pd.concat(dfs) 128 | assert len(df) == 6 129 | assert list(df.columns) == ['x', 'y'] 130 | 131 | df = client.read(backend='backend', query=query, iterator=False) 132 | assert isinstance(df, pd.DataFrame), 'iterator=False return' 133 | 134 | 135 | def col_name(msg): 136 | val = msg.get('slice') or msg.get('label') 137 | return val['name'] 138 | 139 | 140 | def test_format_go_time(): 141 | tz = pytz.timezone('Asia/Jerusalem') 142 | now = datetime.now() 143 | dt = now.astimezone(tz) 144 | ts = v3f.http.format_go_time(dt) 145 | 146 | # 2018-10-04T16:54:05.434079562+03:00 147 | match = re.match( 148 | r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+\d{2}:\d{2}$', ts) 149 | assert match, 'bad timestamp format' 150 | 151 | # ...+03:00 -> (3, 0) 152 | hours, minutes = map(int, ts[ts.find('+')+1:].split(':')) 153 | offset = hours * 60 * 60 + minutes * 60 154 | assert offset == tz.utcoffset(now).total_seconds(), 'bad offset' 155 | 156 | 157 | def new_test_client(address='', session=None): 158 | return v3f.HTTPClient( 159 | address=address or 'http://example.com', 160 | session=session, 161 | should_check_version=False, 162 | ) 163 | -------------------------------------------------------------------------------- /clients/py/tests/test_pbutils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import numpy as np 16 | import pandas as pd 17 | 18 | import v3io_frames.frames_pb2 as fpb 19 | from conftest import here 20 | from v3io_frames import pbutils 21 | 22 | 23 | def test_encode_df(): 24 | labels = { 25 | 'int': 7, 26 | 'str': 'wassup?', 27 | } 28 | 29 | df = pd.read_csv('{}/weather.csv'.format(here)) 30 | df['STATION_CAT'] = df['STATION'].astype('category') 31 | df['WDF2_F'] = df['WDF2'].astype(float) 32 | msg = pbutils.df2msg(df, labels) 33 | 34 | names = [col.name for col in msg.columns] 35 | assert set(names) == set(df.columns), 'columns mismatch' 36 | assert not msg.indices, 'has index' 37 | assert pbutils.pb2py(msg.labels) == labels, 'labels mismatch' 38 | 39 | # Now with index 40 | index_name = 'DATE' 41 | df = df.set_index(index_name) 42 | msg = pbutils.df2msg(df, None) 43 | 44 | names = [col.name for col in msg.columns] 45 | assert set(names) == set(df.columns), 'columns mismatch' 46 | assert msg.indices, 'no index' 47 | assert msg.indices[0].name == index_name, 'bad index name' 48 | 49 | 50 | def test_multi_index(): 51 | tuples = [ 52 | ('bar', 'one'), 53 | ('bar', 'two'), 54 | ('baz', 'one'), 55 | ('baz', 'two'), 56 | ('foo', 'one'), 57 | ('foo', 'two'), 58 | ('qux', 'one'), 59 | ('qux', 'two')] 60 | index = pd.MultiIndex.from_tuples(tuples, names=['first', 'second']) 61 | df = pd.DataFrame(index=index) 62 | df['x'] = range(len(df)) 63 | 64 | data = pbutils.df2msg(df).SerializeToString() 65 | msg = fpb.Frame.FromString(data) 66 | 67 | for col in msg.indices: 68 | values = col.strings 69 | assert len(values) == len(df), 'bad index length' 70 | 71 | 72 | def test_categorical(): 73 | s = pd.Series(['a', 'b', 'c'] * 7, name='cat').astype('category') 74 | col = pbutils.series2col(s, s.name) 75 | assert col.name == s.name, 'bad name' 76 | assert list(col.strings) == list(s), 'bad data' 77 | 78 | 79 | def test_index_cols(): 80 | cols = list('abcdef') 81 | size = 10 82 | df = pd.DataFrame({ 83 | col: np.random.rand(size) for col in cols 84 | }) 85 | 86 | index_cols = np.random.choice(cols, size=2) 87 | cols = set(col for col in cols if col not in index_cols) 88 | msg = pbutils.df2msg(df, index_cols=index_cols) 89 | assert set(col.name for col in msg.columns) == cols, 'bad columns' 90 | assert set(col.name for col in msg.indices) == set(index_cols), \ 91 | 'bad indices' 92 | 93 | 94 | def test_label_col(): 95 | col = fpb.Column( 96 | name='lcol', 97 | kind=fpb.Column.LABEL, 98 | dtype=fpb.STRING, 99 | size=10, 100 | strings=['srv1'], 101 | ) 102 | 103 | s = pbutils.col2series(col, None) 104 | assert len(s) == col.size, 'bad size' 105 | assert pbutils.is_categorical_dtype(s.dtype), 'not categorical' 106 | assert set(s.cat.categories) == {col.strings[0]}, 'bad values' 107 | -------------------------------------------------------------------------------- /clients/py/tests/test_pdutils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import numpy as np 16 | import pandas as pd 17 | from conftest import test_backends 18 | from v3io_frames import pdutils, pbutils 19 | 20 | 21 | def gen_dfs(): 22 | size = 17 23 | columns = ['x', 'y', 'z'] 24 | for year in range(2010, 2017): 25 | index = pd.date_range(str(year), periods=17, name='Date') 26 | values = np.random.rand(size, len(columns)) 27 | yield pd.DataFrame(values, columns=columns, index=index) 28 | 29 | 30 | def test_concat_dfs(): 31 | dfs = list(gen_dfs()) 32 | for backend in test_backends: 33 | df = pdutils.concat_dfs(dfs, backend) 34 | 35 | assert len(df) == sum(len(d) for d in dfs), 'bad number of rows' 36 | assert df.index.name == dfs[0].index.name, 'bad index name' 37 | assert set(df.columns) == set(dfs[0].columns), 'bad columns' 38 | 39 | 40 | def test_concat_dfs_with_multi_index(): 41 | dfs = list(gen_dfs()) 42 | for backend in test_backends: 43 | df = pdutils.concat_dfs(dfs, backend, multi_index=True) 44 | 45 | assert len(df) == sum(len(d) for d in dfs), 'bad number of rows' 46 | assert df.index.name == dfs[0].index.name, 'bad index name' 47 | assert set(df.columns) == set(dfs[0].columns), 'bad columns' 48 | 49 | 50 | def test_empty_result(): 51 | for backend in test_backends: 52 | df = pdutils.concat_dfs([], backend) 53 | assert df.empty, 'non-empty df' 54 | 55 | 56 | def test_empty_result_with_multi_index(): 57 | for backend in test_backends: 58 | df = pdutils.concat_dfs([], backend, multi_index=True) 59 | assert df.empty, 'non-empty df' 60 | 61 | 62 | def gen_cat(value, size): 63 | series = pd.Series([value]) 64 | series = series.astype('category') 65 | return series.reindex(pd.RangeIndex(size), method='pad') 66 | 67 | 68 | def test_concat_dfs_categorical(): 69 | size = 10 70 | df1 = pd.DataFrame({ 71 | 'c': gen_cat('val1', size), 72 | 'v': range(size), 73 | }) 74 | 75 | df2 = pd.DataFrame({ 76 | 'c': gen_cat('val2', size), 77 | 'v': range(size, 2 * size), 78 | }) 79 | 80 | for backend in test_backends: 81 | df = pdutils.concat_dfs([df1, df2], backend) 82 | assert len(df) == len(df1) + len(df2), 'bad length' 83 | assert set(df.columns) == set(df1.columns), 'bad columns' 84 | assert pbutils.is_categorical_dtype(df['c'].dtype), 'not categorical' 85 | assert set(df['c']) == set(df1['c']) | set(df2['c']), 'bad values' 86 | 87 | 88 | def test_concat_dfs_categorical_with_multi_index(): 89 | size = 10 90 | df1 = pd.DataFrame({ 91 | 'c': gen_cat('val1', size), 92 | 'v': range(size), 93 | }) 94 | 95 | df2 = pd.DataFrame({ 96 | 'c': gen_cat('val2', size), 97 | 'v': range(size, 2 * size), 98 | }) 99 | 100 | for backend in test_backends: 101 | df = pdutils.concat_dfs([df1, df2], backend, multi_index=True) 102 | assert len(df) == len(df1) + len(df2), 'bad length' 103 | assert set(df.columns) == set(df1.columns), 'bad columns' 104 | assert pbutils.is_categorical_dtype(df['c'].dtype), 'not categorical' 105 | assert set(df['c']) == set(df1['c']) | set(df2['c']), 'bad values' 106 | -------------------------------------------------------------------------------- /clients/py/tests/test_pip_docker.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import json 16 | from contextlib import contextmanager 17 | from shutil import copy 18 | from subprocess import PIPE, run 19 | from sys import executable 20 | from tempfile import mkdtemp 21 | 22 | import pytest 23 | 24 | from conftest import here, is_travis 25 | 26 | config = ''' 27 | log: 28 | level: debug 29 | 30 | backends: 31 | - type: "csv" 32 | rootDir: "/csv-root" 33 | ''' 34 | 35 | docker_image = 'quay.io/v3io/frames:unstable' 36 | 37 | 38 | # docker run \ 39 | # -v /path/to/config.yaml:/etc/framesd.yaml \ 40 | # quay.io/v3io/frames 41 | 42 | 43 | def docker_ports(cid): 44 | out = run(['docker', 'inspect', cid], stdout=PIPE, check=True) 45 | obj = json.loads(out.stdout.decode('utf-8'))[0] 46 | grpc_port = obj['NetworkSettings']['Ports']['8081/tcp'][0]['HostPort'] 47 | http_port = obj['NetworkSettings']['Ports']['8080/tcp'][0]['HostPort'] 48 | 49 | return grpc_port, http_port 50 | 51 | 52 | @contextmanager 53 | def docker(tmp, cfg_file): 54 | cmd = [ 55 | 'docker', 'run', 56 | '-d', 57 | '-v', '{}:/csv-root'.format(tmp), 58 | '-v', '{}:/etc/framesd.yaml'.format(cfg_file), 59 | '-p', '8080', 60 | '-p', '8081', 61 | docker_image, 62 | ] 63 | proc = run(cmd, stdout=PIPE, check=True) 64 | cid = proc.stdout.decode('utf-8').strip() 65 | grpc_port, http_port = docker_ports(cid) 66 | try: 67 | yield grpc_port, http_port 68 | finally: 69 | run(['docker', 'rm', '-f', cid]) 70 | 71 | 72 | @pytest.mark.skipif(not is_travis, reason='integration test') 73 | def test_pip_docker(): 74 | tmp = mkdtemp() 75 | run(['virtualenv', '-p', executable, tmp], check=True) 76 | python = '{}/bin/python'.format(tmp) 77 | # Run in different directoy so local v3io_frames won't be used 78 | run([python, '-m', 'pip', 'install', 'v3io_frames'], check=True, cwd=tmp) 79 | run(['docker', 'pull', docker_image], check=True) 80 | 81 | cfg_file = '{}/framesd.yaml'.format(tmp) 82 | with open(cfg_file, 'w') as out: 83 | out.write(config) 84 | 85 | copy('{}/weather.csv'.format(here), tmp) 86 | with docker(tmp, cfg_file) as (grpc_port, http_port): 87 | cmd = [ 88 | python, '{}/pip_docker.py'.format(here), 89 | '--grpc-port', grpc_port, 90 | '--http-port', http_port, 91 | ] 92 | # Run in different directoy so local v3io_frames won't be used 93 | run(cmd, check=True, cwd=tmp) 94 | -------------------------------------------------------------------------------- /clients/py/tests/test_v3io_frames.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import json 16 | from contextlib import contextmanager 17 | 18 | from os import environ 19 | 20 | import v3io_frames as v3f 21 | 22 | 23 | @contextmanager 24 | def setenv(key, value): 25 | old = environ.get(key) 26 | environ[key] = value 27 | try: 28 | yield 29 | finally: 30 | if old is not None: 31 | environ[key] = old 32 | 33 | 34 | def test_client(): 35 | c = v3f.Client('localhost:8081', should_check_version=False) 36 | assert isinstance(c, v3f.gRPCClient), 'default is not grpc' 37 | c = v3f.Client('grpc://localhost:8081', should_check_version=False) 38 | assert isinstance(c, v3f.gRPCClient), 'not gRPC' 39 | c = v3f.Client('http://localhost:8081', should_check_version=False) 40 | assert isinstance(c, v3f.HTTPClient), 'not HTTP' 41 | 42 | 43 | def test_client_env(): 44 | url = 'localhost:8080' 45 | data = json.dumps({'url': url}) 46 | with setenv(v3f.SESSION_ENV_KEY, data): 47 | with setenv("V3IO_API", ""): 48 | c = v3f.Client('localhost:8081', should_check_version=False) 49 | 50 | assert c.session.url == url, 'missing URL from env' 51 | 52 | 53 | def test_session_from_env(): 54 | obj = {field.name: field.name for field in v3f.Session.DESCRIPTOR.fields} 55 | data = json.dumps(obj) 56 | with setenv(v3f.SESSION_ENV_KEY, data): 57 | with setenv("V3IO_API", ""): 58 | with setenv("V3IO_ACCESS_KEY", ""): 59 | s = v3f.session_from_env() 60 | 61 | env_obj = {field.name: value for field, value in s.ListFields()} 62 | assert env_obj == obj, 'bad session from environment' 63 | -------------------------------------------------------------------------------- /clients/py/v3io_frames/dtypes.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from datetime import datetime 16 | 17 | import numpy as np 18 | 19 | from . import frames_pb2 as fpb 20 | 21 | 22 | class DType: 23 | @classmethod 24 | def match(cls, value): 25 | return isinstance(value, cls.py_types) 26 | 27 | 28 | class BoolDType(DType): 29 | dtype = fpb.BOOLEAN 30 | col_key = 'bools' 31 | val_key = 'bval' 32 | py_types = (bool, np.bool_) 33 | 34 | @classmethod 35 | def match(cls, value): 36 | # We can't check isinstance since bool is instance of int 37 | return type(value) in cls.py_types 38 | 39 | 40 | class FloatDType(DType): 41 | dtype = fpb.FLOAT 42 | col_key = 'floats' 43 | val_key = 'fval' 44 | py_types = (np.inexact, float) 45 | 46 | 47 | class IntDType(DType): 48 | dtype = fpb.INTEGER 49 | col_key = 'ints' 50 | val_key = 'ival' 51 | py_types = (np.integer, int) 52 | 53 | 54 | class StringDType(DType): 55 | dtype = fpb.STRING 56 | col_key = 'strings' 57 | val_key = 'sval' 58 | py_types = str 59 | 60 | 61 | class TimeDType(DType): 62 | dtype = fpb.TIME 63 | col_key = 'times' 64 | val_key = 'tval' 65 | py_types = datetime 66 | 67 | 68 | # BoolDType must be first 69 | dtypes = [BoolDType] + \ 70 | [dtype for dtype in DType.__subclasses__() if dtype is not BoolDType] 71 | 72 | 73 | def dtype_of(val): 74 | for dtype in dtypes: 75 | if dtype.match(val): 76 | return dtype 77 | 78 | raise TypeError('unknown type - {!r}'.format(val)) 79 | -------------------------------------------------------------------------------- /clients/py/v3io_frames/errors.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [ 16 | 'Error', 'BadRequest', 'MessageError', 'ReadError', 'CreateError', 17 | 'DeleteError', 'ExecuteError', 'WriteError', 'HistoryError', 'VersionError' 18 | ] 19 | 20 | 21 | class Error(Exception): 22 | """v3io_frames Exception""" 23 | 24 | 25 | class BadRequest(Exception): 26 | """An error in query""" 27 | 28 | 29 | class MessageError(Error): 30 | """An error in message""" 31 | 32 | 33 | class ReadError(Error): 34 | """An error in read""" 35 | 36 | 37 | class WriteError(Error): 38 | """An error in write""" 39 | 40 | 41 | class CreateError(Error): 42 | """An error in table creation""" 43 | 44 | 45 | class DeleteError(Error): 46 | """An error in table deletion""" 47 | 48 | 49 | class ExecuteError(Error): 50 | """An error in executing command""" 51 | 52 | 53 | class HistoryError(Error): 54 | """An error in querying history logs""" 55 | 56 | 57 | class VersionError(Error): 58 | """An error in getting server version""" 59 | -------------------------------------------------------------------------------- /clients/py/v3io_frames/pdutils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import warnings 16 | 17 | import pandas as pd 18 | 19 | from .pbutils import is_categorical_dtype 20 | 21 | 22 | def concat_dfs(dfs, backend, frame_factory=pd.DataFrame, concat=pd.concat, multi_index=False): 23 | """Concat sequence of DataFrames, can handle MultiIndex frames.""" 24 | dfs = list(dfs) 25 | if not dfs: 26 | return frame_factory() 27 | 28 | # Make sure concat keep categorical columns 29 | # See https://stackoverflow.com/a/44086708/7650 30 | align_categories(dfs) 31 | 32 | had_index = True 33 | if multi_index and backend == 'tsdb': 34 | # with TSDB backend each dataframe might have different set of indices 35 | # therefore we should build a distinct set of indices in this case 36 | unique_names_set = set() 37 | for df in dfs: 38 | if hasattr(df.index, 'names'): 39 | indices = list(df.index.names) 40 | unique_names_set.update(indices) 41 | else: 42 | unique_names_set.update([df.index.name]) 43 | 44 | time_column = 'time' 45 | names = [] 46 | if time_column in unique_names_set: 47 | # move 'time' column to the first position 48 | unique_names_set.discard(time_column) 49 | names.append(time_column) 50 | 51 | names.extend(sorted(unique_names_set)) 52 | had_index = had_index and 'index' in df.columns 53 | else: 54 | if hasattr(dfs[0].index, 'names'): 55 | names = list(dfs[0].index.names) 56 | else: 57 | names = [dfs[0].index.name] 58 | 59 | had_index = 'index' in dfs[0].columns 60 | 61 | wdf = concat( 62 | [df.reset_index() for df in dfs], 63 | ignore_index=True, 64 | sort=False, 65 | ) 66 | 67 | if len(names) > 1: 68 | # We need a name for set_index, if we don't have one, take the name 69 | # pandas assigned to the column 70 | full_names = [name or wdf.columns[i] for i, name in enumerate(names)] 71 | wdf = wdf.set_index(full_names) 72 | wdf.index.names = names 73 | elif names[0]: 74 | wdf = wdf.set_index(names[0]) 75 | elif names[0] is None: 76 | if not had_index and 'index' in wdf.columns: 77 | del wdf['index'] # Pandas will add 'index' column 78 | 79 | with warnings.catch_warnings(): 80 | warnings.simplefilter('ignore') 81 | wdf.labels = getattr(dfs[0], 'labels', {}) 82 | return wdf 83 | 84 | 85 | def align_categories(dfs): 86 | all_cats = set() 87 | for df in dfs: 88 | for col in df.columns: 89 | if is_categorical_dtype(df[col].dtype): 90 | all_cats.update(df[col].cat.categories) 91 | 92 | for df in dfs: 93 | for col in df.columns: 94 | if is_categorical_dtype(df[col].dtype): 95 | df[col] = df[col].cat.set_categories(all_cats) 96 | 97 | 98 | def should_reorder_columns(backend, query, columns): 99 | # Currently TSDB sorts the columns by itself, 100 | # unless no columns were provided (either via columns or query). 101 | return backend != 'tsdb' or (not columns and (not query or '*' in query)) 102 | -------------------------------------------------------------------------------- /cmd/framesd/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Docker container for framesd, to run 16 | # docker run -p8080:8080 -p8081:8081 -v /path/to/config.yaml:/etc/framesd.yaml 17 | # 18 | # You can also use -e V3IO_GRPC_PORT=9999 and -e V3IO_HTTP_PORT=9998 to set ports 19 | # (don't forget to update the -p accordingly) 20 | 21 | FROM --platform=linux/amd64 golang:1.24-alpine AS build 22 | 23 | WORKDIR /frames 24 | COPY . . 25 | ARG FRAMES_VERSION=unknown 26 | RUN GOOS=linux GOARCH=amd64 go build -ldflags="-X main.Version=${FRAMES_VERSION}" ./cmd/framesd 27 | RUN cp framesd /usr/local/bin 28 | RUN apk add bash 29 | 30 | VOLUME /etc/framesd 31 | ENV V3IO_GRPC_PORT=8081 32 | ENV V3IO_HTTP_PORT=8080 33 | 34 | CMD framesd -grpcAddr :${V3IO_GRPC_PORT} -httpAddr :${V3IO_HTTP_PORT} -config /etc/framesd.yaml 35 | -------------------------------------------------------------------------------- /cmd/framesd/framesd.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package main 22 | 23 | import ( 24 | "flag" 25 | "fmt" 26 | "log" 27 | "net/http" 28 | _ "net/http/pprof" 29 | "os" 30 | "path" 31 | "time" 32 | 33 | "github.com/ghodss/yaml" 34 | "github.com/v3io/frames" 35 | "github.com/v3io/frames/backends/utils" 36 | "github.com/v3io/frames/grpc" 37 | framesHttp "github.com/v3io/frames/http" 38 | ) 39 | 40 | var ( 41 | // Version is framesd version (populated by the build process) 42 | Version = "unknown" 43 | ) 44 | 45 | func main() { 46 | var config struct { 47 | file string 48 | httpAddr string 49 | grpcAddr string 50 | } 51 | 52 | flag.StringVar(&config.file, "config", "", "path to configuration file (YAML)") 53 | flag.StringVar(&config.httpAddr, "httpAddr", ":8080", "address to listen on HTTP") 54 | flag.StringVar(&config.grpcAddr, "grpcAddr", ":8081", "address to listen on gRPC") 55 | flag.Parse() 56 | 57 | log.SetFlags(0) // Show only messages 58 | 59 | fmt.Printf("%s version %s\n", path.Base(os.Args[0]), Version) 60 | 61 | if config.file == "" { 62 | log.Fatal("error: no config file given") 63 | } 64 | 65 | data, err := os.ReadFile(config.file) 66 | if err != nil { 67 | log.Fatalf("error: can't read config - %s", err) 68 | } 69 | 70 | cfg := &frames.Config{} 71 | if err := yaml.Unmarshal(data, &cfg); err != nil { 72 | log.Fatalf("error: can't unmarshal config - %s", err) 73 | } 74 | 75 | frames.DefaultLogLevel = cfg.Log.Level 76 | 77 | framesLogger, err := frames.NewLogger(cfg.Log.Level) 78 | if err != nil { 79 | log.Fatalf("error: can't create logger - %s", err) 80 | } 81 | 82 | var historyServer *utils.HistoryServer 83 | if !cfg.DisableHistory { 84 | historyServer, err = utils.NewHistoryServer(framesLogger, cfg) 85 | if err != nil { 86 | framesLogger.WarnWith("failed to initialize frames-history", "err", err) 87 | } 88 | } 89 | 90 | hsrv, err := framesHttp.NewServer(cfg, config.httpAddr, framesLogger, historyServer, Version) 91 | if err != nil { 92 | log.Fatalf("error: can't create HTTP server - %s", err) 93 | } 94 | 95 | if err := hsrv.Start(); err != nil { 96 | log.Fatalf("error: can't start HTTP server - %s", err) 97 | } 98 | 99 | gsrv, err := grpc.NewServer(cfg, config.grpcAddr, framesLogger, historyServer, Version) 100 | if err != nil { 101 | log.Fatalf("error: can't create gRPC server - %s", err) 102 | } 103 | 104 | if err := gsrv.Start(); err != nil { 105 | log.Fatalf("error: can't start gRPC server - %s", err) 106 | } 107 | 108 | fmt.Printf("server running on http=%s, grpc=%s\n", config.httpAddr, config.grpcAddr) 109 | 110 | // Start profiling endpoint 111 | if !cfg.DisableProfiling { 112 | go func() { 113 | hasLog := framesLogger != nil 114 | 115 | if hasLog { 116 | framesLogger.Info("creating profiling endpoint at :8082. To start profiling run one of the following commands:\n" + 117 | " -> profiling: curl -o cpu.pprof localhost:8082/debug/pprof/profile?seconds=30\n" + 118 | " -> trace: curl -o trace.pprof localhost:8082/debug/pprof/trace?seconds=30") 119 | } 120 | 121 | err := http.ListenAndServe(":8082", nil) 122 | if err != nil && hasLog { 123 | framesLogger.Warn("failed to create profiling endpoint, err: %v", err) 124 | } 125 | }() 126 | } 127 | 128 | for hsrv.State() == frames.RunningState && gsrv.State() == frames.RunningState { 129 | time.Sleep(time.Second) 130 | } 131 | 132 | if err := hsrv.Err(); err != nil { 133 | log.Fatalf("error: HTTP server error - %s", err) 134 | } 135 | 136 | if err := gsrv.Err(); err != nil { 137 | log.Fatalf("error: gRPC server error - %s", err) 138 | } 139 | 140 | fmt.Println("server down") 141 | } 142 | -------------------------------------------------------------------------------- /cmd/framulate/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Iguazio 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Docker container for framesd, to run 16 | # docker run -p8080:8080 -p8081:8081 -v /path/to/config.yaml:/etc/framesd.yaml 17 | # 18 | # You can also use -e V3IO_GRPC_PORT=9999 and -e V3IO_HTTP_PORT=9998 to set ports 19 | # (don't forget to update the -p accordingly) 20 | 21 | 22 | FROM golang:1.14-stretch as build 23 | 24 | WORKDIR /frames 25 | COPY . . 26 | ARG FRAMES_VERSION=unknown 27 | RUN go build -ldflags="-X main.Version=${FRAMES_VERSION}" -o framulate-bin ./cmd/framulate 28 | RUN cp framulate-bin /usr/local/bin/framulate 29 | 30 | FROM debian:jessie-slim 31 | COPY --from=build /usr/local/bin/framulate /usr/local/bin 32 | 33 | ENTRYPOINT ["framulate"] 34 | -------------------------------------------------------------------------------- /cmd/framulate/README.md: -------------------------------------------------------------------------------- 1 | # Framulate 2 | 3 | Runs a stress test on frames 4 | 5 | ## Usage example 6 | ``` 7 | kubectl delete job -n default-tenant framulate; 8 | kubectl delete cm -n default-tenant framulate-config; 9 | cat << EOF | kubectl apply -n default-tenant -f - 10 | kind: ConfigMap 11 | apiVersion: v1 12 | metadata: 13 | name: framulate-config 14 | data: 15 | framulate.yaml: | 16 | transport: 17 | # will use grpc by default. for http, use http://framesd:8080 18 | url: framesd:8081 19 | container_name: test10 20 | access_key: 21 | max_inflight_requests: 512 22 | cleanup: true 23 | scenario: 24 | kind: writeVerify 25 | writeVerify: 26 | verify: true 27 | num_tables: 32 28 | num_series_per_table: 10000 29 | max_parallel_tables_create: 32 30 | max_parallel_series_write: 512 31 | max_parallel_series_verify: 512 32 | write_dummy_series: true 33 | num_datapoints_per_series: 24 34 | --- 35 | apiVersion: batch/v1 36 | kind: Job 37 | metadata: 38 | name: framulate 39 | spec: 40 | template: 41 | spec: 42 | containers: 43 | - name: framulate 44 | image: iguazio/framulate:latest 45 | imagePullPolicy: Never 46 | args: 47 | - "-config-path" 48 | - "/etc/iguazio/framulate/framulate.yaml" 49 | volumeMounts: 50 | - name: config 51 | mountPath: /etc/iguazio/framulate 52 | restartPolicy: Never 53 | volumes: 54 | - name: config 55 | configMap: 56 | name: framulate-config 57 | backoffLimit: 0 58 | EOF 59 | ``` 60 | -------------------------------------------------------------------------------- /cmd/framulate/framulate.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "os" 7 | 8 | "github.com/nuclio/errors" 9 | nucliozap "github.com/nuclio/zap" 10 | "github.com/v3io/frames/framulate" 11 | ) 12 | 13 | func run(configContents string, configPath string) error { 14 | loggerInstance, err := nucliozap.NewNuclioZapCmd("framulate", nucliozap.DebugLevel, os.Stdout) 15 | if err != nil { 16 | return errors.Wrap(err, "Failed to create logger") 17 | } 18 | 19 | config, err := framulate.NewConfigFromContentsOrPath([]byte(configContents), configPath) 20 | if err != nil { 21 | return errors.Wrap(err, "Failed to create config") 22 | } 23 | 24 | framulateInstance, err := framulate.NewFramulate(context.TODO(), 25 | loggerInstance, 26 | config) 27 | if err != nil { 28 | return errors.Wrap(err, "Failed to create framulate") 29 | } 30 | 31 | return framulateInstance.Start() 32 | } 33 | 34 | func main() { 35 | configPath := "" 36 | configContents := "" 37 | 38 | flag.StringVar(&configPath, "config-path", "", "") 39 | flag.StringVar(&configContents, "config-contents", "", "") 40 | 41 | flag.Parse() 42 | 43 | if err := run(configContents, configPath); err != nil { 44 | errors.PrintErrorStack(os.Stderr, err, 10) 45 | os.Exit(1) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /column_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package frames 22 | 23 | import ( 24 | "testing" 25 | "time" 26 | 27 | "github.com/v3io/frames/pb" 28 | ) 29 | 30 | func TestLabelColTimeAt(t *testing.T) { 31 | ts := time.Now() 32 | col := colImpl{ 33 | msg: &pb.Column{ 34 | Times: []int64{ts.UnixNano()}, 35 | Dtype: pb.DType_TIME, 36 | Kind: pb.Column_LABEL, 37 | Size: 20, 38 | }, 39 | } 40 | 41 | ts1, err := col.TimeAt(3) 42 | if err != nil { 43 | t.Fatal(err) 44 | } 45 | 46 | if ts1.Round(time.Millisecond) != ts.Round(time.Millisecond) { 47 | t.Fatalf("bad time %v != %v", ts1, ts) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /colutils.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package frames 22 | 23 | import ( 24 | "fmt" 25 | "time" 26 | ) 27 | 28 | // ZeroTime is zero value for time 29 | var ZeroTime time.Time 30 | 31 | func checkInBounds(col Column, i int) error { 32 | if i >= 0 && i < col.Len() { 33 | return nil 34 | } 35 | 36 | return fmt.Errorf("index %d out of bounds [0:%d]", i, col.Len()) 37 | } 38 | 39 | func intAt(col Column, i int) (int64, error) { 40 | if err := checkInBounds(col, i); err != nil { 41 | return 0, err 42 | } 43 | 44 | if col.DType() != IntType { 45 | return 0, fmt.Errorf("not an int column") 46 | } 47 | 48 | typedCol, err := col.Ints() 49 | if err != nil { 50 | return 0, err 51 | } 52 | 53 | return typedCol[i], nil 54 | } 55 | 56 | func floatAt(col Column, i int) (float64, error) { 57 | if err := checkInBounds(col, i); err != nil { 58 | return 0.0, err 59 | } 60 | 61 | if col.DType() != FloatType { 62 | return 0, fmt.Errorf("not a float64 column") 63 | } 64 | 65 | typedCol, err := col.Floats() 66 | if err != nil { 67 | return 0.0, err 68 | } 69 | 70 | return typedCol[i], nil 71 | } 72 | 73 | func stringAt(col Column, i int) (string, error) { 74 | if err := checkInBounds(col, i); err != nil { 75 | return "", err 76 | } 77 | 78 | if col.DType() != StringType { 79 | return "", fmt.Errorf("not a string column") 80 | } 81 | 82 | typedCol := col.Strings() 83 | return typedCol[i], nil 84 | } 85 | 86 | func timeAt(col Column, i int) (time.Time, error) { 87 | if err := checkInBounds(col, i); err != nil { 88 | return ZeroTime, err 89 | } 90 | 91 | if col.DType() != TimeType { 92 | return ZeroTime, fmt.Errorf("not a time.Time column") 93 | } 94 | 95 | typedCol, err := col.Times() 96 | if err != nil { 97 | return ZeroTime, err 98 | } 99 | 100 | return typedCol[i], nil 101 | } 102 | 103 | func boolAt(col Column, i int) (bool, error) { 104 | if err := checkInBounds(col, i); err != nil { 105 | return false, err 106 | } 107 | 108 | if col.DType() != BoolType { 109 | return false, fmt.Errorf("not a bool column") 110 | } 111 | 112 | typedCol, err := col.Bools() 113 | if err != nil { 114 | return false, err 115 | } 116 | 117 | return typedCol[i], nil 118 | } 119 | -------------------------------------------------------------------------------- /config_example.yaml: -------------------------------------------------------------------------------- 1 | log: 2 | level: "info" 3 | limit: 1000 4 | webApiEndpoint: "http://127.0.0.1:888" 5 | container: "bigdata" 6 | username: "iguazio" 7 | password: "t0ps3cr3t" 8 | 9 | backends: 10 | - type: "kv" 11 | - type: "stream" 12 | - type: "tsdb" 13 | workers: 16 14 | - type: "csv" 15 | rootdir: "/mnt/csvroot" 16 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | /* 22 | Package frames provides an efficient way of moving data from various sources. 23 | 24 | The package is composed os a HTTP web server that can serve data from various 25 | sources and from clients in Go and in Python. 26 | */ 27 | package frames 28 | -------------------------------------------------------------------------------- /frame_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package frames 22 | 23 | import ( 24 | "fmt" 25 | "testing" 26 | ) 27 | 28 | func TestFrameNew(t *testing.T) { 29 | val0, val1, size := int64(7), "n", 10 30 | col0, err := NewLabelColumn("col0", val0, size) 31 | if err != nil { 32 | t.Fatal(err) 33 | } 34 | 35 | col1, err := NewLabelColumn("col1", val1, size) 36 | if err != nil { 37 | t.Fatal(err) 38 | } 39 | 40 | cols := []Column{col0, col1} 41 | 42 | frame, err := NewFrame(cols, nil, nil) 43 | if err != nil { 44 | t.Fatalf("can't create frame - %s", err) 45 | } 46 | 47 | names := frame.Names() 48 | if len(names) != len(cols) { 49 | t.Fatalf("# of columns mismatch - %d != %d", len(names), len(cols)) 50 | } 51 | 52 | for i, name := range names { 53 | col := cols[i] 54 | if col.Name() != name { 55 | t.Fatalf("%d: name mismatch - %q != %q", i, col.Name(), name) 56 | } 57 | 58 | if col.Len() != size { 59 | t.Fatalf("%d: length mismatch - %d != %d", i, col.Len(), size) 60 | } 61 | 62 | switch i { 63 | case 0: 64 | val, _ := col.IntAt(0) 65 | if val != val0 { 66 | t.Fatalf("%d: value mismatch - %d != %d", i, val, val0) 67 | } 68 | case 1: 69 | val, _ := col.StringAt(0) 70 | if val != val1 { 71 | t.Fatalf("%d: value mismatch - %q != %q", i, val, val1) 72 | } 73 | } 74 | 75 | } 76 | } 77 | 78 | func TestFrameSlice(t *testing.T) { 79 | nCols, size := 7, 10 80 | cols := newIntCols(t, nCols, size) 81 | frame, err := NewFrame(cols, nil, nil) 82 | if err != nil { 83 | t.Fatalf("can't create frame - %s", err) 84 | } 85 | 86 | names := frame.Names() 87 | if len(names) != nCols { 88 | t.Fatalf("# of columns mismatch - %d != %d", len(names), nCols) 89 | } 90 | 91 | start, end := 2, 7 92 | frame2, err := frame.Slice(start, end) 93 | if err != nil { 94 | t.Fatalf("can't create slice - %s", err) 95 | } 96 | 97 | if frame2.Len() != end-start { 98 | t.Fatalf("bad # of rows in slice - %d != %d", frame2.Len(), end-start) 99 | } 100 | 101 | names2 := frame2.Names() 102 | if len(names2) != nCols { 103 | t.Fatalf("# of columns mismatch - %d != %d", len(names2), nCols) 104 | } 105 | } 106 | 107 | func TestFrameIndex(t *testing.T) { 108 | nCols, size := 2, 12 109 | cols := newIntCols(t, nCols, size) 110 | 111 | indices := newIntCols(t, 3, size) 112 | 113 | frame, err := NewFrame(cols, indices, nil) 114 | if err != nil { 115 | t.Fatal(err) 116 | } 117 | 118 | if len(frame.Indices()) != len(indices) { 119 | t.Fatalf("index len mismatch (%d != %d)", len(frame.Indices()), len(indices)) 120 | } 121 | } 122 | 123 | func TestNewFrameFromRows(t *testing.T) { 124 | rows := []map[string]interface{}{ 125 | {"x": 1, "y": "a"}, 126 | {"x": 2, "z": 1.0}, 127 | {"x": 3, "y": "b", "z": 2.0}, 128 | } 129 | 130 | indices := []string{"z"} 131 | frame, err := NewFrameFromRows(rows, indices, nil) 132 | if err != nil { 133 | t.Fatal(err) 134 | } 135 | 136 | if frame.Len() != len(rows) { 137 | t.Fatalf("rows len mismatch %d != %d", frame.Len(), len(rows)) 138 | } 139 | 140 | if len(frame.Names()) != 2 { 141 | t.Fatalf("columns len mismatch %d != %d", len(frame.Names()), 2) 142 | } 143 | 144 | if len(frame.Indices()) != len(indices) { 145 | t.Fatalf("indices len mismatch %d != %d", len(frame.Indices()), len(rows)) 146 | } 147 | } 148 | 149 | func TestNewFrameFromRowsMissing(t *testing.T) { 150 | rows := []map[string]interface{}{ 151 | {"x": 1, "y": "a"}, 152 | {"x": 2, "z": 1.0}, 153 | } 154 | 155 | frame, err := NewFrameFromRows(rows, nil, nil) 156 | if err != nil { 157 | t.Fatal(err) 158 | } 159 | 160 | if frame.Len() != 2 { 161 | t.Fatalf("frame length mismatch: %d != 2", frame.Len()) 162 | } 163 | } 164 | 165 | func newIntCols(t *testing.T, numCols int, size int) []Column { 166 | var cols []Column 167 | 168 | for i := 0; i < numCols; i++ { 169 | col, err := NewLabelColumn(fmt.Sprintf("col%d", i), i, size) 170 | if err != nil { 171 | t.Fatalf("can't create column - %s", err) 172 | } 173 | 174 | cols = append(cols, col) 175 | } 176 | 177 | return cols 178 | } 179 | -------------------------------------------------------------------------------- /framulate/config.go: -------------------------------------------------------------------------------- 1 | package framulate 2 | 3 | import ( 4 | "os" 5 | "time" 6 | 7 | "github.com/ghodss/yaml" 8 | "github.com/nuclio/errors" 9 | ) 10 | 11 | type scenarioKind string 12 | 13 | const ( 14 | scenarioKindWriteVerify = "writeVerify" 15 | ) 16 | 17 | type WriteVerifyConfig struct { 18 | NumTables int `json:"num_tables,omitempty"` 19 | NumSeriesPerTable int `json:"num_series_per_table,omitempty"` 20 | MaxParallelTablesCreate int `json:"max_parallel_tables_create,omitempty"` 21 | MaxParallelSeriesWrite int `json:"max_parallel_series_write,omitempty"` 22 | MaxParallelSeriesVerify int `json:"max_parallel_series_verify,omitempty"` 23 | WriteDummySeries bool `json:"write_dummy_series,omitempty"` 24 | NumDatapointsPerSeries int `json:"num_datapoints_per_series,omitempty"` 25 | WriteDelay string `json:"write_delay,omitempty"` 26 | VerificationDelay string `json:"verification_delay,omitempty"` 27 | Verify bool `json:"verify,omitempty"` 28 | 29 | verificationDelay time.Duration 30 | writeDelay time.Duration 31 | } 32 | 33 | type ScenarioConfig struct { 34 | Kind scenarioKind 35 | WriteVerify WriteVerifyConfig 36 | } 37 | 38 | type Transport struct { 39 | URL string `json:"url,omitempty"` 40 | MaxInflightRequests int `json:"max_inflight_requests,omitempty"` 41 | } 42 | 43 | type Config struct { 44 | ContainerName string `json:"container_name,omitempty"` 45 | UserName string `json:"username,omitempty"` 46 | AccessKey string `json:"access_key,omitempty"` 47 | Cleanup bool `json:"cleanup,omitempty"` 48 | MaxTasks int `json:"max_tasks,omitempty"` 49 | Scenario ScenarioConfig `json:"scenario,omitempty"` 50 | Transport Transport `json:"transport,omitempty"` 51 | } 52 | 53 | func NewConfigFromContentsOrPath(configContents []byte, configPath string) (*Config, error) { 54 | var err error 55 | 56 | if len(configContents) == 0 { 57 | if configPath == "" { 58 | return nil, errors.New("Config contents or path must be specified") 59 | } 60 | 61 | configContents, err = os.ReadFile(configPath) 62 | if err != nil { 63 | return nil, errors.Wrapf(err, "Failed to config file at %s", configPath) 64 | } 65 | } 66 | 67 | newConfig := Config{} 68 | 69 | if err := yaml.Unmarshal(configContents, &newConfig); err != nil { 70 | return nil, errors.Wrap(err, "Failed to unmarshal env spec file") 71 | } 72 | 73 | if err := newConfig.validateAndPopulateDefaults(); err != nil { 74 | return nil, errors.Wrap(err, "Failed to validate/popualte defaults") 75 | } 76 | 77 | return &newConfig, nil 78 | } 79 | 80 | func (c *Config) validateAndPopulateDefaults() error { 81 | var err error 82 | 83 | if c.Transport.URL == "" { 84 | c.Transport.URL = "grpc://framesd:8081" 85 | } 86 | 87 | if c.Transport.MaxInflightRequests == 0 { 88 | c.Transport.MaxInflightRequests = 512 89 | } 90 | 91 | if c.MaxTasks == 0 { 92 | c.MaxTasks = 1024 * 1024 93 | } 94 | 95 | if c.Scenario.WriteVerify.VerificationDelay != "" { 96 | c.Scenario.WriteVerify.verificationDelay, err = time.ParseDuration(c.Scenario.WriteVerify.VerificationDelay) 97 | if err != nil { 98 | return errors.Wrap(err, "Failed to parse verification delay") 99 | } 100 | } 101 | 102 | if c.Scenario.WriteVerify.WriteDelay != "" { 103 | c.Scenario.WriteVerify.writeDelay, err = time.ParseDuration(c.Scenario.WriteVerify.WriteDelay) 104 | if err != nil { 105 | return errors.Wrap(err, "Failed to parse write delay") 106 | } 107 | } 108 | 109 | return nil 110 | } 111 | -------------------------------------------------------------------------------- /framulate/framulate.go: -------------------------------------------------------------------------------- 1 | package framulate 2 | 3 | import ( 4 | "context" 5 | nethttp "net/http" 6 | //nolint: golint 7 | _ "net/http/pprof" 8 | "strings" 9 | "time" 10 | 11 | "github.com/nuclio/errors" 12 | "github.com/nuclio/logger" 13 | "github.com/v3io/frames" 14 | "github.com/v3io/frames/grpc" 15 | "github.com/v3io/frames/http" 16 | "github.com/v3io/frames/pb" 17 | "github.com/v3io/frames/repeatingtask" 18 | ) 19 | 20 | type Framulate struct { 21 | ctx context.Context 22 | logger logger.Logger 23 | taskPool *repeatingtask.Pool 24 | config *Config 25 | framesClient frames.Client 26 | scenario scenario 27 | } 28 | 29 | func NewFramulate(ctx context.Context, loggerInstance logger.Logger, config *Config) (*Framulate, error) { 30 | var err error 31 | 32 | newFramulate := Framulate{ 33 | config: config, 34 | logger: loggerInstance, 35 | } 36 | 37 | newFramulate.taskPool, err = repeatingtask.NewPool(ctx, 38 | config.MaxTasks, 39 | config.Transport.MaxInflightRequests) 40 | 41 | if err != nil { 42 | return nil, errors.Wrap(err, "Failed to create pool") 43 | } 44 | 45 | newFramulate.scenario, err = newFramulate.createScenario(config) 46 | if err != nil { 47 | return nil, errors.Wrap(err, "Failed to create scenario") 48 | } 49 | 50 | newFramulate.framesClient, err = newFramulate.createFramesClient(config) 51 | if err != nil { 52 | return nil, errors.Wrap(err, "Failed to create frames client") 53 | } 54 | 55 | go func() { 56 | err := nethttp.ListenAndServe(":8082", nil) 57 | if err != nil { 58 | newFramulate.logger.WarnWith("Failed to create profiling endpoint", "err", err) 59 | } 60 | }() 61 | 62 | newFramulate.logger.DebugWith("Framulate created", "config", config) 63 | 64 | return &newFramulate, nil 65 | } 66 | 67 | func (f *Framulate) Start() error { 68 | doneChan := make(chan struct{}) 69 | 70 | // log statistics periodically 71 | go func() { 72 | for { 73 | select { 74 | case <-time.After(1 * time.Second): 75 | f.scenario.LogStatistics() 76 | case <-doneChan: 77 | return 78 | } 79 | } 80 | }() 81 | 82 | err := f.scenario.Start() 83 | doneChan <- struct{}{} 84 | 85 | if err == nil { 86 | 87 | // final output 88 | f.scenario.LogStatistics() 89 | } 90 | 91 | return err 92 | } 93 | 94 | func (f *Framulate) createFramesClient(config *Config) (frames.Client, error) { 95 | session := pb.Session{ 96 | Container: config.ContainerName, 97 | User: config.UserName, 98 | Token: config.AccessKey, 99 | } 100 | 101 | if strings.HasPrefix(config.Transport.URL, "http") { 102 | f.logger.DebugWith("Creating HTTP client", "url", config.Transport.URL) 103 | 104 | return http.NewClient(config.Transport.URL, 105 | &session, 106 | f.logger) 107 | } 108 | 109 | f.logger.DebugWith("Creating gRPC client", "url", config.Transport.URL) 110 | 111 | return grpc.NewClient(config.Transport.URL, 112 | &session, 113 | f.logger) 114 | } 115 | 116 | func (f *Framulate) createScenario(config *Config) (scenario, error) { 117 | switch config.Scenario.Kind { 118 | case scenarioKindWriteVerify: 119 | return newWriteVerifyScenario(f.logger, f, config) 120 | default: 121 | return nil, errors.Errorf("Undefined scenario: %s", config.Scenario.Kind) 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/v3io/frames 2 | 3 | go 1.24 4 | 5 | require ( 6 | github.com/ghodss/yaml v1.0.0 7 | github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 8 | github.com/nuclio/errors v0.0.4 9 | github.com/nuclio/logger v0.0.1 10 | github.com/nuclio/zap v0.3.1 11 | github.com/pkg/errors v0.8.1 12 | github.com/stretchr/testify v1.8.4 13 | github.com/v3io/v3io-go v0.3.0 14 | github.com/v3io/v3io-tsdb v0.14.1 15 | github.com/valyala/fasthttp v1.44.0 16 | github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 17 | golang.org/x/text v0.25.0 18 | google.golang.org/grpc v1.72.0 19 | google.golang.org/protobuf v1.36.6 20 | ) 21 | 22 | require ( 23 | github.com/andybalholm/brotli v1.0.4 // indirect 24 | github.com/cespare/xxhash v1.1.0 // indirect 25 | github.com/davecgh/go-spew v1.1.1 // indirect 26 | github.com/imdario/mergo v0.3.7 // indirect 27 | github.com/klauspost/compress v1.15.9 // indirect 28 | github.com/logrusorgru/aurora/v4 v4.0.0 // indirect 29 | github.com/pmezard/go-difflib v1.0.0 // indirect 30 | github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a // indirect 31 | github.com/rogpeppe/go-internal v1.14.1 // indirect 32 | github.com/valyala/bytebufferpool v1.0.0 // indirect 33 | go.uber.org/multierr v1.11.0 // indirect 34 | go.uber.org/zap v1.27.0 // indirect 35 | golang.org/x/net v0.40.0 // indirect 36 | golang.org/x/sync v0.14.0 // indirect 37 | golang.org/x/sys v0.33.0 // indirect 38 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250512202823-5a2f75b736a9 // indirect 39 | gopkg.in/yaml.v2 v2.2.8 // indirect 40 | gopkg.in/yaml.v3 v3.0.1 // indirect 41 | zombiezen.com/go/capnproto2 v2.17.0+incompatible // indirect 42 | ) 43 | 44 | replace ( 45 | github.com/v3io/frames => ./ 46 | github.com/v3io/v3io-go => github.com/v3io/v3io-go v0.3.0 47 | github.com/xwb1989/sqlparser => github.com/v3io/sqlparser v0.0.0-20190306105200-4d7273501871 48 | ) 49 | -------------------------------------------------------------------------------- /grpc/end2end_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package grpc_test 22 | 23 | import ( 24 | "fmt" 25 | "net" 26 | "os" 27 | "reflect" 28 | "testing" 29 | "time" 30 | 31 | "github.com/v3io/frames" 32 | "github.com/v3io/frames/grpc" 33 | "github.com/v3io/frames/pb" 34 | ) 35 | 36 | func TestEnd2End(t *testing.T) { 37 | tmpDir, err := os.MkdirTemp("", "frames-grpc-e2e") 38 | if err != nil { 39 | t.Fatal(err) 40 | } 41 | 42 | backendName := "e2e-backend" 43 | cfg := &frames.Config{ 44 | Log: frames.LogConfig{ 45 | Level: "debug", 46 | }, 47 | Backends: []*frames.BackendConfig{ 48 | { 49 | Name: backendName, 50 | Type: "csv", 51 | RootDir: tmpDir, 52 | }, 53 | }, 54 | } 55 | 56 | port, err := freePort() 57 | if err != nil { 58 | t.Fatal(err) 59 | } 60 | 61 | srv, err := grpc.NewServer(cfg, fmt.Sprintf(":%d", port), nil, nil, "") 62 | if err != nil { 63 | t.Fatal(err) 64 | } 65 | if err := srv.Start(); err != nil { 66 | t.Fatal(err) 67 | } 68 | 69 | time.Sleep(100 * time.Millisecond) // Let server start 70 | 71 | url := fmt.Sprintf("localhost:%d", port) 72 | client, err := grpc.NewClient(url, nil, nil) 73 | if err != nil { 74 | t.Fatal(err) 75 | } 76 | 77 | frame, err := makeFrame() 78 | if err != nil { 79 | t.Fatalf("can't create frame - %s", err) 80 | } 81 | 82 | tableName := "e2e" 83 | writeReq := &frames.WriteRequest{ 84 | Backend: backendName, 85 | Table: tableName, 86 | } 87 | 88 | appender, err := client.Write(writeReq) 89 | if err != nil { 90 | t.Fatal(err) 91 | } 92 | 93 | if err := appender.Add(frame); err != nil { 94 | t.Fatal(err) 95 | } 96 | 97 | if err := appender.WaitForComplete(10 * time.Second); err != nil { 98 | t.Fatal(err) 99 | } 100 | 101 | readReq := &pb.ReadRequest{ 102 | Backend: backendName, 103 | Table: tableName, 104 | MessageLimit: 100, 105 | } 106 | 107 | it, err := client.Read(readReq) 108 | if err != nil { 109 | t.Fatal(err) 110 | } 111 | 112 | nRows := 0 113 | 114 | for it.Next() { 115 | iFrame := it.At() 116 | if !reflect.DeepEqual(iFrame.Names(), frame.Names()) { 117 | t.Fatalf("columns mismatch: %v != %v", iFrame.Names(), frame.Names()) 118 | } 119 | nRows += iFrame.Len() 120 | } 121 | 122 | if err := it.Err(); err != nil { 123 | t.Fatal(err) 124 | } 125 | 126 | if nRows != frame.Len() { 127 | t.Fatalf("# of rows mismatch - %d != %d", nRows, frame.Len()) 128 | } 129 | 130 | // Exec 131 | execReq := &pb.ExecRequest{ 132 | Backend: backendName, 133 | Table: tableName, 134 | Command: "ping", 135 | } 136 | 137 | if _, err := client.Exec(execReq); err != nil { 138 | t.Fatalf("can't exec - %s", err) 139 | } 140 | } 141 | 142 | func makeFrame() (frames.Frame, error) { 143 | size := 1027 144 | now := time.Now() 145 | idata := make([]int64, size) 146 | fdata := make([]float64, size) 147 | sdata := make([]string, size) 148 | tdata := make([]time.Time, size) 149 | bdata := make([]bool, size) 150 | 151 | for i := 0; i < size; i++ { 152 | idata[i] = int64(i) 153 | fdata[i] = float64(i) 154 | sdata[i] = fmt.Sprintf("val%d", i) 155 | tdata[i] = now.Add(time.Duration(i) * time.Second) 156 | bdata[i] = i%2 == 0 157 | } 158 | 159 | columns := map[string]interface{}{ 160 | "ints": idata, 161 | "floats": fdata, 162 | "strings": sdata, 163 | "times": tdata, 164 | "bools": bdata, 165 | } 166 | return frames.NewFrameFromMap(columns, nil) 167 | } 168 | 169 | func freePort() (int, error) { 170 | l, err := net.Listen("tcp", ":0") 171 | if err != nil { 172 | return 0, err 173 | } 174 | 175 | l.Close() 176 | return l.Addr().(*net.TCPAddr).Port, nil 177 | } 178 | -------------------------------------------------------------------------------- /http/client_example_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package http 22 | 23 | import ( 24 | "fmt" 25 | "time" 26 | 27 | "github.com/v3io/frames" 28 | "github.com/v3io/frames/pb" 29 | ) 30 | 31 | func ExampleClient() { 32 | tableName := "example_table" 33 | url := "http://localhost:8080" 34 | fmt.Println(">>> connecting") 35 | // nil, nil are session and logger 36 | client, err := NewClient(url, nil, nil) 37 | if err != nil { 38 | fmt.Printf("can't connect to %q - %s", url, err) 39 | return 40 | } 41 | 42 | frame, err := makeFrame() 43 | if err != nil { 44 | fmt.Printf("can't create frame - %s", err) 45 | return 46 | } 47 | 48 | fmt.Println(">>> writing") 49 | writeReq := &frames.WriteRequest{ 50 | Backend: "weather", 51 | Table: tableName, 52 | } 53 | 54 | appender, err := client.Write(writeReq) 55 | if err != nil { 56 | if err != nil { 57 | fmt.Printf("can't write - %s", err) 58 | return 59 | } 60 | } 61 | 62 | if err := appender.Add(frame); err != nil { 63 | fmt.Printf("can't write frame - %s", err) 64 | return 65 | } 66 | 67 | if err := appender.WaitForComplete(10 * time.Second); err != nil { 68 | fmt.Printf("can't wait - %s", err) 69 | return 70 | } 71 | 72 | fmt.Println(">>> reading") 73 | readReq := &pb.ReadRequest{ 74 | Backend: "weather", 75 | Table: tableName, 76 | MessageLimit: 100, 77 | } 78 | 79 | it, err := client.Read(readReq) 80 | if err != nil { 81 | fmt.Printf("can't query - %s", err) 82 | return 83 | } 84 | 85 | for it.Next() { 86 | frame := it.At() 87 | fmt.Println(frame.Names()) 88 | fmt.Printf("%d rows\n", frame.Len()) 89 | fmt.Println("-----------") 90 | } 91 | 92 | if err := it.Err(); err != nil { 93 | fmt.Printf("error in iterator - %s", err) 94 | return 95 | } 96 | } 97 | 98 | func makeFrame() (frames.Frame, error) { 99 | size := 1027 100 | now := time.Now() 101 | idata := make([]int64, size) 102 | fdata := make([]float64, size) 103 | sdata := make([]string, size) 104 | tdata := make([]time.Time, size) 105 | 106 | for i := 0; i < size; i++ { 107 | idata[i] = int64(i) 108 | fdata[i] = float64(i) 109 | sdata[i] = fmt.Sprintf("val%d", i) 110 | tdata[i] = now.Add(time.Duration(i) * time.Second) 111 | } 112 | 113 | columns := map[string]interface{}{ 114 | "ints": idata, 115 | "floats": fdata, 116 | "strings": sdata, 117 | "times": tdata, 118 | } 119 | return frames.NewFrameFromMap(columns, nil) 120 | } 121 | -------------------------------------------------------------------------------- /http/client_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package http 22 | 23 | import ( 24 | "testing" 25 | ) 26 | 27 | func TestNewClient(t *testing.T) { 28 | 29 | hostname := "iguaz.io" 30 | 31 | client, err := NewClient(hostname, nil, nil) 32 | if err != nil { 33 | t.Fatal(err) 34 | } 35 | 36 | if client.logger == nil { 37 | t.Fatal("no logger") 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /http/end2end_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package http_test 22 | 23 | import ( 24 | "encoding/json" 25 | "fmt" 26 | "net" 27 | nhttp "net/http" 28 | "os" 29 | "reflect" 30 | "testing" 31 | "time" 32 | 33 | "github.com/v3io/frames" 34 | "github.com/v3io/frames/http" 35 | "github.com/v3io/frames/pb" 36 | ) 37 | 38 | func TestEnd2End(t *testing.T) { 39 | tmpDir, err := os.MkdirTemp("", "frames-e2e") 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | 44 | backendName := "e2e-backend" 45 | cfg := &frames.Config{ 46 | Log: frames.LogConfig{ 47 | Level: "debug", 48 | }, 49 | Backends: []*frames.BackendConfig{ 50 | { 51 | Name: backendName, 52 | Type: "csv", 53 | RootDir: tmpDir, 54 | }, 55 | }, 56 | } 57 | 58 | port, err := freePort() 59 | if err != nil { 60 | t.Fatal(err) 61 | } 62 | 63 | srv, err := http.NewServer(cfg, fmt.Sprintf(":%d", port), nil, nil, "") 64 | if err != nil { 65 | t.Fatal(err) 66 | } 67 | 68 | if err := srv.Start(); err != nil { 69 | t.Fatal(err) 70 | } 71 | 72 | time.Sleep(100 * time.Millisecond) // Let server start 73 | 74 | url := fmt.Sprintf("http://localhost:%d", port) 75 | client, err := http.NewClient(url, nil, nil) 76 | if err != nil { 77 | t.Fatal(err) 78 | } 79 | 80 | frame, err := makeFrame() 81 | if err != nil { 82 | t.Fatalf("can't create frame - %s", err) 83 | } 84 | 85 | tableName := "e2e" 86 | writeReq := &frames.WriteRequest{ 87 | Backend: backendName, 88 | Table: tableName, 89 | } 90 | 91 | appender, err := client.Write(writeReq) 92 | if err != nil { 93 | t.Fatal(err) 94 | } 95 | 96 | if err := appender.Add(frame); err != nil { 97 | t.Fatal(err) 98 | } 99 | 100 | if err := appender.WaitForComplete(10 * time.Second); err != nil { 101 | t.Fatal(err) 102 | } 103 | 104 | readReq := &pb.ReadRequest{ 105 | Backend: backendName, 106 | Table: tableName, 107 | MessageLimit: 100, 108 | } 109 | 110 | it, err := client.Read(readReq) 111 | if err != nil { 112 | t.Fatal(err) 113 | } 114 | 115 | nRows := 0 116 | 117 | for it.Next() { 118 | iFrame := it.At() 119 | if !reflect.DeepEqual(iFrame.Names(), frame.Names()) { 120 | t.Fatalf("columns mismatch: %v != %v", iFrame.Names(), frame.Names()) 121 | } 122 | nRows += iFrame.Len() 123 | } 124 | 125 | if err := it.Err(); err != nil { 126 | t.Fatal(err) 127 | } 128 | 129 | if nRows != frame.Len() { 130 | t.Fatalf("# of rows mismatch - %d != %d", nRows, frame.Len()) 131 | } 132 | 133 | testGrafana(t, url, backendName, tableName) 134 | 135 | // Exec 136 | execReq := &pb.ExecRequest{ 137 | Backend: backendName, 138 | Table: tableName, 139 | Command: "ping", 140 | } 141 | 142 | if _, err := client.Exec(execReq); err != nil { 143 | t.Fatalf("can't exec - %s", err) 144 | } 145 | } 146 | 147 | func testGrafana(t *testing.T, baseURL string, backend string, table string) { 148 | // ack 149 | ackURL := fmt.Sprintf("%s/", baseURL) 150 | resp, err := nhttp.Get(ackURL) 151 | if err != nil { 152 | t.Fatalf("can't call simplejson API - %s", err) 153 | } 154 | 155 | if resp.StatusCode != nhttp.StatusOK { 156 | t.Fatalf("bad status from simplejson API - %d %s", resp.StatusCode, resp.Status) 157 | } 158 | var responseContent map[string]interface{} 159 | if err := json.NewDecoder(resp.Body).Decode(&responseContent); err != nil { 160 | t.Fatalf("can't decode simplejson API ACK response - %s", err) 161 | } else { 162 | if responseContent["state"].(string) != "running" { 163 | t.Fatalf("wrong simplejson API response on status check: %s", responseContent["state"]) 164 | } 165 | } 166 | } 167 | 168 | func freePort() (int, error) { 169 | l, err := net.Listen("tcp", ":0") 170 | if err != nil { 171 | return 0, err 172 | } 173 | 174 | l.Close() 175 | return l.Addr().(*net.TCPAddr).Port, nil 176 | } 177 | 178 | func makeFrame() (frames.Frame, error) { 179 | size := 1027 180 | now := time.Now() 181 | idata := make([]int64, size) 182 | fdata := make([]float64, size) 183 | sdata := make([]string, size) 184 | tdata := make([]time.Time, size) 185 | bdata := make([]bool, size) 186 | 187 | for i := 0; i < size; i++ { 188 | idata[i] = int64(i) 189 | fdata[i] = float64(i) 190 | sdata[i] = fmt.Sprintf("val%d", i) 191 | tdata[i] = now.Add(time.Duration(i) * time.Second) 192 | bdata[i] = i%2 == 0 193 | } 194 | 195 | columns := map[string]interface{}{ 196 | "ints": idata, 197 | "floats": fdata, 198 | "strings": sdata, 199 | "times": tdata, 200 | "bools": bdata, 201 | } 202 | return frames.NewFrameFromMap(columns, nil) 203 | } 204 | -------------------------------------------------------------------------------- /http/prof.go: -------------------------------------------------------------------------------- 1 | //go:build profile 2 | // +build profile 3 | 4 | /* 5 | Copyright 2018 Iguazio Systems Ltd. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License") with 8 | an addition restriction as set forth herein. You may not use this 9 | file except in compliance with the License. You may obtain a copy of 10 | the License at http://www.apache.org/licenses/LICENSE-2.0. 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 15 | implied. See the License for the specific language governing 16 | permissions and limitations under the License. 17 | 18 | In addition, you may not use the software for any purposes that are 19 | illegal under applicable law, and the grant of the foregoing license 20 | under the Apache 2.0 license is conditioned upon your compliance with 21 | such restriction. 22 | */ 23 | 24 | package http 25 | 26 | import ( 27 | "log" 28 | "net/http" 29 | // Install pprof web handler 30 | _ "net/http/pprof" 31 | "os" 32 | ) 33 | 34 | func init() { 35 | addr := os.Getenv("V3IO_PROFILE_PORT") 36 | if addr == "" { 37 | addr = ":6767" 38 | } 39 | 40 | go func() { 41 | if err := http.ListenAndServe(addr, nil); err != nil { 42 | log.Printf("error: profile: can't listen on %s", addr) 43 | } 44 | }() 45 | } 46 | -------------------------------------------------------------------------------- /http/server_example_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package http 22 | 23 | import ( 24 | "fmt" 25 | "time" 26 | 27 | "github.com/ghodss/yaml" 28 | "github.com/v3io/frames" 29 | ) 30 | 31 | var configData = []byte(` 32 | log: 33 | level: "info" 34 | 35 | backends: 36 | - type: "kv" 37 | - type: "csv" 38 | rootDir = "/tmp" 39 | `) 40 | 41 | func ExampleServer() { 42 | cfg := &frames.Config{} 43 | if err := yaml.Unmarshal(configData, cfg); err != nil { 44 | fmt.Printf("error: can't read config - %s", err) 45 | return 46 | } 47 | 48 | srv, err := NewServer(cfg, ":8080", nil, nil, "") 49 | if err != nil { 50 | fmt.Printf("error: can't create server - %s", err) 51 | return 52 | } 53 | 54 | if err = srv.Start(); err != nil { 55 | fmt.Printf("error: can't start server - %s", err) 56 | return 57 | } 58 | 59 | fmt.Println("server running") 60 | for { 61 | time.Sleep(60 * time.Second) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /http/server_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package http 22 | 23 | import ( 24 | "fmt" 25 | "net/http" 26 | "testing" 27 | 28 | "github.com/v3io/frames" 29 | "github.com/valyala/fasthttp" 30 | ) 31 | 32 | const ( 33 | authHeader = "Authorization" 34 | ) 35 | 36 | func createServer() (*Server, error) { 37 | cfg := &frames.Config{ 38 | Backends: []*frames.BackendConfig{ 39 | { 40 | Name: "weather", 41 | Type: "csv", 42 | }, 43 | }, 44 | } 45 | address := ":8080" 46 | return NewServer(cfg, address, nil, nil, "") 47 | } 48 | 49 | func TestNew(t *testing.T) { 50 | srv, err := createServer() 51 | if err != nil { 52 | t.Fatal(err) 53 | } 54 | 55 | if srv.State() != frames.ReadyState { 56 | t.Fatalf("bad initial state - %q", srv.State()) 57 | } 58 | } 59 | 60 | func TestBasicAuth(t *testing.T) { 61 | r, err := http.NewRequest("GET", "/", nil) 62 | if err != nil { 63 | t.Fatal(err) 64 | } 65 | user, password := "bugs", "duck season" 66 | r.SetBasicAuth(user, password) 67 | 68 | ctx := &fasthttp.RequestCtx{} 69 | ctx.Request.Header.Add(authHeader, r.Header.Get(authHeader)) 70 | 71 | session := &frames.Session{} 72 | srv, err := createServer() 73 | if err != nil { 74 | t.Fatal(err) 75 | } 76 | srv.httpAuth(ctx, session) 77 | if session.User != user { 78 | t.Fatalf("bad user: %q != %q", session.User, user) 79 | } 80 | if session.Password != password { 81 | t.Fatalf("bad password: %q != %q", session.Password, password) 82 | } 83 | } 84 | 85 | func TestBearerAuth(t *testing.T) { 86 | ctx := &fasthttp.RequestCtx{} 87 | token := "valid for 1 game" 88 | headerValue := fmt.Sprintf("%s%s", bearerAuthPrefix, token) 89 | ctx.Request.Header.Add(authHeader, headerValue) 90 | 91 | session := &frames.Session{} 92 | srv, err := createServer() 93 | if err != nil { 94 | t.Fatal(err) 95 | } 96 | srv.httpAuth(ctx, session) 97 | if session.Token != token { 98 | t.Fatalf("bad token: %q != %q", session.Token, token) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /log.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package frames 22 | 23 | import ( 24 | "os" 25 | 26 | "github.com/nuclio/logger" 27 | nucliozap "github.com/nuclio/zap" 28 | ) 29 | 30 | var ( 31 | // DefaultLogLevel is the default log verbosity 32 | DefaultLogLevel string 33 | ) 34 | 35 | // NewLogger returns a new logger 36 | func NewLogger(verbose string) (logger.Logger, error) { 37 | if verbose == "" { 38 | verbose = DefaultLogLevel 39 | } 40 | 41 | logLevel := nucliozap.GetLevelByName(verbose) 42 | return nucliozap.NewNuclioZapCmd("v3io-frames", logLevel, os.Stdout) 43 | } 44 | -------------------------------------------------------------------------------- /log_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package frames 22 | 23 | import ( 24 | "testing" 25 | ) 26 | 27 | func TestNewLogger(t *testing.T) { 28 | logger, err := NewLogger("debug") 29 | if err != nil { 30 | t.Fatalf("can't create logger - %s", err) 31 | } 32 | 33 | if logger == nil { 34 | t.Fatalf("nil logger") 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /marshal.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | // We encode messages in protobuf with an int64 leading size header 22 | 23 | package frames 24 | 25 | import ( 26 | "bytes" 27 | "encoding/binary" 28 | "fmt" 29 | "io" 30 | 31 | "github.com/pkg/errors" 32 | "github.com/v3io/frames/pb" 33 | "google.golang.org/protobuf/proto" 34 | ) 35 | 36 | var ( 37 | byteOrder = binary.LittleEndian 38 | ) 39 | 40 | // Encoder is message encoder 41 | type Encoder struct { 42 | w io.Writer 43 | } 44 | 45 | // NewEncoder returns new Encoder 46 | func NewEncoder(w io.Writer) *Encoder { 47 | return &Encoder{w} 48 | } 49 | 50 | // MarshalFrame serializes a frame to []byte 51 | func MarshalFrame(frame Frame) ([]byte, error) { 52 | iface, ok := frame.(pb.Framed) 53 | if !ok { 54 | return nil, fmt.Errorf("unknown frame type") 55 | } 56 | 57 | return proto.Marshal(iface.Proto()) 58 | } 59 | 60 | // UnmarshalFrame de-serialize a frame from []byte 61 | func UnmarshalFrame(data []byte) (Frame, error) { 62 | msg := &pb.Frame{} 63 | if err := proto.Unmarshal(data, msg); err != nil { 64 | return nil, err 65 | } 66 | 67 | return NewFrameFromProto(msg), nil 68 | } 69 | 70 | // Encode encoders the message to e.w 71 | func (e *Encoder) Encode(msg proto.Message) error { 72 | data, err := proto.Marshal(msg) 73 | if err != nil { 74 | return errors.Wrap(err, "can't encode with message") 75 | } 76 | 77 | size := int64(len(data)) 78 | if err := binary.Write(e.w, byteOrder, size); err != nil { 79 | return errors.Wrap(err, "can't write size header") 80 | } 81 | 82 | n, err := e.w.Write(data) 83 | if err != nil { 84 | return errors.Wrap(err, "can't write message body") 85 | } 86 | 87 | if int64(n) != size { 88 | return errors.Errorf("wrote only %d bytes out of %d", n, size) 89 | } 90 | 91 | return nil 92 | } 93 | 94 | // Decoder is message decoder 95 | type Decoder struct { 96 | r io.Reader 97 | buf *bytes.Buffer 98 | } 99 | 100 | // NewDecoder returns a new Decoder 101 | func NewDecoder(r io.Reader) *Decoder { 102 | var buf bytes.Buffer 103 | return &Decoder{r, &buf} 104 | } 105 | 106 | // Decode decodes message from d.r 107 | func (d *Decoder) Decode(msg proto.Message) error { 108 | var size int64 109 | if err := binary.Read(d.r, byteOrder, &size); err != nil { 110 | if err == io.EOF { 111 | // Propogate EOF to clients 112 | return err 113 | } 114 | return errors.Wrap(err, "can't read header") 115 | } 116 | 117 | d.buf.Reset() 118 | n, err := io.CopyN(d.buf, d.r, size) 119 | if err != nil { 120 | return errors.Wrap(err, "can't read message body") 121 | } 122 | 123 | if n != size { 124 | return errors.Errorf("read only %d bytes out of %d", n, size) 125 | } 126 | 127 | if err := proto.Unmarshal(d.buf.Bytes(), msg); err != nil { 128 | return errors.Wrap(err, "can't decode message body") 129 | } 130 | 131 | return nil 132 | } 133 | -------------------------------------------------------------------------------- /repeatingtask/pool.go: -------------------------------------------------------------------------------- 1 | package repeatingtask 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/nuclio/errors" 7 | ) 8 | 9 | type Pool struct { 10 | ctx context.Context 11 | taskChan chan *Task 12 | workers []*worker 13 | } 14 | 15 | func NewPool(ctx context.Context, maxTasks int, numWorkers int) (*Pool, error) { 16 | newPool := Pool{} 17 | newPool.taskChan = make(chan *Task, maxTasks) 18 | 19 | // create workers 20 | for workerIdx := 0; workerIdx < numWorkers; workerIdx++ { 21 | newWorker, err := newWorker(ctx, &newPool) 22 | if err != nil { 23 | return nil, errors.Wrap(err, "Failed to create worker") 24 | } 25 | 26 | newPool.workers = append(newPool.workers, newWorker) 27 | } 28 | 29 | return &newPool, nil 30 | } 31 | 32 | func (p *Pool) SubmitTaskAndWait(task *Task) TaskErrors { 33 | if err := p.SubmitTask(task); err != nil { 34 | return TaskErrors{ 35 | taskErrors: []*TaskError{ 36 | {Error: errors.Wrap(err, "Failed to submit task")}, 37 | }, 38 | } 39 | } 40 | 41 | return task.Wait() 42 | } 43 | 44 | func (p *Pool) SubmitTask(task *Task) error { 45 | 46 | if err := task.initialize(); err != nil { 47 | return errors.Wrap(err, "Failed to initialize channel") 48 | } 49 | 50 | for parallelIdx := 0; parallelIdx < task.MaxParallel; parallelIdx++ { 51 | select { 52 | case p.taskChan <- task: 53 | default: 54 | return errors.New("Failed to submit task - enlarge the pool max # of tasks") 55 | } 56 | } 57 | 58 | return nil 59 | } 60 | -------------------------------------------------------------------------------- /repeatingtask/pool_test.go: -------------------------------------------------------------------------------- 1 | package repeatingtask 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/nuclio/logger" 8 | nucliozap "github.com/nuclio/zap" 9 | "github.com/pkg/errors" 10 | "github.com/stretchr/testify/suite" 11 | ) 12 | 13 | type handlerConfig struct { 14 | name string 15 | delay time.Duration 16 | errorAfter int 17 | } 18 | 19 | type poolSuite struct { 20 | suite.Suite 21 | pool *Pool 22 | logger logger.Logger 23 | ctx context.Context 24 | } 25 | 26 | func (suite *poolSuite) SetupTest() { 27 | var err error 28 | 29 | suite.logger, _ = nucliozap.NewNuclioZapTest("test") 30 | suite.ctx = context.Background() 31 | 32 | suite.pool, err = NewPool(context.TODO(), 1024, 32) 33 | suite.Require().NoError(err) 34 | } 35 | 36 | func (suite *poolSuite) TestNoParallel() { 37 | suite.T().Skip() 38 | 39 | task := &Task{ 40 | NumReptitions: 16, 41 | MaxParallel: 1, 42 | MaxNumErrors: 0, 43 | Handler: suite.delayingNoConcurrentHandler, 44 | Cookie: 100 * time.Millisecond, 45 | } 46 | 47 | err := suite.pool.SubmitTask(task) 48 | suite.Require().NoError(err) 49 | 50 | <-task.OnCompleteChan 51 | } 52 | 53 | func (suite *poolSuite) TestParallel() { 54 | task1 := &Task{ 55 | NumReptitions: 512, 56 | MaxParallel: 4, 57 | MaxNumErrors: 0, 58 | Handler: suite.delayingErrorHandler, 59 | Cookie: &handlerConfig{ 60 | name: "task1", 61 | delay: 1000 * time.Millisecond, 62 | }, 63 | } 64 | 65 | err := suite.pool.SubmitTask(task1) 66 | suite.Require().NoError(err) 67 | 68 | task2 := &Task{ 69 | NumReptitions: 256, 70 | MaxParallel: 8, 71 | MaxNumErrors: 0, 72 | Handler: suite.delayingErrorHandler, 73 | Cookie: &handlerConfig{ 74 | name: "task2", 75 | delay: 500 * time.Millisecond, 76 | }, 77 | } 78 | 79 | err = suite.pool.SubmitTask(task2) 80 | suite.Require().NoError(err) 81 | 82 | task1Errors := task1.Wait() 83 | task2Errors := task2.Wait() 84 | 85 | suite.Require().NoError(task1Errors.Error()) 86 | suite.Require().NoError(task2Errors.Error()) 87 | } 88 | 89 | func (suite *poolSuite) TestErrors() { 90 | task1 := &Task{ 91 | NumReptitions: 256, 92 | MaxParallel: 4, 93 | MaxNumErrors: 1, 94 | Handler: suite.delayingErrorHandler, 95 | Cookie: &handlerConfig{ 96 | name: "task1", 97 | errorAfter: 20, 98 | }, 99 | } 100 | 101 | //taskErrors := suite.pool.SubmitTaskAndWait(task1) 102 | //suite.Require().Error(taskErrors.Error()) 103 | 104 | task2 := &Task{ 105 | NumReptitions: 128, 106 | MaxParallel: 4, 107 | MaxNumErrors: 4, 108 | Handler: suite.delayingErrorHandler, 109 | Cookie: &handlerConfig{ 110 | name: "task2", 111 | errorAfter: 50, 112 | }, 113 | } 114 | 115 | taskGroup, err := NewTaskGroup() 116 | suite.Require().NoError(err) 117 | 118 | err = suite.pool.SubmitTask(task1) 119 | suite.Require().NoError(err) 120 | 121 | err = suite.pool.SubmitTask(task2) 122 | suite.Require().NoError(err) 123 | 124 | err = taskGroup.AddTask(task1) 125 | suite.Require().NoError(err) 126 | err = taskGroup.AddTask(task2) 127 | suite.Require().NoError(err) 128 | 129 | taskGroupErrors := taskGroup.Wait() 130 | suite.Require().Error(taskGroupErrors.Error()) 131 | suite.logger.DebugWith("Got error", "err", taskGroupErrors.Error()) 132 | } 133 | 134 | func (suite *poolSuite) delayingNoConcurrentHandler(cookie interface{}, repetitionIndex int) error { 135 | suite.logger.DebugWith("Called", "rep", repetitionIndex) 136 | 137 | // TODO: test not running in parallel 138 | time.Sleep(cookie.(time.Duration)) 139 | 140 | return nil 141 | } 142 | 143 | func (suite *poolSuite) delayingErrorHandler(cookie interface{}, repetitionIndex int) error { 144 | handlerConfig := cookie.(*handlerConfig) 145 | 146 | suite.logger.DebugWith("Called", 147 | "rep", repetitionIndex, 148 | "name", handlerConfig.name, 149 | "errorAfter", handlerConfig.errorAfter) 150 | 151 | if handlerConfig.delay != 0 { 152 | time.Sleep(handlerConfig.delay) 153 | } 154 | 155 | if handlerConfig.errorAfter != 0 && repetitionIndex > handlerConfig.errorAfter { 156 | return errors.Errorf("Error at repetition %d", repetitionIndex) 157 | } 158 | 159 | return nil 160 | } 161 | 162 | //func TestPoolSuite(t *testing.T) { 163 | // suite.Run(t, new(poolSuite)) 164 | //} 165 | -------------------------------------------------------------------------------- /repeatingtask/task.go: -------------------------------------------------------------------------------- 1 | package repeatingtask 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | 8 | "github.com/nuclio/errors" 9 | ) 10 | 11 | const ( 12 | InfiniteFailures = -1 13 | ) 14 | 15 | type TaskError struct { 16 | Repetition int 17 | Error error 18 | } 19 | 20 | type TaskErrors struct { 21 | taskErrors []*TaskError 22 | stringValue string 23 | } 24 | 25 | func (te *TaskErrors) String() string { 26 | if te.stringValue != "" { 27 | return te.stringValue 28 | } 29 | 30 | errorString := "" 31 | 32 | for _, taskError := range te.taskErrors { 33 | errorString += fmt.Sprintf("%d: %s\n", 34 | taskError.Repetition, 35 | errors.GetErrorStackString(taskError.Error, 10)) 36 | } 37 | 38 | te.stringValue = errorString 39 | 40 | return te.stringValue 41 | } 42 | 43 | func (te *TaskErrors) Error() error { 44 | if len(te.taskErrors) == 0 { 45 | return nil 46 | } 47 | 48 | return errors.New(te.String()) 49 | } 50 | 51 | type Task struct { 52 | NumReptitions int 53 | MaxParallel int 54 | Handler func(interface{}, int) error 55 | OnCompleteChan chan struct{} 56 | Timeout time.Duration 57 | ErrorsChan chan *TaskError 58 | MaxNumErrors int 59 | Cookie interface{} 60 | 61 | repititionIndex uint64 62 | numCompletions uint64 63 | numInstancesInTaskChan uint64 64 | lock sync.Locker 65 | } 66 | 67 | func (t *Task) initialize() error { 68 | t.lock = &sync.Mutex{} 69 | t.OnCompleteChan = make(chan struct{}, 1) 70 | t.ErrorsChan = make(chan *TaskError, t.NumReptitions) 71 | 72 | return nil 73 | } 74 | 75 | func (t *Task) Wait() TaskErrors { 76 | <-t.OnCompleteChan 77 | 78 | // read errors 79 | var taskErrors TaskErrors 80 | done := false 81 | 82 | for !done { 83 | select { 84 | case taskError := <-t.ErrorsChan: 85 | taskErrors.taskErrors = append(taskErrors.taskErrors, taskError) 86 | default: 87 | done = true 88 | } 89 | } 90 | 91 | return taskErrors 92 | } 93 | -------------------------------------------------------------------------------- /repeatingtask/taskgroup.go: -------------------------------------------------------------------------------- 1 | package repeatingtask 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | "github.com/nuclio/errors" 8 | ) 9 | 10 | type TaskGroupErrors struct { 11 | taskErrors []TaskErrors 12 | } 13 | 14 | func (tge *TaskGroupErrors) Errors() []error { 15 | var errors []error 16 | 17 | for _, err := range tge.taskErrors { 18 | if err.Error() != nil { 19 | errors = append(errors, err.Error()) 20 | } 21 | } 22 | 23 | return errors 24 | } 25 | 26 | func (tge *TaskGroupErrors) Error() error { 27 | taskGroupErrors := tge.Errors() 28 | 29 | if len(taskGroupErrors) == 0 { 30 | return nil 31 | } 32 | 33 | errorString := "" 34 | for _, err := range taskGroupErrors { 35 | errorString += fmt.Sprintf("%s\n", err.Error()) 36 | } 37 | 38 | return errors.New(errorString) 39 | } 40 | 41 | type TaskGroup struct { 42 | tasks []*Task 43 | tasksLock sync.Locker 44 | } 45 | 46 | func NewTaskGroup() (*TaskGroup, error) { 47 | return &TaskGroup{ 48 | tasksLock: &sync.Mutex{}, 49 | }, nil 50 | } 51 | 52 | func (t *TaskGroup) AddTask(task *Task) error { 53 | t.tasksLock.Lock() 54 | t.tasks = append(t.tasks, task) 55 | t.tasksLock.Unlock() 56 | 57 | return nil 58 | } 59 | 60 | func (t *TaskGroup) Wait() TaskGroupErrors { 61 | taskGroupErrors := TaskGroupErrors{} 62 | 63 | // iterate over tasks and read into task group errors 64 | for _, task := range t.tasks { 65 | 66 | // wait for task and add task errors 67 | taskGroupErrors.taskErrors = append(taskGroupErrors.taskErrors, task.Wait()) 68 | } 69 | 70 | return taskGroupErrors 71 | } 72 | -------------------------------------------------------------------------------- /repeatingtask/worker.go: -------------------------------------------------------------------------------- 1 | package repeatingtask 2 | 3 | import ( 4 | "context" 5 | "sync/atomic" 6 | ) 7 | 8 | type worker struct { 9 | pool *Pool 10 | ctx context.Context 11 | } 12 | 13 | func newWorker(ctx context.Context, pool *Pool) (*worker, error) { 14 | newWorker := worker{ 15 | pool: pool, 16 | ctx: ctx, 17 | } 18 | 19 | go newWorker.handleTasks() // nolint: errcheck 20 | 21 | return &newWorker, nil 22 | } 23 | 24 | func (w *worker) handleTasks() error { 25 | for { 26 | task := <-w.pool.taskChan 27 | _ = w.handleTask(task) 28 | } 29 | } 30 | 31 | func (w *worker) handleTask(task *Task) error { 32 | 33 | // check if task errored out. if so, just return since we assume that the worker 34 | // that injected the last error will mark it as complete 35 | if len(task.ErrorsChan) > task.MaxNumErrors { 36 | return nil 37 | } 38 | 39 | // increment repetition count and check if we passed it. if we did, don't handle the task 40 | repetitionIndex := atomic.AddUint64(&task.repititionIndex, 1) 41 | if int(repetitionIndex) > task.NumReptitions { 42 | return nil 43 | } 44 | 45 | // the index the user wants to see is 0 based 46 | repetitionIndex-- 47 | 48 | // call the task 49 | err := task.Handler(task.Cookie, int(repetitionIndex)) 50 | 51 | // if there was an error, shove it to the error channel 52 | if err != nil { 53 | task.ErrorsChan <- &TaskError{ 54 | Repetition: int(repetitionIndex), 55 | Error: err, 56 | } 57 | } 58 | 59 | // increment # of times we completed 60 | currentNumCompletions := atomic.AddUint64(&task.numCompletions, 1) 61 | 62 | // signal that we're done if there were more failures than allowed or that w're simply done 63 | if int(currentNumCompletions) >= task.NumReptitions || 64 | len(task.ErrorsChan) > task.MaxNumErrors { 65 | w.signalTaskComplete(task) 66 | } 67 | 68 | // return our instance of the task into the pool so that another worker can handle it 69 | w.pool.taskChan <- task 70 | 71 | return nil 72 | } 73 | 74 | func (w *worker) signalTaskComplete(task *Task) { 75 | 76 | // write to the channel, but don't block. it's possible that many workers are signaling 77 | // completion of the task (e.g. multiple errors exceeding threshold) 78 | select { 79 | case task.OnCompleteChan <- struct{}{}: 80 | default: 81 | return 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /rowiter.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package frames 22 | 23 | import ( 24 | "fmt" 25 | "sync" 26 | ) 27 | 28 | type rowIterator struct { 29 | // names for columns with no name 30 | noNames map[int]string 31 | columns []Column 32 | err error 33 | frame Frame 34 | includeIndex bool 35 | numCols int 36 | once sync.Once 37 | row map[string]interface{} 38 | rowNum int 39 | } 40 | 41 | func newRowIterator(frame Frame, includeIndex bool) *rowIterator { 42 | return &rowIterator{ 43 | frame: frame, 44 | includeIndex: includeIndex, 45 | } 46 | } 47 | 48 | func (it *rowIterator) init() { 49 | names := it.frame.Names() 50 | it.numCols = len(names) 51 | size := len(names) 52 | if it.includeIndex { 53 | size += len(it.frame.Indices()) 54 | } 55 | 56 | nameMap := make(map[string]bool) 57 | columns := make([]Column, size) 58 | noNames := make(map[int]string) 59 | for i, name := range names { 60 | // FIXME: Currently we allow duplicate names 61 | col, err := it.frame.Column(name) 62 | if err != nil { 63 | it.err = err 64 | return 65 | } 66 | 67 | if name == "" { 68 | name = fmt.Sprintf("col-%d", i) 69 | noNames[i] = name 70 | } 71 | 72 | if _, ok := nameMap[name]; ok { 73 | it.err = fmt.Errorf("duplicate column name - %q", name) 74 | return 75 | } 76 | 77 | nameMap[name] = true 78 | columns[i] = col 79 | } 80 | 81 | if it.includeIndex { 82 | for i, col := range it.frame.Indices() { 83 | name := col.Name() 84 | if name == "" { 85 | if i == 0 { 86 | name = "idx" 87 | } else { 88 | name = fmt.Sprintf("idx-%d", i) 89 | } 90 | noNames[it.numCols+i] = name 91 | } 92 | 93 | if _, ok := nameMap[name]; ok { 94 | it.err = fmt.Errorf("duplicate name in columns and indices - %q", name) 95 | return 96 | } 97 | 98 | nameMap[name] = true 99 | columns[it.numCols+i] = col 100 | } 101 | } 102 | 103 | it.noNames = noNames 104 | it.columns = columns 105 | } 106 | 107 | func (it *rowIterator) Next() bool { 108 | it.once.Do(it.init) 109 | 110 | if it.err != nil || it.rowNum >= it.frame.Len() { 111 | return false 112 | } 113 | 114 | it.row, it.err = it.getRow() 115 | if it.err != nil { 116 | return false 117 | } 118 | 119 | it.rowNum++ 120 | return true 121 | } 122 | 123 | func (it *rowIterator) Err() error { 124 | return it.err 125 | } 126 | 127 | func (it *rowIterator) Row() map[string]interface{} { 128 | if it.err != nil { 129 | return nil 130 | } 131 | 132 | return it.row 133 | } 134 | 135 | func (it *rowIterator) Indices() map[string]interface{} { 136 | return nil // TODO 137 | } 138 | 139 | func (it *rowIterator) RowNum() int { 140 | return it.rowNum - 1 141 | } 142 | 143 | func (it *rowIterator) getRow() (map[string]interface{}, error) { 144 | row := make(map[string]interface{}) 145 | for colNum, col := range it.columns { 146 | var value interface{} 147 | var err error 148 | switch col.DType() { 149 | case IntType: 150 | value, err = col.IntAt(it.rowNum) 151 | case FloatType: 152 | value, err = col.FloatAt(it.rowNum) 153 | case StringType: 154 | value, err = col.StringAt(it.rowNum) 155 | case TimeType: 156 | value, err = col.TimeAt(it.rowNum) 157 | case BoolType: 158 | value, err = col.BoolAt(it.rowNum) 159 | default: 160 | err = fmt.Errorf("%s:%d - unknown dtype - %d", col.Name(), it.rowNum, col.DType()) 161 | } 162 | 163 | if err != nil { 164 | return nil, err 165 | } 166 | 167 | name := col.Name() 168 | if name == "" { 169 | name = it.noNames[colNum] 170 | } 171 | row[name] = value 172 | } 173 | 174 | return row, nil 175 | } 176 | -------------------------------------------------------------------------------- /rowiter_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package frames 22 | 23 | import ( 24 | "fmt" 25 | "testing" 26 | "time" 27 | ) 28 | 29 | func TestRowIterator(t *testing.T) { 30 | frame, err := makeFrame() 31 | if err != nil { 32 | t.Fatalf("can't create frame - %s", err) 33 | } 34 | 35 | it := frame.IterRows(false) 36 | for rowNum := 0; it.Next(); rowNum++ { 37 | if it.RowNum() != rowNum { 38 | t.Fatalf("rowNum mismatch %d != %d", rowNum, it.RowNum()) 39 | } 40 | 41 | row := it.Row() 42 | if row == nil { 43 | t.Fatalf("empty row") 44 | } 45 | 46 | for name, val := range row { 47 | col, err := frame.Column(name) 48 | if err != nil { 49 | t.Fatalf("can't get column %q", name) 50 | } 51 | switch col.DType() { 52 | case IntType: 53 | cval, err := col.IntAt(rowNum) 54 | if err != nil { 55 | t.Fatalf("can't get int at %d", rowNum) 56 | } 57 | 58 | if cval != val { 59 | t.Fatalf("%s:%d bad value %v != %v", name, rowNum, val, cval) 60 | } 61 | case FloatType: 62 | cval, err := col.FloatAt(rowNum) 63 | if err != nil { 64 | t.Fatalf("can't get float at %d", rowNum) 65 | } 66 | 67 | if cval != val { 68 | t.Fatalf("%s:%d bad value %v != %v", name, rowNum, val, cval) 69 | } 70 | case StringType: 71 | cval, err := col.StringAt(rowNum) 72 | if err != nil { 73 | t.Fatalf("can't get string at %d", rowNum) 74 | } 75 | 76 | if cval != val { 77 | t.Fatalf("%s:%d bad value %v != %v", name, rowNum, val, cval) 78 | } 79 | case TimeType: 80 | cval, err := col.TimeAt(rowNum) 81 | if err != nil { 82 | t.Fatalf("can't get time at %d", rowNum) 83 | } 84 | 85 | if cval != val { 86 | t.Fatalf("%s:%d bad value %v != %v", name, rowNum, val, cval) 87 | } 88 | case BoolType: 89 | cval, err := col.BoolAt(rowNum) 90 | if err != nil { 91 | t.Fatalf("can't get bool at %d", rowNum) 92 | } 93 | 94 | if cval != val { 95 | t.Fatalf("%s:%d bad value %v != %v", name, rowNum, val, cval) 96 | } 97 | } 98 | } 99 | } 100 | 101 | if err := it.Err(); err != nil { 102 | t.Fatalf("iteration error - %s", err) 103 | } 104 | } 105 | 106 | func TestRowIteratorIndex(t *testing.T) { 107 | t.Skip("TODO") 108 | } 109 | 110 | func TestRowIteratorIndices(t *testing.T) { 111 | t.Skip("TODO") 112 | } 113 | 114 | func TestRowAll(t *testing.T) { 115 | frame, err := makeFrame() 116 | if err != nil { 117 | t.Fatal(err) 118 | } 119 | 120 | it := frame.IterRows(true) 121 | if ok := it.Next(); !ok { 122 | t.Fatalf("can't advance iterator") 123 | } 124 | 125 | if err := it.Err(); err != nil { 126 | t.Fatalf("error in next - %s", err) 127 | } 128 | 129 | row := it.Row() 130 | if len(row) != len(frame.Names())+len(frame.Indices()) { 131 | t.Fatalf("bad row - %+v\n", row) 132 | } 133 | } 134 | 135 | func TestNoName(t *testing.T) { 136 | col1, err := NewSliceColumn("ints", []int64{1, 2, 3}) 137 | if err != nil { 138 | t.Fatal(err) 139 | } 140 | 141 | col2, err := NewSliceColumn("", []string{"a", "b", "c"}) 142 | if err != nil { 143 | t.Fatal(err) 144 | } 145 | 146 | col3, err := NewSliceColumn("floats", []float64{1.0, 2.0, 3.0}) 147 | if err != nil { 148 | t.Fatal(err) 149 | } 150 | 151 | cols := []Column{col1, col2, col3} 152 | frame, err := NewFrame(cols, nil, nil) 153 | if err != nil { 154 | t.Fatal(err) 155 | } 156 | 157 | it := frame.IterRows(false) 158 | it.Next() 159 | if err := it.Err(); err != nil { 160 | t.Fatal(err) 161 | } 162 | 163 | row := it.Row() 164 | if len(row) != len(cols) { 165 | t.Fatalf("wrong number of columns - %d != %d", len(row), len(cols)) 166 | } 167 | 168 | for _, col := range cols { 169 | name := col.Name() 170 | if name != "" { 171 | if _, ok := row[name]; !ok { 172 | t.Fatalf("can't find column %q", col.Name()) 173 | } 174 | } 175 | } 176 | 177 | for name := range row { 178 | if name == "" { 179 | t.Fatalf("empty column name") 180 | } 181 | } 182 | } 183 | 184 | // TODO: Unite with http/end2end_test.go 185 | func makeFrame() (Frame, error) { 186 | size := 1027 187 | now := time.Now() 188 | idata := make([]int64, size) 189 | fdata := make([]float64, size) 190 | sdata := make([]string, size) 191 | tdata := make([]time.Time, size) 192 | bdata := make([]bool, size) 193 | 194 | for i := 0; i < size; i++ { 195 | idata[i] = int64(i) 196 | fdata[i] = float64(i) 197 | sdata[i] = fmt.Sprintf("val%d", i) 198 | tdata[i] = now.Add(time.Duration(i) * time.Second) 199 | bdata[i] = i%2 == 0 200 | } 201 | 202 | columns := map[string]interface{}{ 203 | "ints": idata, 204 | "floats": fdata, 205 | "strings": sdata, 206 | "times": tdata, 207 | "bools": bdata, 208 | } 209 | return NewFrameFromMap(columns, nil) 210 | } 211 | -------------------------------------------------------------------------------- /server.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package frames 22 | 23 | // ServerState is state of server 24 | type ServerState string 25 | 26 | // Possible server states 27 | const ( 28 | ReadyState ServerState = "ready" 29 | RunningState ServerState = "running" 30 | ErrorState ServerState = "error" 31 | ) 32 | 33 | // Server is frames server interface 34 | type Server interface { 35 | Start() error 36 | State() ServerState 37 | Err() error 38 | } 39 | 40 | // ServerBase have common functionality for server 41 | type ServerBase struct { 42 | err error 43 | state ServerState 44 | } 45 | 46 | // NewServerBase returns a new server base 47 | func NewServerBase() *ServerBase { 48 | return &ServerBase{ 49 | state: ReadyState, 50 | } 51 | } 52 | 53 | // Err returns the server error 54 | func (s *ServerBase) Err() error { 55 | return s.err 56 | } 57 | 58 | // SetState sets the server state 59 | func (s *ServerBase) SetState(state ServerState) { 60 | s.state = state 61 | } 62 | 63 | // State return the server state 64 | func (s *ServerBase) State() ServerState { 65 | return s.state 66 | } 67 | 68 | // SetError sets current error and will change state to ErrorState 69 | func (s *ServerBase) SetError(err error) { 70 | s.err = err 71 | s.state = ErrorState 72 | } 73 | -------------------------------------------------------------------------------- /sql.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package frames 22 | 23 | import ( 24 | "fmt" 25 | "strings" 26 | 27 | "github.com/xwb1989/sqlparser" 28 | ) 29 | 30 | // Query is query structure 31 | type Query struct { 32 | Table string 33 | Columns []string 34 | Filter string 35 | GroupBy string 36 | } 37 | 38 | // ParseSQL parsers SQL query to a Query struct 39 | func ParseSQL(sql string) (*Query, error) { 40 | stmt, err := sqlparser.Parse(sql) 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | slct, ok := stmt.(*sqlparser.Select) 46 | if !ok { 47 | return nil, fmt.Errorf("not a SELECT statement") 48 | } 49 | 50 | if nTables := len(slct.From); nTables != 1 { 51 | return nil, fmt.Errorf("nan select from only one table (got %d)", nTables) 52 | } 53 | 54 | aliased, ok := slct.From[0].(*sqlparser.AliasedTableExpr) 55 | if !ok { 56 | return nil, fmt.Errorf("not a table select") 57 | } 58 | 59 | table, ok := aliased.Expr.(sqlparser.TableName) 60 | if !ok { 61 | return nil, fmt.Errorf("not a table in FROM field") 62 | } 63 | 64 | query := &Query{ 65 | Table: table.Name.String(), 66 | } 67 | 68 | for _, sexpr := range slct.SelectExprs { 69 | switch col := sexpr.(type) { 70 | case *sqlparser.AliasedExpr: 71 | if !col.As.IsEmpty() { 72 | return nil, fmt.Errorf("SELECT ... AS ... is not supported") 73 | } 74 | 75 | colName, ok := col.Expr.(*sqlparser.ColName) 76 | if !ok { 77 | return nil, fmt.Errorf("unknown columns type - %T", col.Expr) 78 | } 79 | 80 | query.Columns = append(query.Columns, colName.Name.String()) 81 | default: 82 | return nil, fmt.Errorf("unknown SELECT column type - %T", sexpr) 83 | } 84 | } 85 | 86 | if len(query.Columns) == 0 { 87 | return nil, fmt.Errorf("no columns") 88 | } 89 | 90 | if slct.Where != nil { 91 | query.Filter = strings.TrimSpace(sqlparser.String(slct.Where)) 92 | } 93 | 94 | if slct.GroupBy != nil { 95 | // TODO: Do we want to extract just the table names? Then GroupBy will be []string 96 | query.GroupBy = strings.TrimSpace(sqlparser.String(slct.GroupBy)) 97 | } 98 | 99 | return query, nil 100 | } 101 | -------------------------------------------------------------------------------- /sql_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package frames 22 | 23 | import ( 24 | "reflect" 25 | "testing" 26 | ) 27 | 28 | func TestSimple(t *testing.T) { 29 | sql := ` 30 | SELECT first, last 31 | FROM employees 32 | WHERE last IS NOT NULL 33 | GROUP BY dept 34 | ` 35 | query, err := ParseSQL(sql) 36 | if err != nil { 37 | t.Fatalf("error parsing - %s", err) 38 | } 39 | 40 | expected := &Query{ 41 | Columns: []string{"first", "last"}, 42 | Table: "employees", 43 | Filter: "where last is not null", 44 | GroupBy: "group by dept", 45 | } 46 | 47 | if !reflect.DeepEqual(query, expected) { 48 | t.Logf("q: %q", query.GroupBy) 49 | t.Logf("e: %q", expected.GroupBy) 50 | t.Fatalf("wrong result - %+v", query) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /test/csv_integration_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "testing" 7 | "time" 8 | 9 | "github.com/nuclio/logger" 10 | "github.com/stretchr/testify/suite" 11 | "github.com/v3io/frames" 12 | "github.com/v3io/frames/pb" 13 | v3io "github.com/v3io/v3io-go/pkg/dataplane" 14 | ) 15 | 16 | var ( 17 | random = rand.New(rand.NewSource(time.Now().Unix())) 18 | size = 200 19 | ) 20 | 21 | type CsvTestSuite struct { 22 | suite.Suite 23 | tablePath string 24 | suiteTimestamp int64 25 | basicQueryTime int64 26 | client frames.Client 27 | backendName string 28 | } 29 | 30 | func GetCsvTestsConstructorFunc() SuiteCreateFunc { 31 | return func(client frames.Client, _ v3io.Container, _ logger.Logger) suite.TestingSuite { 32 | return &CsvTestSuite{client: client, backendName: "csv"} 33 | } 34 | } 35 | 36 | func (csvSuite *CsvTestSuite) SetupSuite() { 37 | csvSuite.Require().NotNil(csvSuite.client, "client not set") 38 | } 39 | 40 | func (csvSuite *CsvTestSuite) generateSampleFrame(t testing.TB) frames.Frame { 41 | var ( 42 | columns []frames.Column 43 | col frames.Column 44 | err error 45 | ) 46 | 47 | bools := make([]bool, size) 48 | for i := range bools { 49 | if random.Float64() < 0.5 { 50 | bools[i] = true 51 | } 52 | } 53 | col, err = frames.NewSliceColumn("bools", bools) 54 | csvSuite.Require().NoError(err) 55 | columns = append(columns, col) 56 | 57 | col = FloatCol(t, "floats", size) 58 | columns = append(columns, col) 59 | 60 | ints := make([]int64, size) 61 | for i := range ints { 62 | ints[i] = random.Int63() 63 | } 64 | col, err = frames.NewSliceColumn("ints", ints) 65 | csvSuite.Require().NoError(err) 66 | columns = append(columns, col) 67 | 68 | col = StringCol(t, "strings", size) 69 | columns = append(columns, col) 70 | 71 | times := make([]time.Time, size) 72 | for i := range times { 73 | times[i] = time.Now().Add(time.Duration(i) * time.Second) 74 | } 75 | col, err = frames.NewSliceColumn("times", times) 76 | csvSuite.Require().NoError(err) 77 | columns = append(columns, col) 78 | 79 | frame, err := frames.NewFrame(columns, nil, nil) 80 | csvSuite.Require().NoError(err) 81 | 82 | return frame 83 | } 84 | 85 | func (csvSuite *CsvTestSuite) TestAll() { 86 | table := fmt.Sprintf("csv_test_all%d", time.Now().UnixNano()) 87 | 88 | csvSuite.T().Log("write") 89 | frame := csvSuite.generateSampleFrame(csvSuite.T()) 90 | wreq := &frames.WriteRequest{ 91 | Backend: csvSuite.backendName, 92 | Table: table, 93 | } 94 | 95 | appender, err := csvSuite.client.Write(wreq) 96 | csvSuite.Require().NoError(err) 97 | 98 | err = appender.Add(frame) 99 | csvSuite.Require().NoError(err) 100 | 101 | err = appender.WaitForComplete(10 * time.Second) 102 | csvSuite.Require().NoError(err) 103 | 104 | time.Sleep(3 * time.Second) // Let DB sync 105 | 106 | csvSuite.T().Log("read") 107 | rreq := &pb.ReadRequest{ 108 | Backend: csvSuite.backendName, 109 | Table: table, 110 | } 111 | 112 | it, err := csvSuite.client.Read(rreq) 113 | csvSuite.Require().NoError(err) 114 | 115 | for it.Next() { 116 | // TODO: More checks 117 | fr := it.At() 118 | csvSuite.Require().Contains([]int{fr.Len(), fr.Len() - 1}, frame.Len(), "wrong length") 119 | } 120 | 121 | csvSuite.Require().NoError(it.Err()) 122 | 123 | csvSuite.T().Log("delete") 124 | dreq := &pb.DeleteRequest{ 125 | Backend: csvSuite.backendName, 126 | Table: table, 127 | } 128 | 129 | err = csvSuite.client.Delete(dreq) 130 | csvSuite.Require().NoError(err) 131 | } 132 | -------------------------------------------------------------------------------- /test/fileContentIterator_integration_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/v3io/frames" 8 | "github.com/v3io/frames/v3ioutils" 9 | v3io "github.com/v3io/v3io-go/pkg/dataplane" 10 | ) 11 | 12 | func deleteObj(path string, container v3io.Container) { 13 | _ = container.DeleteObjectSync(&v3io.DeleteObjectInput{Path: path}) 14 | } 15 | 16 | func TestFileContentIterator(t *testing.T) { 17 | container := createTestContainer(t) 18 | 19 | path := "/test_file_iterator.txt" 20 | fileSize := 1024 * 1024 * 3 21 | expected := make([]byte, fileSize) 22 | 23 | for i := range expected { 24 | expected[i] = 'a' 25 | } 26 | 27 | putObjectInput := &v3io.PutObjectInput{ 28 | Path: path, 29 | Body: []byte(expected), 30 | } 31 | 32 | err := container.PutObjectSync(putObjectInput) 33 | if err != nil { 34 | t.Fatalf("failed to put object, err: %v", err) 35 | } 36 | defer deleteObj(path, container) 37 | logger, _ := frames.NewLogger("") 38 | iter, err := v3ioutils.NewFileContentIterator(path, 2*1024*1024, container, logger) 39 | 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | 44 | var actual []byte 45 | for iter.Next() { 46 | actual = append(actual, iter.At()...) 47 | } 48 | 49 | if iter.Error() != nil { 50 | t.Fatalf("failed to iterate over file, err: %v", iter.Error()) 51 | } 52 | 53 | if string(actual) != string(expected) { 54 | t.Fatalf("actual does not match expected\n expected: %v\n actual: %v", expected, actual) 55 | } 56 | } 57 | 58 | func TestFileContentLineIterator(t *testing.T) { 59 | container := createTestContainer(t) 60 | 61 | path := "/test_file_line_iterator.txt" 62 | lineSize := 10 63 | expected := make([]string, lineSize) 64 | 65 | for i := 0; i < lineSize; i++ { 66 | expected[i] = "12345" 67 | } 68 | 69 | putObjectInput := &v3io.PutObjectInput{ 70 | Path: path, 71 | Body: []byte(strings.Join(expected, "\n") + "\n"), 72 | } 73 | 74 | err := container.PutObjectSync(putObjectInput) 75 | if err != nil { 76 | t.Fatalf("failed to put object, err: %v", err) 77 | } 78 | defer deleteObj(path, container) 79 | logger, _ := frames.NewLogger("") 80 | iter, err := v3ioutils.NewFileContentLineIterator(path, 20, container, logger) 81 | 82 | if err != nil { 83 | t.Fatal(err) 84 | } 85 | 86 | var i int 87 | for iter.Next() { 88 | if string(iter.At()) != expected[i] { 89 | t.Fatalf("actual does not match expected\n expected: %v\n actual: %v", expected[i], iter.At()) 90 | } 91 | i++ 92 | } 93 | 94 | if iter.Error() != nil { 95 | t.Fatalf("failed to iterate over file, err: %v", iter.Error()) 96 | } 97 | 98 | if i != len(expected) { 99 | t.Fatalf("nunmber of lines do not match, expected: %v, actual: %v", len(expected), i) 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /test/stream_integration_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | 8 | "github.com/nuclio/logger" 9 | "github.com/stretchr/testify/suite" 10 | "github.com/v3io/frames" 11 | "github.com/v3io/frames/pb" 12 | v3io "github.com/v3io/v3io-go/pkg/dataplane" 13 | ) 14 | 15 | type StreamTestSuite struct { 16 | suite.Suite 17 | tablePath string 18 | suiteTimestamp int64 19 | basicQueryTime int64 20 | client frames.Client 21 | backendName string 22 | v3ioContainer v3io.Container 23 | internalLogger logger.Logger 24 | } 25 | 26 | func GetStreamTestsConstructorFunc() SuiteCreateFunc { 27 | return func(client frames.Client, v3ioContainer v3io.Container, internalLogger logger.Logger) suite.TestingSuite { 28 | return &StreamTestSuite{client: client, 29 | backendName: "stream", 30 | v3ioContainer: v3ioContainer, 31 | internalLogger: internalLogger} 32 | } 33 | } 34 | 35 | func (streamSuite *StreamTestSuite) generateSampleFrame(t testing.TB) frames.Frame { 36 | size := 60 // TODO 37 | times := make([]time.Time, size) 38 | end := time.Now().Truncate(time.Hour) 39 | for i := range times { 40 | times[i] = end.Add(-time.Duration(size-i) * time.Second * 300) 41 | } 42 | 43 | index, err := frames.NewSliceColumn("idx", times) 44 | if err != nil { 45 | t.Fatal(err) 46 | } 47 | 48 | columns := []frames.Column{ 49 | FloatCol(t, "cpu", index.Len()), 50 | FloatCol(t, "mem", index.Len()), 51 | FloatCol(t, "disk", index.Len()), 52 | } 53 | 54 | frame, err := frames.NewFrame(columns, []frames.Column{index}, nil) 55 | if err != nil { 56 | t.Fatal(err) 57 | } 58 | 59 | return frame 60 | } 61 | 62 | func (streamSuite *StreamTestSuite) SetupSuite() { 63 | streamSuite.Require().NotNil(streamSuite.client, "client not set") 64 | } 65 | 66 | func (streamSuite *StreamTestSuite) TestAll() { 67 | table := fmt.Sprintf("frames_ci/stream_test_all%d", time.Now().UnixNano()) 68 | 69 | streamSuite.T().Log("create") 70 | req := &pb.CreateRequest{ 71 | Backend: streamSuite.backendName, 72 | Table: table, 73 | RetentionHours: 48, 74 | Shards: 1, 75 | } 76 | 77 | if err := streamSuite.client.Create(req); err != nil { 78 | streamSuite.T().Fatal(err) 79 | } 80 | 81 | streamSuite.T().Log("write") 82 | frame := streamSuite.generateSampleFrame(streamSuite.T()) 83 | wreq := &frames.WriteRequest{ 84 | Backend: streamSuite.backendName, 85 | Table: table, 86 | } 87 | 88 | appender, err := streamSuite.client.Write(wreq) 89 | streamSuite.Require().NoError(err) 90 | 91 | err = appender.Add(frame) 92 | streamSuite.Require().NoError(err) 93 | 94 | err = appender.WaitForComplete(3 * time.Second) 95 | streamSuite.Require().NoError(err) 96 | 97 | time.Sleep(3 * time.Second) // Let DB sync 98 | 99 | streamSuite.T().Log("read") 100 | rreq := &pb.ReadRequest{ 101 | Backend: streamSuite.backendName, 102 | Table: table, 103 | Seek: "earliest", 104 | ShardId: "0", 105 | } 106 | 107 | it, err := streamSuite.client.Read(rreq) 108 | streamSuite.Require().NoError(err) 109 | 110 | for it.Next() { 111 | // TODO: More checks 112 | fr := it.At() 113 | streamSuite.Require().Contains([]int{fr.Len(), fr.Len() - 1}, frame.Len(), "wrong length") 114 | } 115 | 116 | streamSuite.Require().NoError(it.Err()) 117 | 118 | streamSuite.T().Log("delete") 119 | dreq := &pb.DeleteRequest{ 120 | Backend: streamSuite.backendName, 121 | Table: table, 122 | } 123 | 124 | err = streamSuite.client.Delete(dreq) 125 | streamSuite.Require().NoError(err) 126 | } 127 | -------------------------------------------------------------------------------- /test/test_utils.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "reflect" 7 | "testing" 8 | "time" 9 | 10 | "github.com/nuclio/logger" 11 | "github.com/stretchr/testify/suite" 12 | "github.com/v3io/frames" 13 | "github.com/v3io/frames/pb" 14 | v3io "github.com/v3io/v3io-go/pkg/dataplane" 15 | ) 16 | 17 | type SuiteCreateFunc = func(frames.Client, v3io.Container, logger.Logger) suite.TestingSuite 18 | 19 | func FloatCol(t testing.TB, name string, size int) frames.Column { 20 | floats := make([]float64, size) 21 | for i := range floats { 22 | floats[i] = float64(i) 23 | } 24 | 25 | col, err := frames.NewSliceColumn(name, floats) 26 | if err != nil { 27 | t.Fatal(err) 28 | } 29 | 30 | return col 31 | } 32 | 33 | func IntCol(t testing.TB, name string, size int) frames.Column { 34 | random := rand.New(rand.NewSource(time.Now().UnixNano())) 35 | floats := make([]int64, size) 36 | for i := range floats { 37 | floats[i] = random.Int63() 38 | } 39 | 40 | col, err := frames.NewSliceColumn(name, floats) 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | 45 | return col 46 | } 47 | 48 | func StringCol(t testing.TB, name string, size int) frames.Column { 49 | strings := make([]string, size) 50 | for i := range strings { 51 | strings[i] = fmt.Sprintf("val-%d", i) 52 | } 53 | 54 | col, err := frames.NewSliceColumn(name, strings) 55 | if err != nil { 56 | t.Fatal(err) 57 | } 58 | return col 59 | } 60 | 61 | func BoolCol(t testing.TB, name string, size int) frames.Column { 62 | bools := make([]bool, size) 63 | for i := range bools { 64 | bools[i] = true 65 | } 66 | 67 | col, err := frames.NewSliceColumn(name, bools) 68 | if err != nil { 69 | t.Fatal(err) 70 | } 71 | return col 72 | } 73 | 74 | func TimeCol(t testing.TB, name string, size int) frames.Column { 75 | times := make([]time.Time, size) 76 | now := time.Now() 77 | for i := range times { 78 | times[i] = now.Add(time.Duration(i) * time.Hour) 79 | } 80 | 81 | col, err := frames.NewSliceColumn(name, times) 82 | if err != nil { 83 | t.Fatal(err) 84 | } 85 | return col 86 | } 87 | 88 | func initializeNullColumns(length int) []*pb.NullValuesMap { 89 | nullValues := make([]*pb.NullValuesMap, length) 90 | for i := 0; i < length; i++ { 91 | nullValues[i] = &pb.NullValuesMap{NullColumns: make(map[string]bool)} 92 | } 93 | return nullValues 94 | } 95 | 96 | func validateFramesAreEqual(s suite.Suite, frame1, frame2 frames.Frame) { 97 | // Check length 98 | s.Require().Equal(frame1.Len(), frame2.Len(), "frames length is different") 99 | 100 | // Check Indices 101 | frame1IndicesCount, frame2IndicesCount := len(frame1.Indices()), len(frame2.Indices()) 102 | s.Require().Equal(frame1IndicesCount, frame2IndicesCount, "frames indices length is different") 103 | frame1IndicesNames, frame2IndicesNames := make([]string, frame1IndicesCount), make([]string, frame2IndicesCount) 104 | for i := 0; i < frame1IndicesCount; i++ { 105 | frame1IndicesNames[i] = frame1.Indices()[i].Name() 106 | frame2IndicesNames[i] = frame2.Indices()[i].Name() 107 | } 108 | s.Require().EqualValues(frame1IndicesNames, frame2IndicesNames, "frames index names are different") 109 | 110 | // Check columns 111 | s.Require().EqualValues(frame1.Names(), frame2.Names(), "frames column names are different") 112 | frame1Data := iteratorToSlice(frame1.IterRows(true)) 113 | frame2Data := iteratorToSlice(frame2.IterRows(true)) 114 | 115 | s.Require().True(compareMapSlice(frame1Data, frame2Data), 116 | "frames values mismatch, frame1: %v \n, frame2: %v", frame1Data, frame2Data) 117 | } 118 | 119 | func iteratorToSlice(iter frames.RowIterator) []map[string]interface{} { 120 | var response []map[string]interface{} 121 | for iter.Next() { 122 | response = append(response, iter.Row()) 123 | } 124 | return response 125 | } 126 | 127 | func FrameToDataMap(frame frames.Frame) map[string]map[string]interface{} { 128 | iter := frame.IterRows(true) 129 | keyColumnName := frame.Indices()[0].Name() 130 | 131 | response := make(map[string]map[string]interface{}) 132 | for iter.Next() { 133 | currentKey := fmt.Sprintf("%v", iter.Row()[keyColumnName]) 134 | response[currentKey] = iter.Row() 135 | } 136 | 137 | return response 138 | } 139 | 140 | func compareMapSlice(a, b []map[string]interface{}) bool { 141 | if len(a) != len(b) { 142 | return false 143 | } 144 | 145 | for _, currentMapA := range a { 146 | foundMap := false 147 | for _, currentMapB := range b { 148 | if reflect.DeepEqual(currentMapA, currentMapB) { 149 | foundMap = true 150 | break 151 | } 152 | } 153 | 154 | if !foundMap { 155 | return false 156 | } 157 | } 158 | 159 | return true 160 | } 161 | -------------------------------------------------------------------------------- /types_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package frames 22 | 23 | import ( 24 | "testing" 25 | ) 26 | 27 | func TestSchemaFieldProperty(t *testing.T) { 28 | t.Skip("FIXME - PB") 29 | 30 | key, value := "yale", 42 31 | field := &SchemaField{} 32 | /* TODO 33 | field.Properties = map[string]interface{}{ 34 | key: value, 35 | } 36 | */ 37 | 38 | prop, ok := field.Property(key) 39 | if !ok { 40 | t.Fatal("property not found") 41 | } 42 | 43 | if prop != value { 44 | t.Fatalf("%q property mismatch: %v != %v", key, prop, value) 45 | } 46 | 47 | _, ok = field.Property("no such property") 48 | if ok { 49 | t.Fatal("found non existing property") 50 | } 51 | 52 | // Check with nil Properties 53 | field = &SchemaField{} 54 | _, ok = field.Property(key) 55 | if ok { 56 | t.Fatal("found non existing property (nil)") 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /v3ioutils/FileContentIterator.go: -------------------------------------------------------------------------------- 1 | package v3ioutils 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/nuclio/logger" 7 | v3io "github.com/v3io/v3io-go/pkg/dataplane" 8 | v3ioerrors "github.com/v3io/v3io-go/pkg/errors" 9 | ) 10 | 11 | const ( 12 | maxRetries = 10 13 | ) 14 | 15 | type FileContentIterator struct { 16 | container v3io.Container 17 | nextOffset int 18 | step int 19 | path string 20 | responseChan chan *v3io.Response 21 | currentData []byte 22 | err error 23 | gotLastChunk bool 24 | retries int 25 | logger logger.Logger 26 | } 27 | 28 | func NewFileContentIterator(path string, bytesStep int, container v3io.Container, logger logger.Logger) (*FileContentIterator, error) { 29 | iter := &FileContentIterator{container: container, 30 | step: bytesStep, 31 | path: path, 32 | responseChan: make(chan *v3io.Response, 1), 33 | logger: logger} 34 | 35 | input := &v3io.GetObjectInput{Path: path, NumBytes: bytesStep} 36 | _, err := container.GetObject(input, nil, iter.responseChan) 37 | 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | iter.nextOffset = bytesStep 43 | return iter, nil 44 | } 45 | 46 | func (iter *FileContentIterator) Next() bool { 47 | if iter.gotLastChunk { 48 | return false 49 | } 50 | 51 | res := <-iter.responseChan 52 | if res.Error != nil { 53 | if errWithStatusCode, ok := res.Error.(v3ioerrors.ErrorWithStatusCode); ok && 54 | iter.retries < maxRetries && 55 | (errWithStatusCode.StatusCode() >= http.StatusInternalServerError || 56 | errWithStatusCode.StatusCode() == http.StatusConflict) { 57 | iter.retries++ 58 | 59 | input := res.Request().Input.(*v3io.GetObjectInput) 60 | _, err := iter.container.GetObject(input, nil, iter.responseChan) 61 | if err != nil { 62 | iter.logger.WarnWith("got error fetching file content", 63 | "input", input, "num-retries", iter.retries, "err", res.Error) 64 | iter.err = err 65 | return false 66 | } 67 | return iter.Next() 68 | } 69 | 70 | iter.logger.WarnWith("got error fetching file content after all retries", 71 | "input", res.Request().Input.(*v3io.GetObjectInput), 72 | "num-retries", iter.retries, "err", res.Error) 73 | iter.err = res.Error 74 | return false 75 | } 76 | 77 | iter.retries = 0 78 | iter.currentData = res.Body() 79 | 80 | if res.HTTPResponse.StatusCode() == http.StatusPartialContent { 81 | 82 | input := &v3io.GetObjectInput{Path: iter.path, 83 | Offset: iter.nextOffset, 84 | NumBytes: iter.step} 85 | _, err := iter.container.GetObject(input, nil, iter.responseChan) 86 | 87 | if err != nil { 88 | iter.err = err 89 | return false 90 | } 91 | 92 | iter.nextOffset = iter.nextOffset + iter.step 93 | } else { 94 | iter.gotLastChunk = true 95 | } 96 | 97 | return true 98 | } 99 | 100 | func (iter *FileContentIterator) At() []byte { 101 | return iter.currentData 102 | } 103 | 104 | func (iter *FileContentIterator) Error() error { 105 | return iter.err 106 | } 107 | -------------------------------------------------------------------------------- /v3ioutils/FileContentLineIterator.go: -------------------------------------------------------------------------------- 1 | package v3ioutils 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | 7 | "github.com/nuclio/logger" 8 | v3io "github.com/v3io/v3io-go/pkg/dataplane" 9 | ) 10 | 11 | type FileContentLineIterator struct { 12 | fileContentIterator *FileContentIterator 13 | currentLines [][]byte 14 | currentRow int 15 | err error 16 | } 17 | 18 | func NewFileContentLineIterator(path string, bytesStep int, container v3io.Container, logger logger.Logger) (*FileContentLineIterator, error) { 19 | 20 | contentIter, err := NewFileContentIterator(path, bytesStep, container, logger) 21 | if err != nil { 22 | return nil, err 23 | } 24 | iter := &FileContentLineIterator{fileContentIterator: contentIter, currentLines: nil} 25 | return iter, nil 26 | } 27 | 28 | func (iter *FileContentLineIterator) Next() bool { 29 | if iter.currentLines == nil { 30 | if iter.fileContentIterator.Next() { 31 | iter.currentLines = bytes.Split(iter.fileContentIterator.At(), []byte{'\n'}) 32 | return true 33 | } 34 | 35 | iter.err = iter.fileContentIterator.Error() 36 | return false 37 | 38 | } 39 | 40 | if iter.currentRow == len(iter.currentLines)-2 { 41 | leftover := iter.currentLines[len(iter.currentLines)-1] // will be either a partial line or an empty string 42 | if iter.fileContentIterator.Next() { 43 | iter.currentLines = bytes.Split(iter.fileContentIterator.At(), []byte{'\n'}) 44 | iter.currentLines[0] = append(leftover, iter.currentLines[0]...) 45 | iter.currentRow = 0 46 | return true 47 | } 48 | 49 | if iter.fileContentIterator.Error() != nil { 50 | iter.err = iter.fileContentIterator.Error() 51 | } else if len(leftover) > 0 { 52 | iter.err = fmt.Errorf("got partial data in last line: %v", leftover) 53 | } 54 | 55 | return false 56 | } 57 | 58 | iter.currentRow++ 59 | return true 60 | } 61 | 62 | func (iter *FileContentLineIterator) At() []byte { 63 | return iter.currentLines[iter.currentRow] 64 | } 65 | 66 | func (iter *FileContentLineIterator) Error() error { 67 | return iter.err 68 | } 69 | -------------------------------------------------------------------------------- /v3ioutils/listDirAsyncIter.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package v3ioutils 22 | 23 | import ( 24 | v3io "github.com/v3io/v3io-go/pkg/dataplane" 25 | ) 26 | 27 | type FileCursor interface { 28 | Err() error 29 | Next() bool 30 | GetFilePath() string 31 | } 32 | 33 | type FilesCursor struct { 34 | input *v3io.GetContainerContentsInput 35 | container v3io.Container 36 | 37 | currentFile v3io.Content 38 | currentError error 39 | itemIndex int 40 | currentResponse *v3io.GetContainerContentsOutput 41 | } 42 | 43 | func NewFilesCursor(container v3io.Container, input *v3io.GetContainerContentsInput) (FileCursor, error) { 44 | 45 | newFilesIterator := &FilesCursor{ 46 | container: container, 47 | input: input, 48 | } 49 | 50 | res, err := container.GetContainerContentsSync(input) 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | res.Release() 56 | newFilesIterator.currentResponse = res.Output.(*v3io.GetContainerContentsOutput) 57 | return newFilesIterator, nil 58 | } 59 | 60 | // error returns the last error 61 | func (ic *FilesCursor) Err() error { 62 | return ic.currentError 63 | } 64 | 65 | // Next gets the next matching item. this may potentially block as this lazy loads items from the collection 66 | func (ic *FilesCursor) Next() bool { 67 | if ic.itemIndex >= len(ic.currentResponse.Contents) && ic.currentResponse.NextMarker == "" { 68 | return false 69 | } 70 | 71 | if ic.itemIndex < len(ic.currentResponse.Contents) { 72 | ic.currentFile = ic.currentResponse.Contents[ic.itemIndex] 73 | ic.itemIndex++ 74 | } else { 75 | ic.input.Marker = ic.currentResponse.NextMarker 76 | res, err := ic.container.GetContainerContentsSync(ic.input) 77 | if err != nil { 78 | ic.currentError = err 79 | return false 80 | } 81 | res.Release() 82 | ic.currentResponse = res.Output.(*v3io.GetContainerContentsOutput) 83 | ic.itemIndex = 0 84 | return ic.Next() 85 | } 86 | 87 | return true 88 | } 89 | 90 | func (ic *FilesCursor) GetFilePath() string { 91 | return ic.currentFile.Key 92 | } 93 | -------------------------------------------------------------------------------- /v3ioutils/schema_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Iguazio Systems Ltd. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License") with 5 | an addition restriction as set forth herein. You may not use this 6 | file except in compliance with the License. You may obtain a copy of 7 | the License at http://www.apache.org/licenses/LICENSE-2.0. 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | 15 | In addition, you may not use the software for any purposes that are 16 | illegal under applicable law, and the grant of the foregoing license 17 | under the Apache 2.0 license is conditioned upon your compliance with 18 | such restriction. 19 | */ 20 | 21 | package v3ioutils 22 | 23 | import ( 24 | "reflect" 25 | "testing" 26 | ) 27 | 28 | const schemaTst = ` 29 | { 30 | "fields": [ 31 | { 32 | "name": "age", 33 | "type": "long", 34 | "nullable": true 35 | }, 36 | { 37 | "name": "job", 38 | "type": "string", 39 | "nullable": true 40 | }, 41 | { 42 | "name": "marital", 43 | "type": "string", 44 | "nullable": true 45 | }, 46 | { 47 | "name": "education", 48 | "type": "string", 49 | "nullable": true 50 | }, 51 | { 52 | "name": "default", 53 | "type": "string", 54 | "nullable": true 55 | }, 56 | { 57 | "name": "balance", 58 | "type": "long", 59 | "nullable": true 60 | }, 61 | { 62 | "name": "housing", 63 | "type": "string", 64 | "nullable": true 65 | }, 66 | { 67 | "name": "loan", 68 | "type": "string", 69 | "nullable": true 70 | }, 71 | { 72 | "name": "contact", 73 | "type": "string", 74 | "nullable": true 75 | }, 76 | { 77 | "name": "day", 78 | "type": "long", 79 | "nullable": true 80 | }, 81 | { 82 | "name": "month", 83 | "type": "string", 84 | "nullable": true 85 | }, 86 | { 87 | "name": "duration", 88 | "type": "long", 89 | "nullable": true 90 | }, 91 | { 92 | "name": "campaign", 93 | "type": "long", 94 | "nullable": true 95 | }, 96 | { 97 | "name": "pdays", 98 | "type": "long", 99 | "nullable": true 100 | }, 101 | { 102 | "name": "previous", 103 | "type": "long", 104 | "nullable": true 105 | }, 106 | { 107 | "name": "poutcome", 108 | "type": "string", 109 | "nullable": true 110 | }, 111 | { 112 | "name": "y", 113 | "type": "string", 114 | "nullable": true 115 | }, 116 | { 117 | "name": "id", 118 | "type": "long", 119 | "nullable": false 120 | } 121 | ], 122 | "key": "id", 123 | "hashingBucketNum": 0 124 | } 125 | ` 126 | 127 | func TestNewSchema(t *testing.T) { 128 | schema, err := SchemaFromJSON([]byte(schemaTst)) 129 | if err != nil { 130 | t.Fatal(err) 131 | } 132 | 133 | oldSchema, ok := schema.(*OldV3ioSchema) 134 | if !ok { 135 | t.Fatalf("can't get underlying schema object") 136 | } 137 | 138 | if nFields := len(oldSchema.Fields); nFields != 18 { 139 | t.Fatalf("wrong number of fields %d != %d", nFields, 18) 140 | } 141 | } 142 | 143 | // Regression test for IG-13261 144 | func TestMerge(t *testing.T) { 145 | schema1 := OldV3ioSchema{Fields: []OldSchemaField{ 146 | {Name: "a", Type: "long"}, 147 | {Name: "b", Type: "long"}, 148 | }} 149 | schema2 := schema1 150 | res, err := schema1.merge(&schema2) 151 | if err != nil { 152 | t.Fatal(err) 153 | } 154 | if res { 155 | t.Fatal("merge with self should not report that schema was changed") 156 | } 157 | if !reflect.DeepEqual(schema1, schema2) { 158 | t.Fatal("merge with self should not cause any modifications to schema") 159 | } 160 | } 161 | --------------------------------------------------------------------------------