├── runtime.txt ├── apps ├── helpers │ ├── __init__.py │ └── cache.py ├── services │ ├── __init__.py │ └── excel.py ├── abstracts │ ├── __init__.py │ └── excel2api.py ├── routes │ ├── __init__.py │ ├── csv.py │ └── excel.py ├── __init__.py └── controllers │ ├── __init__.py │ ├── csv.py │ └── excel.py ├── Procfile ├── tests ├── __init__.py ├── controllers │ ├── __init__.py │ ├── csv.py │ └── excel.py └── test_setting.py ├── action.yml ├── requirements.txt ├── docker-compose.yml ├── Dockerfile ├── conf ├── env.yml └── log.ini ├── setting.py ├── main.py ├── .gitignore ├── LICENSE.txt ├── connection.py ├── README.md └── logs └── log.log /runtime.txt: -------------------------------------------------------------------------------- 1 | python-3.10.6 2 | -------------------------------------------------------------------------------- /apps/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | from apps.helpers.cache import cache 2 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: uvicorn main:app --host=0.0.0.0 --port=${PORT:-5000} 2 | -------------------------------------------------------------------------------- /apps/services/__init__.py: -------------------------------------------------------------------------------- 1 | from apps.services.excel import Excel as ExcelServiceApi 2 | -------------------------------------------------------------------------------- /apps/abstracts/__init__.py: -------------------------------------------------------------------------------- 1 | from apps.abstracts.excel2api import ExcelToApi as AbstractExcelToApi 2 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | from tests.controllers import * 2 | from tests.test_setting import TestSetting 3 | -------------------------------------------------------------------------------- /apps/routes/__init__.py: -------------------------------------------------------------------------------- 1 | from apps.routes.csv import router as csv_router 2 | from apps.routes.excel import router as excel_router 3 | -------------------------------------------------------------------------------- /action.yml: -------------------------------------------------------------------------------- 1 | name: Excel2api 2 | description: Convert your excel as api 3 | author: Ferdina Kusumah 4 | runs: 5 | using: python3 6 | main: main.py -------------------------------------------------------------------------------- /apps/__init__.py: -------------------------------------------------------------------------------- 1 | from apps.abstracts import * 2 | from apps.controllers import * 3 | from apps.routes import * 4 | from apps.services import * 5 | -------------------------------------------------------------------------------- /tests/controllers/__init__.py: -------------------------------------------------------------------------------- 1 | from tests.controllers.csv import TestCsvServiceApi 2 | from tests.controllers.excel import TestExcelServiceApi 3 | -------------------------------------------------------------------------------- /apps/controllers/__init__.py: -------------------------------------------------------------------------------- 1 | from apps.controllers.csv import CsvController as CsvController 2 | from apps.controllers.excel import ExcelController as ExcelController 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | gunicorn==20.1.0 2 | pandas==1.4.4 3 | uvicorn==0.18.3 4 | PyYAML==5.4 5 | mock==4.0.1 6 | fastapi==0.81.0 7 | requests==2.28.1 8 | redis==4.3.4 9 | xlrd==1.2.0 10 | orjson==3.8.0 11 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | excel2api: 5 | build: 6 | context: ./ 7 | dockerfile: Dockerfile 8 | restart: always 9 | ports: 10 | - "8000:8000" 11 | -------------------------------------------------------------------------------- /tests/test_setting.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from setting import Setting 4 | 5 | 6 | class TestSetting(unittest.TestCase): 7 | 8 | def test_load_config(self): 9 | """Unit test function list""" 10 | _setting = Setting() 11 | _conf = _setting.load_config 12 | self.assertIsNotNone(_conf) 13 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8.1-alpine3.11 2 | 3 | RUN apk add --no-cache gcc musl-dev make libffi libffi-dev 4 | 5 | WORKDIR /usr/src/app 6 | 7 | COPY . . 8 | RUN pip install --quiet --upgrade pip && \ 9 | pip install --quiet --no-cache-dir -r requirements.txt 10 | 11 | CMD ["uvicorn", "app:app", "--reload", "--port", "8000", "--host", "0.0.0.0", "--workers", "10", "--limit-concurrency", "100"] 12 | -------------------------------------------------------------------------------- /conf/env.yml: -------------------------------------------------------------------------------- 1 | env: "dev" 2 | app: 3 | name: "excel2api" 4 | log: 5 | path: "./conf/log.ini" 6 | redis: 7 | local: 8 | host: "localhost" 9 | port: 6379 10 | password: "rahasia" 11 | db_cache: 0 12 | min_ttl: 360 # seconds 13 | dev: 14 | host: "redis-13137.c9.us-east-1-4.ec2.cloud.redislabs.com" 15 | port: 13137 16 | password: "UX9o8zEhOBCzVwzAPmTzHOzgmRR013Oi" 17 | db_cache: 0 18 | min_ttl: 360 # seconds 19 | -------------------------------------------------------------------------------- /setting.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import yaml 4 | 5 | 6 | class Setting: 7 | 8 | def __init__(self, current_pah: str = os.getcwd()): 9 | self.__file_path = "{}/conf/env.yml".format(current_pah) 10 | 11 | @property 12 | def load_config(self) -> dict: 13 | """Load config from file yml to json 14 | :return: dict 15 | """ 16 | with open(self.__file_path, 'r') as stream: 17 | return yaml.safe_load(stream) 18 | -------------------------------------------------------------------------------- /apps/abstracts/excel2api.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | 4 | class ExcelToApi(ABC): 5 | 6 | @abstractmethod 7 | def fetch_excel(self, sheet_name: str = None, sheet_url: str = None) -> dict: 8 | """Fetch data excel from url 9 | :param sheet_name: str 10 | :param sheet_url: str 11 | :return: dict 12 | """ 13 | pass 14 | 15 | @abstractmethod 16 | def fetch_csv(self, file_url: str = None) -> dict: 17 | """Fetch data excel from url 18 | :param file_url: str 19 | :return: dict 20 | """ 21 | pass 22 | -------------------------------------------------------------------------------- /conf/log.ini: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys = root,default 3 | 4 | [handlers] 5 | keys = consoleHandler, fileHandler 6 | 7 | [formatters] 8 | keys = simpleFormatter 9 | 10 | [logger_root] 11 | level = DEBUG 12 | handlers = consoleHandler, fileHandler 13 | 14 | [logger_default] 15 | level = DEBUG 16 | handlers = consoleHandler, fileHandler 17 | qualname = default 18 | 19 | [handler_fileHandler] 20 | class = logging.handlers.RotatingFileHandler 21 | level = DEBUG 22 | formatter = simpleFormatter 23 | encoding = utf-8 24 | args = ('logs/log.log', 'a', 5*1024*1024, 100, None, 0) 25 | 26 | [handler_consoleHandler] 27 | class = StreamHandler 28 | level = DEBUG 29 | formatter = simpleFormatter 30 | args = (sys.stdout,) 31 | 32 | [formatter_simpleFormatter] 33 | format = %(asctime)s - %(name)s - %(levelname)s - %(message)s -------------------------------------------------------------------------------- /apps/routes/csv.py: -------------------------------------------------------------------------------- 1 | from http import HTTPStatus 2 | 3 | from fastapi import APIRouter 4 | from starlette.responses import JSONResponse 5 | 6 | from apps.controllers import CsvController 7 | 8 | router = APIRouter() 9 | 10 | 11 | @router.get("/v1/api/csv") 12 | def fetch_csv(sheet_url: str, row_range: str = None, column_range: str = None): 13 | """Fetch csv data from url 14 | :param sheet_url: str 15 | example: https://example/data/full_data.csv 16 | :param row_range: str 17 | example: - 1:2 mean (from row 1 until row before 2) 18 | - :2 or 2 mean (get all data limit 2) 19 | :param column_range: str 20 | example: - date,name,age mean (get all column only selected field) 21 | :return: dict 22 | """ 23 | _data = CsvController() 24 | _result = _data.fetch(sheet_url, row_range, column_range) 25 | return JSONResponse({'status': HTTPStatus.OK, 'data': _result}, status_code=HTTPStatus.OK) 26 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | # define core engine 2 | from fastapi import FastAPI 3 | from starlette import applications 4 | from starlette.middleware.cors import CORSMiddleware 5 | 6 | # import controller 7 | from apps.routes import excel_router, csv_router 8 | from connection import Connection 9 | 10 | app = FastAPI() 11 | app.add_middleware( 12 | CORSMiddleware, 13 | allow_origins=['*'], 14 | allow_credentials=True, 15 | allow_methods=["*"], 16 | allow_headers=["*"], 17 | ) 18 | # register routes 19 | app.include_router(excel_router) 20 | app.include_router(csv_router) 21 | 22 | 23 | @app.on_event("startup") 24 | def startup(): 25 | applications.conf = dict() 26 | # define connection 27 | conn = Connection() 28 | # get config 29 | applications.conf['config'] = conn.get_config 30 | applications.conf['cache'] = conn.cache_conn() 31 | 32 | 33 | @app.on_event("shutdown") 34 | def shutdown(): 35 | print("[Execute value when shutdown]") 36 | -------------------------------------------------------------------------------- /apps/controllers/csv.py: -------------------------------------------------------------------------------- 1 | import orjson 2 | 3 | from apps.services import ExcelServiceApi 4 | 5 | 6 | class CsvController(ExcelServiceApi): 7 | 8 | def __init__(self): 9 | ExcelServiceApi.__init__(self) 10 | 11 | def fetch(self, sheet_url: str, row_range: str, column_range: str) -> dict: 12 | """Convert all data from pandas to dictionary 13 | :param sheet_url: str 14 | :param row_range: str 15 | :param column_range: str 16 | :return: dict 17 | """ 18 | df = self.fetch_csv(sheet_url) 19 | if row_range is not None: 20 | tmp = row_range.split(':') 21 | s = int(tmp[0]) if len(tmp) > 1 and tmp[0] != '' else None 22 | e = int(tmp[1]) if len(tmp) > 1 and tmp[1] != '' else int(tmp[0]) 23 | df = df.iloc[s:e] 24 | if column_range is not None: 25 | df = df[column_range.split(',')] 26 | 27 | return orjson.loads(df.to_json(orient='records')) 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | ./conf/env.yml 3 | *__pycache__ 4 | *.DS_Store 5 | *.vscode 6 | *.log 7 | *.py[cod] 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Pycharm Extension 13 | .idea/ 14 | 15 | # Distribution / packaging 16 | .Python 17 | env/ 18 | build/ 19 | develop-eggs/ 20 | dist/ 21 | downloads/ 22 | eggs/ 23 | .eggs/ 24 | lib/ 25 | lib64/ 26 | parts/ 27 | sdist/ 28 | var/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *,cover 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Sphinx documentation 58 | docs/_build/ 59 | 60 | # PyBuilder 61 | target/ -------------------------------------------------------------------------------- /apps/services/excel.py: -------------------------------------------------------------------------------- 1 | import ssl 2 | 3 | import pandas 4 | 5 | from apps.abstracts import AbstractExcelToApi 6 | from apps.helpers import cache 7 | 8 | # unblocking checking certs https 9 | ssl._create_default_https_context = ssl._create_unverified_context 10 | 11 | 12 | class Excel(AbstractExcelToApi): 13 | 14 | def __init__(self): 15 | AbstractExcelToApi.__init__(self) 16 | 17 | @cache(ttl=360) 18 | def fetch_excel(self, sheet_name: str = None, sheet_url: str = None) -> dict: 19 | """Fetch data excel from url 20 | :param sheet_name: str 21 | :param sheet_url: str 22 | :return: dict 23 | """ 24 | return pandas.read_excel(sheet_url, sheet_name=sheet_name).to_json(orient='records') 25 | 26 | @cache(ttl=360) 27 | def fetch_csv(self, file_url: str = None) -> dict: 28 | """Fetch data excel from url 29 | :param file_url: str 30 | :return: dict 31 | """ 32 | return pandas.read_csv(file_url).to_json(orient='records') 33 | -------------------------------------------------------------------------------- /apps/controllers/excel.py: -------------------------------------------------------------------------------- 1 | import orjson 2 | 3 | from apps.services import ExcelServiceApi 4 | 5 | 6 | class ExcelController(ExcelServiceApi): 7 | 8 | def __init__(self): 9 | ExcelServiceApi.__init__(self) 10 | 11 | def fetch(self, sheet_name: str, sheet_url: str, row_range: str, column_range: str) -> dict: 12 | """Convert all data from pandas to dictionary 13 | :param sheet_name: str 14 | :param sheet_url: str 15 | :param row_range: str 16 | :param column_range: str 17 | :return: dict 18 | """ 19 | df = self.fetch_excel(sheet_name, sheet_url) 20 | if row_range is not None: 21 | tmp = row_range.split(':') 22 | s = int(tmp[0]) if len(tmp) > 1 and tmp[0] != '' else None 23 | e = int(tmp[1]) if len(tmp) > 1 and tmp[1] != '' else int(tmp[0]) 24 | df = df.iloc[s:e] 25 | if column_range is not None: 26 | df = df[column_range.split(',')] 27 | 28 | return orjson.loads(df.to_json(orient='records')) 29 | -------------------------------------------------------------------------------- /apps/routes/excel.py: -------------------------------------------------------------------------------- 1 | from http import HTTPStatus 2 | 3 | from fastapi import APIRouter 4 | from starlette.responses import JSONResponse 5 | 6 | from apps.controllers import ExcelController 7 | 8 | router = APIRouter() 9 | 10 | 11 | @router.get("/v1/api/excel") 12 | def fetch_excel(sheet_name: str, sheet_url: str, row_range: str = None, column_range: str = None): 13 | """Fetch csv data from url 14 | :param sheet_name: str 15 | example: Sheet1 16 | :param sheet_url: str 17 | example: https://example/data/full_data.xlsx 18 | :param row_range: str 19 | example: - 1:2 mean (from row 1 until row before 2) 20 | - :2 or 2 mean (get all data limit 2) 21 | :param column_range: str 22 | example: - date,name,age mean (get all column only selected field) 23 | :return: dict 24 | """ 25 | _data = ExcelController() 26 | _result = _data.fetch(sheet_name, sheet_url, row_range, column_range) 27 | return JSONResponse({'status': HTTPStatus.OK, 'data': _result}, status_code=HTTPStatus.OK) 28 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | Copyright (c) 2018 Ferdina kusumah 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | The above copyright notice and this permission notice shall be included in all 10 | copies or substantial portions of the Software. 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 13 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 14 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 15 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 16 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 17 | SOFTWARE. -------------------------------------------------------------------------------- /connection.py: -------------------------------------------------------------------------------- 1 | # thread connection 2 | import logging 3 | import logging.config 4 | from os.path import dirname, join 5 | 6 | # core database 7 | # redis connection 8 | import redis 9 | 10 | # import setting 11 | from setting import Setting 12 | 13 | 14 | class Connection: 15 | 16 | def __init__(self): 17 | self.__setting = Setting() 18 | logs_path = join(dirname(__file__), self.__setting.load_config['log']['path']) 19 | logging.config.fileConfig(fname=logs_path, disable_existing_loggers=False) 20 | # Get the logger specified in the file 21 | self.logger = logging.getLogger(__name__) 22 | 23 | def cache_conn(self) -> object: 24 | """Define cache database configuration 25 | :return: object 26 | """ 27 | _config = self.get_config 28 | # cache database 29 | return redis.StrictRedis( 30 | connection_pool=redis.ConnectionPool( 31 | host=_config['redis'][_config['env']]['host'], 32 | port=_config['redis'][_config['env']]['port'], 33 | db=_config['redis'][_config['env']]['db_cache'], 34 | password=_config['redis'][_config['env']]['password']) 35 | ) 36 | 37 | @property 38 | def get_config(self): 39 | """Load configuration from file 40 | :return: 41 | """ 42 | return self.__setting.load_config 43 | -------------------------------------------------------------------------------- /apps/helpers/cache.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import orjson 3 | from functools import wraps 4 | 5 | import pandas 6 | from starlette import applications 7 | 8 | 9 | def cache(ttl=None): 10 | def decorator(func): 11 | @wraps(func) 12 | def decorated_function(*args, **kwargs): 13 | __hash = '{}'.format(func.__name__) 14 | if args: 15 | __hash = '{}:{}'.format(func.__name__, ':'.join(str(doc) for doc in args[1:])) 16 | 17 | query_hash = hashlib.sha1(__hash.encode('utf-8')).hexdigest() 18 | 19 | def _is_cache_exists(q_hash): 20 | return not applications.conf['cache'].get(q_hash) is None 21 | 22 | def _get_cache_values(q_hash): 23 | return pandas.read_json(orjson.loads(applications.conf['cache'].get(q_hash))) 24 | 25 | def _set_cache_and_return_values(q_hash, data, time_to_live): 26 | if time_to_live is None: 27 | time_to_live = applications.conf['config']['redis'][applications.conf['config']['env']]['min_ttl'] 28 | 29 | applications.conf['cache'].set(q_hash, orjson.dumps(data), ex=time_to_live) 30 | return _get_cache_values(query_hash) 31 | 32 | if _is_cache_exists(query_hash): 33 | return _get_cache_values(query_hash) 34 | 35 | return _set_cache_and_return_values(query_hash, func(*args, **kwargs), ttl) 36 | 37 | return decorated_function 38 | 39 | return decorator 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Convert your excel and csv as API 2 | 3 | It can read from excel and csv file url 4 | 5 | Documentation 6 | ----------- 7 | Please open [https://excel2api.herokuapp.com/docs](https://excel2api.herokuapp.com/docs) 8 | 9 | Documentation explanation: 10 | * `host/v1/api/excel` 11 | * query string `sheet_name` is sheet name to open 12 | * query string `sheet_url` is url where excel will be open 13 | * query string `row_range` is range row: 14 | * `1:50` mean will display data from row 1 to 49 15 | * `:50` mean will display data from first to 49 16 | * `50` mean will display data from first to 49 17 | * query string `column_range` is range row: 18 | * `date` mean will display only column `date` 19 | * `date,age` mean will display only column `date` and `age` 20 | 21 | * `host/v1/api/csv` 22 | * query string `sheet_url` is url where csv will be open 23 | * query string `row_range` is range row: 24 | * `1:50` mean will display data from row 1 to 49 25 | * `:50` mean will display data from first to 49 26 | * `50` mean will display data from first to 49 27 | * query string `column_range` is range row: 28 | * `date` mean will display only column `date` 29 | * `date,age` mean will display only column `date` and `age` 30 | 31 | Manual Installation 32 | ------------ 33 | * Need python 3.7+ 34 | * Backed by python [fast api](https://github.com/tiangolo/fastapi) 35 | * `pip install -r requirements.txt` 36 | * Run with command `uvicorn main:app --reload --port 8000` 37 | 38 | Manual Installation 39 | ------------ 40 | * Need python 3.7+ 41 | * `pip install -r requirements.txt` 42 | * change configuration in `conf/.env.yaml` 43 | * Backed by python [fast api](https://github.com/tiangolo/fastapi) 44 | 45 | Running unit test 46 | ------------ 47 | ```shell script 48 | python -m unittest discover -v -s ./tests 49 | ``` 50 | 51 | 52 | Contribution 53 | ------------ 54 | We are always happy to have new contributions. 55 | We have `marked issues good for anyone looking to get started`, and welcome. -------------------------------------------------------------------------------- /tests/controllers/csv.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from starlette import applications 4 | from starlette.testclient import TestClient 5 | 6 | from app import app 7 | from connection import Connection 8 | # import setting 9 | from setting import Setting 10 | 11 | client = TestClient(app) 12 | 13 | 14 | class TestCsvServiceApi(unittest.TestCase): 15 | 16 | def setUp(self): 17 | _setting = Setting() 18 | applications.conf = dict() 19 | conn = Connection() 20 | # get config 21 | applications.conf['config'] = _setting.load_config 22 | applications.conf['cache'] = conn.cache_conn() 23 | 24 | def test_positive(self): 25 | """Raw test positive test""" 26 | _url = 'http://localhost:8000/v1/api/csv?sheet_url=https://covid.ourworldindata.org/data/ecdc/full_data.csv&column_range=date,location,new_cases&row_range=1:2' 27 | response = client.get(_url) 28 | self.assertEqual(response.status_code, 200) 29 | self.assertEqual(response.json()['status'], 200) 30 | 31 | def test_with_no_column_range(self): 32 | """Test with no column range""" 33 | _url = 'http://localhost:8000/v1/api/csv?sheet_url=https://covid.ourworldindata.org/data/ecdc/full_data.csv&row_range=1:2' 34 | response = client.get(_url) 35 | self.assertEqual(response.status_code, 200) 36 | self.assertEqual(response.json()['status'], 200) 37 | 38 | def test_with_no_row_range(self): 39 | """Test with no row range""" 40 | _url = 'http://localhost:8000/v1/api/csv?sheet_url=https://covid.ourworldindata.org/data/ecdc/full_data.csv&column_range=date,location,new_cases' 41 | response = client.get(_url) 42 | self.assertEqual(response.status_code, 200) 43 | self.assertEqual(response.json()['status'], 200) 44 | 45 | def test_with_no_row_and_column_range(self): 46 | """Test with no column and row range""" 47 | _url = 'http://localhost:8000/v1/api/csv?sheet_url=https://covid.ourworldindata.org/data/ecdc/full_data.csv' 48 | response = client.get(_url) 49 | self.assertEqual(response.status_code, 200) 50 | self.assertEqual(response.json()['status'], 200) 51 | 52 | def test_with_no_sheet_url(self): 53 | """Test with no sheet url""" 54 | _url = 'http://localhost:8000/v1/api/csv?column_range=date,location,new_cases&row_range=1:2' 55 | response = client.get(_url) 56 | self.assertEqual(response.status_code, 422) 57 | -------------------------------------------------------------------------------- /tests/controllers/excel.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from starlette import applications 4 | from starlette.testclient import TestClient 5 | 6 | from app import app 7 | from connection import Connection 8 | # import setting 9 | from setting import Setting 10 | 11 | client = TestClient(app) 12 | 13 | 14 | class TestExcelServiceApi(unittest.TestCase): 15 | 16 | def setUp(self): 17 | _setting = Setting() 18 | applications.conf = dict() 19 | conn = Connection() 20 | # get config 21 | applications.conf['config'] = _setting.load_config 22 | applications.conf['cache'] = conn.cache_conn() 23 | 24 | def test_positive(self): 25 | """Raw test positive test""" 26 | _url = 'http://localhost:8000/v1/api/excel?sheet_name=Sheet1&sheet_url=https://mdcune.psych.ucla.edu/modules/bioinformatics/extras/QTL_Sample_data.xls/at_download/file&column_range=ID,Point,Strain&row_range=3:5' 27 | response = client.get(_url) 28 | self.assertEqual(response.status_code, 200) 29 | self.assertEqual(response.json()['status'], 200) 30 | 31 | def test_with_no_column_range(self): 32 | """Test with no column range""" 33 | _url = 'http://localhost:8000/v1/api/excel?sheet_name=Sheet1&sheet_url=https://mdcune.psych.ucla.edu/modules/bioinformatics/extras/QTL_Sample_data.xls/at_download/file&row_range=3:5' 34 | response = client.get(_url) 35 | self.assertEqual(response.status_code, 200) 36 | self.assertEqual(response.json()['status'], 200) 37 | 38 | def test_with_no_row_range(self): 39 | """Test with no row range""" 40 | _url = 'http://localhost:8000/v1/api/excel?sheet_name=Sheet1&sheet_url=https://mdcune.psych.ucla.edu/modules/bioinformatics/extras/QTL_Sample_data.xls/at_download/file&column_range=ID,Point,Strain' 41 | response = client.get(_url) 42 | self.assertEqual(response.status_code, 200) 43 | self.assertEqual(response.json()['status'], 200) 44 | 45 | def test_with_no_row_and_column_range(self): 46 | """Test with no column and row range""" 47 | _url = 'http://localhost:8000/v1/api/excel?sheet_name=Sheet1&sheet_url=https://mdcune.psych.ucla.edu/modules/bioinformatics/extras/QTL_Sample_data.xls/at_download/file' 48 | response = client.get(_url) 49 | self.assertEqual(response.status_code, 200) 50 | self.assertEqual(response.json()['status'], 200) 51 | 52 | def test_with_no_sheet_url(self): 53 | """Test with no sheet url""" 54 | _url = 'http://localhost:8000/v1/api/excel?sheet_name=Sheet1' 55 | response = client.get(_url) 56 | self.assertEqual(response.status_code, 422) 57 | 58 | def test_with_no_sheet_name(self): 59 | """Test with no sheet name""" 60 | _url = 'http://localhost:8000/v1/api/excel?sheet_url=https://mdcune.psych.ucla.edu/modules/bioinformatics/extras/QTL_Sample_data.xls/at_download/file' 61 | response = client.get(_url) 62 | self.assertEqual(response.status_code, 422) 63 | -------------------------------------------------------------------------------- /logs/log.log: -------------------------------------------------------------------------------- 1 | 2021-06-16 22:33:52,629 - uvicorn.error - INFO - Application startup complete. 2 | 2021-06-16 22:34:50,238 - uvicorn.error - INFO - Shutting down 3 | 2021-06-16 22:34:50,342 - uvicorn.error - INFO - Finished server process [43020] 4 | 2021-06-16 22:38:11,575 - uvicorn.error - INFO - Application startup complete. 5 | 2021-06-16 22:39:47,280 - uvicorn.error - ERROR - Exception in ASGI application 6 | Traceback (most recent call last): 7 | File "/Users/ferdinakusumah/my_env/lib/python3.8/site-packages/uvicorn/protocols/http/httptools_impl.py", line 390, in run_asgi 8 | result = await app(self.scope, self.receive, self.send) 9 | File "/Users/ferdinakusumah/my_env/lib/python3.8/site-packages/uvicorn/middleware/proxy_headers.py", line 45, in __call__ 10 | return await self.app(scope, receive, send) 11 | File "/Users/ferdinakusumah/my_env/lib/python3.8/site-packages/fastapi/applications.py", line 199, in __call__ 12 | await super().__call__(scope, receive, send) 13 | File "/Users/ferdinakusumah/my_env/lib/python3.8/site-packages/starlette/applications.py", line 112, in __call__ 14 | await self.middleware_stack(scope, receive, send) 15 | File "/Users/ferdinakusumah/my_env/lib/python3.8/site-packages/starlette/middleware/errors.py", line 181, in __call__ 16 | raise exc from None 17 | File "/Users/ferdinakusumah/my_env/lib/python3.8/site-packages/starlette/middleware/errors.py", line 159, in __call__ 18 | await self.app(scope, receive, _send) 19 | File "/Users/ferdinakusumah/my_env/lib/python3.8/site-packages/starlette/middleware/cors.py", line 78, in __call__ 20 | await self.app(scope, receive, send) 21 | File "/Users/ferdinakusumah/my_env/lib/python3.8/site-packages/starlette/exceptions.py", line 82, in __call__ 22 | raise exc from None 23 | File "/Users/ferdinakusumah/my_env/lib/python3.8/site-packages/starlette/exceptions.py", line 71, in __call__ 24 | await self.app(scope, receive, sender) 25 | File "/Users/ferdinakusumah/my_env/lib/python3.8/site-packages/starlette/routing.py", line 580, in __call__ 26 | await route.handle(scope, receive, send) 27 | File "/Users/ferdinakusumah/my_env/lib/python3.8/site-packages/starlette/routing.py", line 241, in handle 28 | await self.app(scope, receive, send) 29 | File "/Users/ferdinakusumah/my_env/lib/python3.8/site-packages/starlette/routing.py", line 52, in app 30 | response = await func(request) 31 | File "/Users/ferdinakusumah/my_env/lib/python3.8/site-packages/fastapi/routing.py", line 214, in app 32 | raw_response = await run_endpoint_function( 33 | File "/Users/ferdinakusumah/my_env/lib/python3.8/site-packages/fastapi/routing.py", line 151, in run_endpoint_function 34 | return await run_in_threadpool(dependant.call, **values) 35 | File "/Users/ferdinakusumah/my_env/lib/python3.8/site-packages/starlette/concurrency.py", line 40, in run_in_threadpool 36 | return await loop.run_in_executor(None, func, *args) 37 | File "/usr/local/opt/python@3.8/Frameworks/Python.framework/Versions/3.8/lib/python3.8/concurrent/futures/thread.py", line 57, in run 38 | result = self.fn(*self.args, **self.kwargs) 39 | File "/Users/ferdinakusumah/Development/OPENSOURCE/excel2api/./apps/routes/excel.py", line 26, in fetch_excel 40 | _result = _data.fetch(sheet_name, sheet_url, row_range, column_range) 41 | File "/Users/ferdinakusumah/Development/OPENSOURCE/excel2api/./apps/controllers/excel.py", line 19, in fetch 42 | df = self.fetch_excel(sheet_name, sheet_url) 43 | File "/Users/ferdinakusumah/Development/OPENSOURCE/excel2api/./apps/helpers/cache.py", line 32, in decorated_function 44 | if _is_cache_exists(query_hash): 45 | File "/Users/ferdinakusumah/Development/OPENSOURCE/excel2api/./apps/helpers/cache.py", line 20, in _is_cache_exists 46 | return not applications.conf['cache'].get(q_hash) is None 47 | File "/Users/ferdinakusumah/my_env/lib/python3.8/site-packages/redis/client.py", line 1579, in get 48 | return self.execute_command('GET', name) 49 | File "/Users/ferdinakusumah/my_env/lib/python3.8/site-packages/redis/client.py", line 875, in execute_command 50 | conn = self.connection or pool.get_connection(command_name, **options) 51 | File "/Users/ferdinakusumah/my_env/lib/python3.8/site-packages/redis/connection.py", line 1185, in get_connection 52 | connection.connect() 53 | File "/Users/ferdinakusumah/my_env/lib/python3.8/site-packages/redis/connection.py", line 557, in connect 54 | raise ConnectionError(self._error_message(e)) 55 | redis.exceptions.ConnectionError: Error 61 connecting to redis-13137.c9.us-east-1-4.ec2.cloud.redislabs.com:13137. Connection refused. 56 | 2021-06-16 22:39:59,138 - uvicorn.error - INFO - Shutting down 57 | 2021-06-16 22:39:59,244 - uvicorn.error - INFO - Finished server process [44309] 58 | --------------------------------------------------------------------------------