├── tests ├── __init__.py ├── pricehist │ ├── __init__.py │ ├── outputs │ │ ├── __init__.py │ │ ├── test_beancount.py │ │ ├── test_ledger.py │ │ ├── test_csv.py │ │ ├── test_gnucashsql.py │ │ └── test_json.py │ ├── sources │ │ ├── test_alphavantage │ │ │ ├── digital-partial.csv │ │ │ ├── physical-partial.csv │ │ │ ├── ibm-partial.json │ │ │ ├── eur-aud-partial.json │ │ │ ├── btc-aud-partial.json │ │ │ ├── ibm-partial-adj.json │ │ │ └── search-ibm.json │ │ ├── test_coinbasepro │ │ │ ├── 2020-01-01--2020-10-16.json │ │ │ ├── 2020-10-17--2021-01-07.json │ │ │ ├── recent.json │ │ │ ├── products-partial.json │ │ │ └── currencies-partial.json │ │ ├── test_ecb │ │ │ └── eurofxref-hist-empty.xml │ │ ├── test_coindesk │ │ │ ├── supported-currencies-partial.json │ │ │ ├── recent.json │ │ │ └── all-partial.json │ │ ├── test_coinmarketcap │ │ │ ├── crypto-partial.json │ │ │ └── recent-id1-id2782.json │ │ ├── test_bankofcanada │ │ │ ├── recent.json │ │ │ └── all-partial.json │ │ ├── test_yahoo │ │ │ ├── inrx-with-null.json │ │ │ ├── tsla-recent.json │ │ │ └── ibm-long-partial.json │ │ ├── test_basesource.py │ │ ├── test_ecb.py │ │ ├── test_bankofcanada.py │ │ ├── test_coindesk.py │ │ └── test_yahoo.py │ ├── test_sources.py │ ├── test_isocurrencies.py │ ├── test_exceptions.py │ ├── test_format.py │ ├── test_logger.py │ ├── test_series.py │ ├── test_beanprice.py │ ├── test_cli.py │ └── test_fetch.py └── live.sh ├── src └── pricehist │ ├── resources │ ├── __init__.py │ └── gnucash.sql │ ├── __init__.py │ ├── beanprice │ ├── ecb.py │ ├── yahoo.py │ ├── coindesk.py │ ├── coinbasepro.py │ ├── alphavantage.py │ ├── bankofcanada.py │ ├── coinmarketcap.py │ ├── exchangeratehost.py │ └── __init__.py │ ├── price.py │ ├── outputs │ ├── baseoutput.py │ ├── __init__.py │ ├── beancount.py │ ├── json.py │ ├── csv.py │ ├── ledger.py │ └── gnucashsql.py │ ├── sources │ ├── __init__.py │ ├── ecb.py │ ├── bankofcanada.py │ ├── basesource.py │ ├── exchangeratehost.py │ ├── coindesk.py │ ├── coinbasepro.py │ └── yahoo.py │ ├── logger.py │ ├── series.py │ ├── format.py │ ├── fetch.py │ ├── exceptions.py │ └── isocurrencies.py ├── .flake8 ├── example-gnuplot.png ├── .gitignore ├── tox.ini ├── .gitlab-ci.yml ├── LICENSE ├── pyproject.toml └── Makefile /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/pricehist/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/pricehist/resources/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/pricehist/outputs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/pricehist/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "1.4.14" 2 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 88 3 | extend-ignore = E203, W503 4 | -------------------------------------------------------------------------------- /example-gnuplot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chrisberkhout/pricehist/HEAD/example-gnuplot.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .pytest_cache 2 | .vimrc 3 | dist/ 4 | .coverage 5 | htmlcov/ 6 | .tox/ 7 | __pycache__ 8 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_alphavantage/digital-partial.csv: -------------------------------------------------------------------------------- 1 | currency code,currency name 2 | BTC,Bitcoin 3 | ETH,Ethereum 4 | -------------------------------------------------------------------------------- /src/pricehist/beanprice/ecb.py: -------------------------------------------------------------------------------- 1 | from pricehist import beanprice 2 | from pricehist.sources.ecb import ECB 3 | 4 | Source = beanprice.source(ECB()) 5 | -------------------------------------------------------------------------------- /src/pricehist/beanprice/yahoo.py: -------------------------------------------------------------------------------- 1 | from pricehist import beanprice 2 | from pricehist.sources.yahoo import Yahoo 3 | 4 | Source = beanprice.source(Yahoo()) 5 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_alphavantage/physical-partial.csv: -------------------------------------------------------------------------------- 1 | currency code,currency name 2 | AUD,Australian Dollar 3 | EUR,Euro 4 | USD,United States Dollar 5 | -------------------------------------------------------------------------------- /src/pricehist/beanprice/coindesk.py: -------------------------------------------------------------------------------- 1 | from pricehist import beanprice 2 | from pricehist.sources.coindesk import CoinDesk 3 | 4 | Source = beanprice.source(CoinDesk()) 5 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | isolated_build = True 3 | envlist = py38,py39 4 | 5 | [testenv] 6 | deps = poetry 7 | commands = 8 | poetry install 9 | poetry run make test 10 | -------------------------------------------------------------------------------- /src/pricehist/beanprice/coinbasepro.py: -------------------------------------------------------------------------------- 1 | from pricehist import beanprice 2 | from pricehist.sources.coinbasepro import CoinbasePro 3 | 4 | Source = beanprice.source(CoinbasePro()) 5 | -------------------------------------------------------------------------------- /src/pricehist/beanprice/alphavantage.py: -------------------------------------------------------------------------------- 1 | from pricehist import beanprice 2 | from pricehist.sources.alphavantage import AlphaVantage 3 | 4 | Source = beanprice.source(AlphaVantage()) 5 | -------------------------------------------------------------------------------- /src/pricehist/beanprice/bankofcanada.py: -------------------------------------------------------------------------------- 1 | from pricehist import beanprice 2 | from pricehist.sources.bankofcanada import BankOfCanada 3 | 4 | Source = beanprice.source(BankOfCanada()) 5 | -------------------------------------------------------------------------------- /src/pricehist/beanprice/coinmarketcap.py: -------------------------------------------------------------------------------- 1 | from pricehist import beanprice 2 | from pricehist.sources.coinmarketcap import CoinMarketCap 3 | 4 | Source = beanprice.source(CoinMarketCap()) 5 | -------------------------------------------------------------------------------- /src/pricehist/price.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from decimal import Decimal 3 | 4 | 5 | @dataclass(frozen=True) 6 | class Price: 7 | date: str 8 | amount: Decimal 9 | -------------------------------------------------------------------------------- /src/pricehist/beanprice/exchangeratehost.py: -------------------------------------------------------------------------------- 1 | from pricehist import beanprice 2 | from pricehist.sources.exchangeratehost import ExchangeRateHost 3 | 4 | Source = beanprice.source(ExchangeRateHost()) 5 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_coinbasepro/2020-01-01--2020-10-16.json: -------------------------------------------------------------------------------- 1 | [ 2 | [ 3 | 1602806400, 4 | 9588, 5 | 9860, 6 | 9828.84, 7 | 9672.41, 8 | 1068.08144123 9 | ], 10 | [ 11 | 1577836800, 12 | 6388.91, 13 | 6471.44, 14 | 6400.02, 15 | 6410.22, 16 | 491.94797816 17 | ] 18 | ] 19 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_coinbasepro/2020-10-17--2021-01-07.json: -------------------------------------------------------------------------------- 1 | [ 2 | [ 3 | 1609977600, 4 | 29516.98, 5 | 32900, 6 | 29818.73, 7 | 32120.19, 8 | 5957.46980324 9 | ], 10 | [ 11 | 1602892800, 12 | 9630.1, 13 | 9742.61, 14 | 9675.29, 15 | 9706.33, 16 | 385.03505036 17 | ] 18 | ] 19 | -------------------------------------------------------------------------------- /src/pricehist/outputs/baseoutput.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | from pricehist.format import Format 4 | from pricehist.series import Series 5 | from pricehist.sources.basesource import BaseSource 6 | 7 | 8 | class BaseOutput(ABC): 9 | @abstractmethod 10 | def format(self, series: Series, source: BaseSource, fmt: Format) -> str: 11 | pass # pragma: nocover 12 | -------------------------------------------------------------------------------- /src/pricehist/outputs/__init__.py: -------------------------------------------------------------------------------- 1 | from .beancount import Beancount 2 | from .csv import CSV 3 | from .gnucashsql import GnuCashSQL 4 | from .json import JSON 5 | from .ledger import Ledger 6 | 7 | default = "csv" 8 | 9 | by_type = { 10 | "beancount": Beancount(), 11 | "csv": CSV(), 12 | "json": JSON(), 13 | "jsonl": JSON(jsonl=True), 14 | "gnucash-sql": GnuCashSQL(), 15 | "ledger": Ledger(), 16 | } 17 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_ecb/eurofxref-hist-empty.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | Reference rates 4 | 5 | European Central Bank 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_coindesk/supported-currencies-partial.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "currency": "AUD", 4 | "country": "Australian Dollar" 5 | }, 6 | { 7 | "currency": "BTC", 8 | "country": "Bitcoin" 9 | }, 10 | { 11 | "currency": "CUP", 12 | "country": "Cuban Peso" 13 | }, 14 | { 15 | "currency": "EUR", 16 | "country": "Euro" 17 | }, 18 | { 19 | "currency": "USD", 20 | "country": "United States Dollar" 21 | }, 22 | { 23 | "currency": "XBT", 24 | "country": "Bitcoin" 25 | } 26 | ] 27 | -------------------------------------------------------------------------------- /tests/pricehist/test_sources.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from pricehist import sources 4 | 5 | 6 | def test_formatted_includes_ecb(): 7 | lines = sources.formatted().splitlines() 8 | assert any(re.match(r"ecb +European Central Bank", line) for line in lines) 9 | 10 | 11 | def test_formatted_names_aligned(): 12 | lines = sources.formatted().splitlines() 13 | offsets = [len(re.match(r"(\w+ +)[^ ]", line)[1]) for line in lines] 14 | first = offsets[0] 15 | assert first > 1 16 | assert all(offset == first for offset in offsets) 17 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_coindesk/recent.json: -------------------------------------------------------------------------------- 1 | { 2 | "bpi": { 3 | "2021-01-01": 38204.8987, 4 | "2021-01-02": 41853.1942, 5 | "2021-01-03": 42925.6366, 6 | "2021-01-04": 41751.2249, 7 | "2021-01-05": 43890.3534, 8 | "2021-01-06": 47190.09, 9 | "2021-01-07": 50862.227 10 | }, 11 | "disclaimer": "This data was produced from the CoinDesk Bitcoin Price Index. BPI value data returned as AUD.", 12 | "time": { 13 | "updated": "Jan 8, 2021 00:03:00 UTC", 14 | "updatedISO": "2021-01-08T00:03:00+00:00" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | image: python:latest 2 | 3 | variables: 4 | PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" 5 | POETRY_CACHE_DIR: "$CI_PROJECT_DIR/.cache/poetry" 6 | 7 | cache: 8 | paths: 9 | - .cache/pip 10 | - .cache/poetry 11 | 12 | before_script: 13 | - python -V 14 | - pip install poetry 15 | - poetry install 16 | 17 | pre-commit: 18 | script: 19 | - make pre-commit 20 | 21 | test: 22 | script: 23 | - poetry run pytest 24 | 25 | test-live: 26 | script: 27 | - tests/live.sh 28 | 29 | coverage: 30 | script: 31 | - poetry run coverage run --source=pricehist -m pytest 32 | - poetry run coverage report 33 | coverage: '/^TOTAL.+?(\d+\%)$/' 34 | -------------------------------------------------------------------------------- /src/pricehist/sources/__init__.py: -------------------------------------------------------------------------------- 1 | from .alphavantage import AlphaVantage 2 | from .bankofcanada import BankOfCanada 3 | from .coinbasepro import CoinbasePro 4 | from .coindesk import CoinDesk 5 | from .coinmarketcap import CoinMarketCap 6 | from .ecb import ECB 7 | from .yahoo import Yahoo 8 | 9 | by_id = { 10 | source.id(): source 11 | for source in [ 12 | AlphaVantage(), 13 | BankOfCanada(), 14 | CoinbasePro(), 15 | CoinDesk(), 16 | CoinMarketCap(), 17 | ECB(), 18 | Yahoo(), 19 | ] 20 | } 21 | 22 | 23 | def formatted(): 24 | width = max([len(k) for k, v in by_id.items()]) 25 | lines = [k.ljust(width + 4) + v.name() for k, v in by_id.items()] 26 | return "\n".join(lines) 27 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_coindesk/all-partial.json: -------------------------------------------------------------------------------- 1 | { 2 | "bpi": { 3 | "2010-07-18": 0.0984, 4 | "2010-07-19": 0.093, 5 | "2010-07-20": 0.0851, 6 | "2010-07-21": 0.0898, 7 | "2010-07-22": 0.0567, 8 | "2010-07-23": 0.07, 9 | "2010-07-24": 0.0609, 10 | "2021-01-01": 38204.8987, 11 | "2021-01-02": 41853.1942, 12 | "2021-01-03": 42925.6366, 13 | "2021-01-04": 41751.2249, 14 | "2021-01-05": 43890.3534, 15 | "2021-01-06": 47190.09, 16 | "2021-01-07": 50862.227 17 | }, 18 | "disclaimer": "This data was produced from the CoinDesk Bitcoin Price Index. BPI value data returned as AUD.", 19 | "time": { 20 | "updated": "Jan 8, 2021 00:03:00 UTC", 21 | "updatedISO": "2021-01-08T00:03:00+00:00" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_coinmarketcap/crypto-partial.json: -------------------------------------------------------------------------------- 1 | { 2 | "status": { 3 | "timestamp": "2021-07-16T10:08:28.938Z", 4 | "error_code": 0, 5 | "error_message": null, 6 | "elapsed": 18, 7 | "credit_count": 0, 8 | "notice": null 9 | }, 10 | "data": [ 11 | { 12 | "id": 1, 13 | "name": "Bitcoin", 14 | "symbol": "BTC", 15 | "slug": "bitcoin", 16 | "rank": 1, 17 | "is_active": 1, 18 | "first_historical_data": "2013-04-28T18:47:21.000Z", 19 | "last_historical_data": "2021-07-16T09:59:03.000Z", 20 | "platform": null 21 | }, 22 | { 23 | "id": 1027, 24 | "name": "Ethereum", 25 | "symbol": "ETH", 26 | "slug": "ethereum", 27 | "rank": 2, 28 | "is_active": 1, 29 | "first_historical_data": "2015-08-07T14:49:30.000Z", 30 | "last_historical_data": "2021-07-16T09:59:04.000Z", 31 | "platform": null 32 | } 33 | ] 34 | } 35 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_coinbasepro/recent.json: -------------------------------------------------------------------------------- 1 | [ 2 | [ 3 | 1609977600, 4 | 29516.98, 5 | 32900, 6 | 29818.73, 7 | 32120.19, 8 | 5957.46980324 9 | ], 10 | [ 11 | 1609891200, 12 | 27105.01, 13 | 29949, 14 | 27655.04, 15 | 29838.52, 16 | 4227.05067035 17 | ], 18 | [ 19 | 1609804800, 20 | 24413.62, 21 | 27989, 22 | 26104.4, 23 | 27654.01, 24 | 4036.27720179 25 | ], 26 | [ 27 | 1609718400, 28 | 22055, 29 | 26199, 30 | 25624.7, 31 | 26115.94, 32 | 6304.41029978 33 | ], 34 | [ 35 | 1609632000, 36 | 24500, 37 | 27195.46, 38 | 25916.75, 39 | 25644.41, 40 | 4975.13927959 41 | ], 42 | [ 43 | 1609545600, 44 | 22000, 45 | 27000, 46 | 24071.26, 47 | 25907.35, 48 | 7291.88538639 49 | ], 50 | [ 51 | 1609459200, 52 | 23512.7, 53 | 24250, 54 | 23706.73, 55 | 24070.97, 56 | 1830.04655405 57 | ] 58 | ] 59 | -------------------------------------------------------------------------------- /src/pricehist/logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | 4 | 5 | class Formatter(logging.Formatter): 6 | def format(self, record): 7 | s = record.msg % record.args if record.args else record.msg 8 | 9 | if record.exc_info: 10 | record.exc_text = self.formatException(record.exc_info) 11 | if s[-1:] != "\n": 12 | s = s + "\n" 13 | s = s + "\n".join([f" {line}" for line in record.exc_text.splitlines()]) 14 | 15 | if record.levelno != logging.INFO: 16 | s = "\n".join([f"{record.levelname} {line}" for line in s.splitlines()]) 17 | 18 | return s 19 | 20 | 21 | def init(): 22 | handler = logging.StreamHandler(sys.stderr) 23 | handler.setFormatter(Formatter()) 24 | logging.root.addHandler(handler) 25 | logging.root.setLevel(logging.INFO) 26 | logging.getLogger("charset_normalizer").disabled = True 27 | 28 | 29 | def show_debug(): 30 | logging.root.setLevel(logging.DEBUG) 31 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_bankofcanada/recent.json: -------------------------------------------------------------------------------- 1 | { 2 | "terms": { 3 | "url": "https://www.bankofcanada.ca/terms/" 4 | }, 5 | "seriesDetail": { 6 | "FXCADUSD": { 7 | "label": "CAD/USD", 8 | "description": "Canadian dollar to US dollar daily exchange rate", 9 | "dimension": { 10 | "key": "d", 11 | "name": "date" 12 | } 13 | } 14 | }, 15 | "observations": [ 16 | { 17 | "d": "2021-01-04", 18 | "FXCADUSD": { 19 | "v": "0.7843" 20 | } 21 | }, 22 | { 23 | "d": "2021-01-05", 24 | "FXCADUSD": { 25 | "v": "0.7870" 26 | } 27 | }, 28 | { 29 | "d": "2021-01-06", 30 | "FXCADUSD": { 31 | "v": "0.7883" 32 | } 33 | }, 34 | { 35 | "d": "2021-01-07", 36 | "FXCADUSD": { 37 | "v": "0.7870" 38 | } 39 | } 40 | ] 41 | } 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Chris Berkhout 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tests/pricehist/test_isocurrencies.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from pricehist import isocurrencies 4 | 5 | 6 | def test_current(): 7 | currency = isocurrencies.by_code()["EUR"] 8 | assert currency.code == "EUR" 9 | assert currency.number == 978 10 | assert currency.minor_units == 2 11 | assert currency.name == "Euro" 12 | assert "GERMANY" in currency.countries 13 | assert "FRANCE" in currency.countries 14 | assert not currency.is_fund 15 | assert not currency.historical 16 | assert not currency.withdrawal_date 17 | 18 | 19 | def test_historical(): 20 | currency = isocurrencies.by_code()["DEM"] 21 | assert currency.code == "DEM" 22 | assert currency.number == 276 23 | assert currency.minor_units is None 24 | assert currency.name == "Deutsche Mark" 25 | assert "GERMANY" in currency.countries 26 | assert not currency.is_fund 27 | assert currency.historical 28 | assert currency.withdrawal_date == "2002-03" 29 | 30 | 31 | def test_data_dates(): 32 | assert datetime.strptime(isocurrencies.current_data_date(), "%Y-%m-%d") 33 | assert datetime.strptime(isocurrencies.historical_data_date(), "%Y-%m-%d") 34 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "pricehist" 3 | version = "1.4.14" 4 | description = "Fetch and format historical price data" 5 | authors = ["Chris Berkhout "] 6 | license = "MIT" 7 | keywords = ["historical", "prices", "plaintext", "accounting", "csv", "gnucash", "ledger", "hledger", "beancount"] 8 | readme = "README.md" 9 | homepage = "https://gitlab.com/chrisberkhout/pricehist" 10 | repository = "https://gitlab.com/chrisberkhout/pricehist" 11 | include = [ 12 | "LICENSE", 13 | "example-gnuplot.png", 14 | ] 15 | 16 | [tool.poetry.dependencies] 17 | python = "^3.8.1" 18 | requests = "^2.25.1" 19 | lxml = "^5.1.0" 20 | cssselect = "^1.1.0" 21 | curlify = "^2.2.1" 22 | 23 | [poetry.group.dev.dependencies] 24 | pytest = "^8.3.2" 25 | black = "^22.10.0" 26 | flake8 = "^7.1.0" 27 | isort = "^5.8.0" 28 | responses = "^0.13.3" 29 | coverage = "^5.5" 30 | pytest-mock = "^3.6.1" 31 | tox = "^3.24.3" 32 | 33 | [build-system] 34 | requires = ["poetry-core>=1.0.0"] 35 | build-backend = "poetry.core.masonry.api" 36 | 37 | [tool.poetry.scripts] 38 | pricehist = "pricehist.cli:cli" 39 | 40 | [tool.isort] 41 | profile = "black" 42 | multi_line_output = 3 43 | 44 | [tool.pytest.ini_options] 45 | markers = [] 46 | -------------------------------------------------------------------------------- /tests/pricehist/test_exceptions.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import pytest 4 | 5 | from pricehist import exceptions 6 | 7 | 8 | def test_handler_logs_debug_information(caplog): 9 | with caplog.at_level(logging.DEBUG): 10 | try: 11 | with exceptions.handler(): 12 | raise exceptions.RequestError("Some message") 13 | except SystemExit: 14 | pass 15 | 16 | assert caplog.records[0].levelname == "DEBUG" 17 | assert "exception encountered" in caplog.records[0].message 18 | assert caplog.records[0].exc_info 19 | 20 | 21 | def test_handler_exits_nonzero(caplog): 22 | with pytest.raises(SystemExit) as e: 23 | with exceptions.handler(): 24 | raise exceptions.RequestError("Some message") 25 | 26 | assert e.value.code == 1 27 | 28 | 29 | def test_handler_logs_critical_information(caplog): 30 | with caplog.at_level(logging.CRITICAL): 31 | try: 32 | with exceptions.handler(): 33 | raise exceptions.RequestError("Some message") 34 | except SystemExit: 35 | pass 36 | 37 | assert any( 38 | [ 39 | "CRITICAL" == r.levelname and "Some message" in r.message 40 | for r in caplog.records 41 | ] 42 | ) 43 | -------------------------------------------------------------------------------- /src/pricehist/outputs/beancount.py: -------------------------------------------------------------------------------- 1 | """ 2 | Beancount output 3 | 4 | Supports the `Beancount `_ plain text accounting 5 | format. 6 | 7 | The default output should be valid for Beancount. Customizing it via formatting 8 | options may generate invalid output, so users should keep the requirements of 9 | the Beancount format in mind. 10 | 11 | Relevant sections of the Beancount documentation: 12 | 13 | * `Commodities / Currencies 14 | `_ 15 | * `Prices `_ 16 | * `Fetching Prices in Beancount 17 | `_ 18 | 19 | Classes: 20 | 21 | Beancount 22 | 23 | """ 24 | 25 | from pricehist.format import Format 26 | 27 | from .baseoutput import BaseOutput 28 | 29 | 30 | class Beancount(BaseOutput): 31 | def format(self, series, source=None, fmt=Format()): 32 | output = "" 33 | for price in series.prices: 34 | date = fmt.format_date(price.date) 35 | base = fmt.base or series.base 36 | quote = fmt.quote or series.quote 37 | quote_amount = fmt.format_quote_amount(quote, price.amount) 38 | output += f"{date} price {base} {quote_amount}\n" 39 | return output 40 | -------------------------------------------------------------------------------- /tests/pricehist/outputs/test_beancount.py: -------------------------------------------------------------------------------- 1 | from decimal import Decimal 2 | 3 | import pytest 4 | 5 | from pricehist.format import Format 6 | from pricehist.outputs.beancount import Beancount 7 | from pricehist.price import Price 8 | from pricehist.series import Series 9 | 10 | 11 | @pytest.fixture 12 | def out(): 13 | return Beancount() 14 | 15 | 16 | @pytest.fixture 17 | def series(): 18 | prices = [ 19 | Price("2021-01-01", Decimal("24139.4648")), 20 | Price("2021-01-02", Decimal("26533.576")), 21 | Price("2021-01-03", Decimal("27001.2846")), 22 | ] 23 | return Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03", prices) 24 | 25 | 26 | def test_format_basics(out, series, mocker): 27 | source = mocker.MagicMock() 28 | result = out.format(series, source, Format()) 29 | assert result == ( 30 | "2021-01-01 price BTC 24139.4648 EUR\n" 31 | "2021-01-02 price BTC 26533.576 EUR\n" 32 | "2021-01-03 price BTC 27001.2846 EUR\n" 33 | ) 34 | 35 | 36 | def test_format_custom(out, series, mocker): 37 | source = mocker.MagicMock() 38 | fmt = Format(base="XBT", quote="EURO", thousands=".", decimal=",", datesep="/") 39 | result = out.format(series, source, fmt) 40 | assert result == ( 41 | "2021/01/01 price XBT 24.139,4648 EURO\n" 42 | "2021/01/02 price XBT 26.533,576 EURO\n" 43 | "2021/01/03 price XBT 27.001,2846 EURO\n" 44 | ) 45 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_alphavantage/ibm-partial.json: -------------------------------------------------------------------------------- 1 | { 2 | "Meta Data": { 3 | "1. Information": "Daily Time Series with Splits and Dividend Events", 4 | "2. Symbol": "IBM", 5 | "3. Last Refreshed": "2021-07-20", 6 | "4. Output Size": "Full size", 7 | "5. Time Zone": "US/Eastern" 8 | }, 9 | "Time Series (Daily)": { 10 | "2021-01-11": { 11 | "1. open": "127.95", 12 | "2. high": "129.675", 13 | "3. low": "127.66", 14 | "4. close": "128.58" 15 | }, 16 | "2021-01-08": { 17 | "1. open": "128.57", 18 | "2. high": "129.32", 19 | "3. low": "126.98", 20 | "4. close": "128.53" 21 | }, 22 | "2021-01-07": { 23 | "1. open": "130.04", 24 | "2. high": "130.46", 25 | "3. low": "128.26", 26 | "4. close": "128.99" 27 | }, 28 | "2021-01-06": { 29 | "1. open": "126.9", 30 | "2. high": "131.88", 31 | "3. low": "126.72", 32 | "4. close": "129.29" 33 | }, 34 | "2021-01-05": { 35 | "1. open": "125.01", 36 | "2. high": "126.68", 37 | "3. low": "124.61", 38 | "4. close": "126.14" 39 | }, 40 | "2021-01-04": { 41 | "1. open": "125.85", 42 | "2. high": "125.9174", 43 | "3. low": "123.04", 44 | "4. close": "123.94" 45 | }, 46 | "2020-12-31": { 47 | "1. open": "124.22", 48 | "2. high": "126.03", 49 | "3. low": "123.99", 50 | "4. close": "125.88" 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/pricehist/outputs/json.py: -------------------------------------------------------------------------------- 1 | """ 2 | JSON output 3 | 4 | Date, number and base/quote formatting options will be respected. 5 | 6 | Classes: 7 | 8 | JSON 9 | 10 | """ 11 | 12 | import io 13 | import json 14 | 15 | from pricehist.format import Format 16 | 17 | from .baseoutput import BaseOutput 18 | 19 | 20 | class JSON(BaseOutput): 21 | def __init__(self, jsonl=False): 22 | self.jsonl = jsonl 23 | 24 | def format(self, series, source, fmt=Format()): 25 | data = [] 26 | output = io.StringIO() 27 | 28 | base = fmt.base or series.base 29 | quote = fmt.quote or series.quote 30 | 31 | for price in series.prices: 32 | date = fmt.format_date(price.date) 33 | if fmt.jsonnums: 34 | amount = float(price.amount) 35 | else: 36 | amount = fmt.format_num(price.amount) 37 | 38 | data.append( 39 | { 40 | "date": date, 41 | "base": base, 42 | "quote": quote, 43 | "amount": amount, 44 | "source": source.id(), 45 | "type": series.type, 46 | } 47 | ) 48 | 49 | if self.jsonl: 50 | for row in data: 51 | json.dump(row, output, ensure_ascii=False) 52 | output.write("\n") 53 | else: 54 | json.dump(data, output, ensure_ascii=False, indent=2) 55 | output.write("\n") 56 | 57 | return output.getvalue() 58 | -------------------------------------------------------------------------------- /src/pricehist/outputs/csv.py: -------------------------------------------------------------------------------- 1 | """ 2 | CSV output 3 | 4 | Comma Separated Values output is easily processed with other command-line tools 5 | or imported into a spreadsheet or database. 6 | 7 | Python's `csv `_ module is used to 8 | produce Excel-style CSV output, except with UNIX-style line endings. The field 9 | delimiter can be set with a formatting option, and date, number and base/quote 10 | formatting options will be respected. 11 | 12 | Classes: 13 | 14 | CSV 15 | 16 | """ 17 | 18 | import csv 19 | import io 20 | 21 | from pricehist.format import Format 22 | 23 | from .baseoutput import BaseOutput 24 | 25 | 26 | class CSV(BaseOutput): 27 | def format(self, series, source, fmt=Format()): 28 | output = io.StringIO() 29 | writer = csv.writer( 30 | output, 31 | delimiter=fmt.csvdelim, 32 | lineterminator="\n", 33 | quotechar='"', 34 | doublequote=True, 35 | skipinitialspace=False, 36 | quoting=csv.QUOTE_MINIMAL, 37 | ) 38 | 39 | header = ["date", "base", "quote", "amount", "source", "type"] 40 | writer.writerow(header) 41 | 42 | base = fmt.base or series.base 43 | quote = fmt.quote or series.quote 44 | 45 | for price in series.prices: 46 | date = fmt.format_date(price.date) 47 | amount = fmt.format_num(price.amount) 48 | writer.writerow([date, base, quote, amount, source.id(), series.type]) 49 | 50 | return output.getvalue() 51 | -------------------------------------------------------------------------------- /tests/pricehist/outputs/test_ledger.py: -------------------------------------------------------------------------------- 1 | from decimal import Decimal 2 | 3 | import pytest 4 | 5 | from pricehist.format import Format 6 | from pricehist.outputs.ledger import Ledger 7 | from pricehist.price import Price 8 | from pricehist.series import Series 9 | 10 | 11 | @pytest.fixture 12 | def out(): 13 | return Ledger() 14 | 15 | 16 | @pytest.fixture 17 | def series(): 18 | prices = [ 19 | Price("2021-01-01", Decimal("24139.4648")), 20 | Price("2021-01-02", Decimal("26533.576")), 21 | Price("2021-01-03", Decimal("27001.2846")), 22 | ] 23 | return Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03", prices) 24 | 25 | 26 | def test_format_basics(out, series, mocker): 27 | source = mocker.MagicMock() 28 | result = out.format(series, source, Format()) 29 | assert result == ( 30 | "P 2021-01-01 00:00:00 BTC 24139.4648 EUR\n" 31 | "P 2021-01-02 00:00:00 BTC 26533.576 EUR\n" 32 | "P 2021-01-03 00:00:00 BTC 27001.2846 EUR\n" 33 | ) 34 | 35 | 36 | def test_format_custom(out, series, mocker): 37 | source = mocker.MagicMock() 38 | fmt = Format( 39 | base="XBT", 40 | quote="€", 41 | time="23:59:59", 42 | thousands=".", 43 | decimal=",", 44 | symbol="left", 45 | datesep="/", 46 | ) 47 | result = out.format(series, source, fmt) 48 | assert result == ( 49 | "P 2021/01/01 23:59:59 XBT €24.139,4648\n" 50 | "P 2021/01/02 23:59:59 XBT €26.533,576\n" 51 | "P 2021/01/03 23:59:59 XBT €27.001,2846\n" 52 | ) 53 | -------------------------------------------------------------------------------- /src/pricehist/outputs/ledger.py: -------------------------------------------------------------------------------- 1 | """ 2 | Ledger output 3 | 4 | Supports both `Ledger `_ and 5 | `hledger `_ plain text accounting formats. 6 | 7 | By default the output should be valid for Ledger, but can be customized for 8 | hledger or other variants via formatting options. Invalid variants are 9 | possible, so the user should be familiar with the requirements of the target 10 | format. 11 | 12 | Relevant sections of the Ledger manual: 13 | 14 | * `Commodities and Currencies 15 | `_ 16 | * `Commoditized Amounts 17 | `_ 18 | 19 | Relevant sections of the hledger manual: 20 | 21 | * `Declaring market prices `_: 22 | * `Declaring commodities .git/hooks/pre-commit 31 | chmod +x .git/hooks/pre-commit 32 | 33 | .PHONY: pre-commit 34 | pre-commit: ## Checks to run before each commit 35 | poetry run isort src tests --check 36 | poetry run black src tests --check 37 | poetry run flake8 src tests 38 | 39 | .PHONY: tox 40 | tox: ## Run tests via tox 41 | poetry run tox 42 | 43 | .PHONY: fetch-iso-data 44 | fetch-iso-data: ## Fetch the latest copy of the ISO 4217 currency data 45 | wget -O src/pricehist/resources/list-one.xml \ 46 | https://www.six-group.com/dam/download/financial-information/data-center/iso-currrency/lists/list-one.xml 47 | wget -O src/pricehist/resources/list-three.xml \ 48 | https://www.six-group.com/dam/download/financial-information/data-center/iso-currrency/lists/list-three.xml 49 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_alphavantage/eur-aud-partial.json: -------------------------------------------------------------------------------- 1 | { 2 | "Meta Data": { 3 | "1. Information": "Forex Daily Prices (open, high, low, close)", 4 | "2. From Symbol": "EUR", 5 | "3. To Symbol": "AUD", 6 | "4. Output Size": "Full size", 7 | "5. Last Refreshed": "2021-07-27 11:35:00", 8 | "6. Time Zone": "UTC" 9 | }, 10 | "Time Series FX (Daily)": { 11 | "2021-01-11": { 12 | "1. open": "1.57496", 13 | "2. high": "1.58318", 14 | "3. low": "1.57290", 15 | "4. close": "1.57823" 16 | }, 17 | "2021-01-08": { 18 | "1. open": "1.57879", 19 | "2. high": "1.58140", 20 | "3. low": "1.57177", 21 | "4. close": "1.57350" 22 | }, 23 | "2021-01-07": { 24 | "1. open": "1.57901", 25 | "2. high": "1.58650", 26 | "3. low": "1.57757", 27 | "4. close": "1.57893" 28 | }, 29 | "2021-01-06": { 30 | "1. open": "1.58390", 31 | "2. high": "1.58800", 32 | "3. low": "1.57640", 33 | "4. close": "1.57932" 34 | }, 35 | "2021-01-05": { 36 | "1. open": "1.59698", 37 | "2. high": "1.59886", 38 | "3. low": "1.58100", 39 | "4. close": "1.58389" 40 | }, 41 | "2021-01-04": { 42 | "1. open": "1.58741", 43 | "2. high": "1.60296", 44 | "3. low": "1.58550", 45 | "4. close": "1.59718" 46 | }, 47 | "2021-01-01": { 48 | "1. open": "1.58730", 49 | "2. high": "1.58730", 50 | "3. low": "1.58504", 51 | "4. close": "1.58668" 52 | }, 53 | "2020-12-31": { 54 | "1. open": "1.59946", 55 | "2. high": "1.60138", 56 | "3. low": "1.58230", 57 | "4. close": "1.58730" 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /tests/pricehist/outputs/test_csv.py: -------------------------------------------------------------------------------- 1 | from decimal import Decimal 2 | 3 | import pytest 4 | 5 | from pricehist.format import Format 6 | from pricehist.outputs.csv import CSV 7 | from pricehist.price import Price 8 | from pricehist.series import Series 9 | 10 | 11 | @pytest.fixture 12 | def out(): 13 | return CSV() 14 | 15 | 16 | @pytest.fixture 17 | def series(): 18 | prices = [ 19 | Price("2021-01-01", Decimal("24139.4648")), 20 | Price("2021-01-02", Decimal("26533.576")), 21 | Price("2021-01-03", Decimal("27001.2846")), 22 | ] 23 | return Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03", prices) 24 | 25 | 26 | def test_format_basics(out, series, mocker): 27 | source = mocker.MagicMock() 28 | source.id = mocker.MagicMock(return_value="sourceid") 29 | result = out.format(series, source, Format()) 30 | assert result == ( 31 | "date,base,quote,amount,source,type\n" 32 | "2021-01-01,BTC,EUR,24139.4648,sourceid,close\n" 33 | "2021-01-02,BTC,EUR,26533.576,sourceid,close\n" 34 | "2021-01-03,BTC,EUR,27001.2846,sourceid,close\n" 35 | ) 36 | 37 | 38 | def test_format_custom(out, series, mocker): 39 | source = mocker.MagicMock() 40 | source.id = mocker.MagicMock(return_value="sourceid") 41 | fmt = Format( 42 | base="XBT", quote="€", thousands=".", decimal=",", datesep="/", csvdelim="/" 43 | ) 44 | result = out.format(series, source, fmt) 45 | assert result == ( 46 | "date/base/quote/amount/source/type\n" 47 | '"2021/01/01"/XBT/€/24.139,4648/sourceid/close\n' 48 | '"2021/01/02"/XBT/€/26.533,576/sourceid/close\n' 49 | '"2021/01/03"/XBT/€/27.001,2846/sourceid/close\n' 50 | ) 51 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_coinbasepro/products-partial.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": "BTC-EUR", 4 | "base_currency": "BTC", 5 | "quote_currency": "EUR", 6 | "base_min_size": "0.0001", 7 | "base_max_size": "200", 8 | "quote_increment": "0.01", 9 | "base_increment": "0.00000001", 10 | "display_name": "BTC/EUR", 11 | "min_market_funds": "10", 12 | "max_market_funds": "600000", 13 | "margin_enabled": false, 14 | "fx_stablecoin": false, 15 | "post_only": false, 16 | "limit_only": false, 17 | "cancel_only": false, 18 | "trading_disabled": false, 19 | "status": "online", 20 | "status_message": "" 21 | }, 22 | { 23 | "id": "ETH-GBP", 24 | "base_currency": "ETH", 25 | "quote_currency": "GBP", 26 | "base_min_size": "0.001", 27 | "base_max_size": "1400", 28 | "quote_increment": "0.01", 29 | "base_increment": "0.00000001", 30 | "display_name": "ETH/GBP", 31 | "min_market_funds": "10", 32 | "max_market_funds": "1000000", 33 | "margin_enabled": false, 34 | "fx_stablecoin": false, 35 | "post_only": false, 36 | "limit_only": false, 37 | "cancel_only": false, 38 | "trading_disabled": false, 39 | "status": "online", 40 | "status_message": "" 41 | }, 42 | { 43 | "id": "DOGE-EUR", 44 | "base_currency": "DOGE", 45 | "quote_currency": "EUR", 46 | "base_min_size": "1", 47 | "base_max_size": "690000", 48 | "quote_increment": "0.0001", 49 | "base_increment": "0.1", 50 | "display_name": "DOGE/EUR", 51 | "min_market_funds": "5.0", 52 | "max_market_funds": "100000", 53 | "margin_enabled": false, 54 | "fx_stablecoin": false, 55 | "post_only": false, 56 | "limit_only": false, 57 | "cancel_only": false, 58 | "trading_disabled": false, 59 | "status": "online", 60 | "status_message": "" 61 | } 62 | ] 63 | -------------------------------------------------------------------------------- /tests/pricehist/test_format.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | from decimal import Decimal 3 | 4 | from pricehist.format import Format 5 | 6 | 7 | def test_fromargs(): 8 | arg_values = { 9 | "formatquote": None, 10 | "formattime": "23:59:59", 11 | "formatdecimal": None, 12 | "formatthousands": None, 13 | "formatsymbol": None, 14 | "formatdatesep": None, 15 | "formatcsvdelim": None, 16 | "formatbase": None, 17 | "formatjsonnums": None, 18 | } 19 | args = namedtuple("args", arg_values.keys())(**arg_values) 20 | fmt = Format.fromargs(args) 21 | assert fmt.time == "23:59:59" 22 | assert fmt.symbol == "rightspace" 23 | 24 | 25 | def test_format_date(): 26 | assert Format().format_date("2021-01-01") == "2021-01-01" 27 | assert Format(datesep="/").format_date("2021-01-01") == "2021/01/01" 28 | 29 | 30 | def test_format_quote_amount(): 31 | assert ( 32 | Format(decimal=",").format_quote_amount("USD", Decimal("1234.5678")) 33 | == "1234,5678 USD" 34 | ) 35 | assert ( 36 | Format(symbol="rightspace").format_quote_amount("USD", Decimal("1234.5678")) 37 | == "1234.5678 USD" 38 | ) 39 | assert ( 40 | Format(symbol="right").format_quote_amount("€", Decimal("1234.5678")) 41 | == "1234.5678€" 42 | ) 43 | assert ( 44 | Format(symbol="leftspace").format_quote_amount("£", Decimal("1234.5678")) 45 | == "£ 1234.5678" 46 | ) 47 | assert ( 48 | Format(symbol="left").format_quote_amount("$", Decimal("1234.5678")) 49 | == "$1234.5678" 50 | ) 51 | 52 | 53 | def test_format_num(): 54 | assert Format().format_num(Decimal("1234.5678")) == "1234.5678" 55 | assert ( 56 | Format(decimal=",", thousands=".").format_num(Decimal("1234.5678")) 57 | == "1.234,5678" 58 | ) 59 | -------------------------------------------------------------------------------- /tests/pricehist/test_logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | 4 | from pricehist import logger 5 | 6 | 7 | class Record: 8 | pass 9 | 10 | 11 | def test_formatter_no_prefix_for_info(): 12 | record = Record() 13 | record.levelno = logging.INFO 14 | record.levelname = "INFO" 15 | record.msg = "A message %s" 16 | record.args = "for you" 17 | record.exc_info = None 18 | record.exc_text = "" 19 | 20 | s = logger.Formatter().format(record) 21 | 22 | assert s == "A message for you" 23 | 24 | 25 | def test_formatter_prefix_for_other_levels(): 26 | record = Record() 27 | record.levelno = logging.WARNING 28 | record.levelname = "WARNING" 29 | record.msg = "A warning %s" 30 | record.args = "for you" 31 | record.exc_info = None 32 | record.exc_text = "" 33 | 34 | s = logger.Formatter().format(record) 35 | 36 | assert s == "WARNING A warning for you" 37 | 38 | 39 | def test_formatter_formats_given_exception(): 40 | 41 | try: 42 | raise Exception("Something happened") 43 | except Exception: 44 | exc_info = sys.exc_info() 45 | 46 | record = Record() 47 | record.levelno = logging.DEBUG 48 | record.levelname = "DEBUG" 49 | record.msg = "An exception %s:" 50 | record.args = "for you" 51 | record.exc_info = exc_info 52 | record.exc_text = "" 53 | 54 | s = logger.Formatter().format(record) 55 | lines = s.splitlines() 56 | 57 | assert "DEBUG An exception for you:" in lines 58 | assert "DEBUG Traceback (most recent call last):" in lines 59 | assert any('DEBUG File "' in line for line in lines) 60 | assert "DEBUG Exception: Something happened" in lines 61 | 62 | 63 | def test_init_sets_dest_formatter_and_level(capfd): 64 | logger.init() 65 | logging.info("Test message") 66 | out, err = capfd.readouterr() 67 | assert "Test message" not in out 68 | assert "Test message" in err.splitlines() 69 | assert logging.root.level == logging.INFO 70 | 71 | 72 | def test_show_debug(): 73 | logger.show_debug() 74 | assert logging.root.level == logging.DEBUG 75 | -------------------------------------------------------------------------------- /src/pricehist/format.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | 4 | @dataclass(frozen=True) 5 | class Format: 6 | base: str = None 7 | quote: str = None 8 | time: str = "00:00:00" 9 | decimal: str = "." 10 | thousands: str = "" 11 | symbol: str = "rightspace" 12 | datesep: str = "-" 13 | csvdelim: str = "," 14 | jsonnums: bool = False 15 | 16 | @classmethod 17 | def fromargs(cls, args): 18 | def if_not_none(value, default): 19 | return default if value is None else value 20 | 21 | default = cls() 22 | return cls( 23 | base=if_not_none(args.formatbase, default.base), 24 | quote=if_not_none(args.formatquote, default.quote), 25 | time=if_not_none(args.formattime, default.time), 26 | decimal=if_not_none(args.formatdecimal, default.decimal), 27 | thousands=if_not_none(args.formatthousands, default.thousands), 28 | symbol=if_not_none(args.formatsymbol, default.symbol), 29 | datesep=if_not_none(args.formatdatesep, default.datesep), 30 | csvdelim=if_not_none(args.formatcsvdelim, default.csvdelim), 31 | jsonnums=if_not_none(args.formatjsonnums, default.jsonnums), 32 | ) 33 | 34 | def format_date(self, date): 35 | return str(date).replace("-", self.datesep) 36 | 37 | def format_quote_amount(self, quote, amount): 38 | formatted_amount = self.format_num(amount) 39 | 40 | if self.symbol == "left": 41 | qa_parts = [quote, formatted_amount] 42 | elif self.symbol == "leftspace": 43 | qa_parts = [quote, " ", formatted_amount] 44 | elif self.symbol == "right": 45 | qa_parts = [formatted_amount, quote] 46 | else: 47 | qa_parts = [formatted_amount, " ", quote] 48 | 49 | quote_amount = "".join(qa_parts) 50 | 51 | return quote_amount 52 | 53 | def format_num(self, num): 54 | parts = f"{num:,}".split(".") 55 | parts[0] = parts[0].replace(",", self.thousands) 56 | result = self.decimal.join(parts) 57 | return result 58 | -------------------------------------------------------------------------------- /src/pricehist/resources/gnucash.sql: -------------------------------------------------------------------------------- 1 | -- Created by pricehist {version} at {timestamp} 2 | 3 | BEGIN; 4 | 5 | -- The GnuCash database must already have entries for the relevant commodities. 6 | -- These statements fail and later changes are skipped if that isn't the case. 7 | CREATE TEMPORARY TABLE guids (mnemonic TEXT NOT NULL, guid TEXT NOT NULL); 8 | INSERT INTO guids VALUES ({base}, (SELECT guid FROM commodities WHERE mnemonic = {base} LIMIT 1)); 9 | INSERT INTO guids VALUES ({quote}, (SELECT guid FROM commodities WHERE mnemonic = {quote} LIMIT 1)); 10 | 11 | -- Create a staging table for the new price data. 12 | -- Doing this via a SELECT ensures the correct date type across databases. 13 | CREATE TEMPORARY TABLE new_prices AS 14 | SELECT p.guid, p.date, c.mnemonic AS base, c.mnemonic AS quote, p.source, p.type, p.value_num, p.value_denom 15 | FROM prices p, commodities c 16 | WHERE FALSE; 17 | 18 | -- Populate the staging table. 19 | {values_comment}INSERT INTO new_prices (guid, date, base, quote, source, type, value_num, value_denom) VALUES 20 | {values_comment}{values} 21 | {values_comment}; 22 | 23 | -- Get some numbers for the summary. 24 | CREATE TEMPORARY TABLE summary (description TEXT, num INT); 25 | INSERT INTO summary VALUES ('staged rows', (SELECT COUNT(*) FROM new_prices)); 26 | INSERT INTO summary VALUES ('pre-existing rows', (SELECT COUNT(*) FROM new_prices tp, prices p where p.guid = tp.guid)); 27 | INSERT INTO summary VALUES ('additional rows', (SELECT COUNT(*) FROM new_prices WHERE guid NOT IN (SELECT guid FROM prices))); 28 | 29 | -- Insert the new prices into the prices table, unless they're already there. 30 | INSERT INTO prices (guid, commodity_guid, currency_guid, date, source, type, value_num, value_denom) 31 | SELECT tp.guid, g1.guid, g2.guid, tp.date, tp.source, tp.type, tp.value_num, tp.value_denom 32 | FROM new_prices tp, guids g1, guids g2 33 | WHERE tp.base = g1.mnemonic 34 | AND tp.quote = g2.mnemonic 35 | AND tp.guid NOT IN (SELECT guid FROM prices) 36 | ; 37 | 38 | -- Show the final relevant rows of the main prices table 39 | SELECT 'final' AS status, p.* FROM prices p WHERE p.guid IN (SELECT guid FROM new_prices) ORDER BY p.date; 40 | 41 | -- Show the summary. 42 | SELECT * FROM summary; 43 | 44 | COMMIT; 45 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_alphavantage/btc-aud-partial.json: -------------------------------------------------------------------------------- 1 | { 2 | "Meta Data": { 3 | "1. Information": "Daily Prices and Volumes for Digital Currency", 4 | "2. Digital Currency Code": "BTC", 5 | "3. Digital Currency Name": "Bitcoin", 6 | "4. Market Code": "AUD", 7 | "5. Market Name": "Australian Dollar", 8 | "6. Last Refreshed": "2021-07-28 00:00:00", 9 | "7. Time Zone": "UTC" 10 | }, 11 | "Time Series (Digital Currency Daily)": { 12 | "2021-01-09": { 13 | "1. open": "55074.06950240", 14 | "2. high": "56150.17720000", 15 | "3. low": "52540.71680000", 16 | "4. close": "54397.30924680", 17 | "5. volume": "75785.97967500" 18 | }, 19 | "2021-01-08": { 20 | "1. open": "53507.50941120", 21 | "2. high": "56923.63300000", 22 | "3. low": "49528.31000000", 23 | "4. close": "55068.43820140", 24 | "5. volume": "139789.95749900" 25 | }, 26 | "2021-01-07": { 27 | "1. open": "49893.81535840", 28 | "2. high": "54772.88310000", 29 | "3. low": "49256.92200000", 30 | "4. close": "53507.23802320", 31 | "5. volume": "132825.70043700" 32 | }, 33 | "2021-01-06": { 34 | "1. open": "46067.47523820", 35 | "2. high": "50124.29161740", 36 | "3. low": "45169.81872000", 37 | "4. close": "49893.81535840", 38 | "5. volume": "127139.20131000" 39 | }, 40 | "2021-01-05": { 41 | "1. open": "43408.17136500", 42 | "2. high": "46624.45840000", 43 | "3. low": "40572.50600000", 44 | "4. close": "46067.47523820", 45 | "5. volume": "116049.99703800" 46 | }, 47 | "2021-01-04": { 48 | "1. open": "44779.08784700", 49 | "2. high": "45593.18400000", 50 | "3. low": "38170.72220000", 51 | "4. close": "43406.76014740", 52 | "5. volume": "140899.88569000" 53 | }, 54 | "2021-01-03": { 55 | "1. open": "43661.51206300", 56 | "2. high": "47191.80858340", 57 | "3. low": "43371.85965060", 58 | "4. close": "44779.08784700", 59 | "5. volume": "120957.56675000" 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_alphavantage/ibm-partial-adj.json: -------------------------------------------------------------------------------- 1 | { 2 | "Meta Data": { 3 | "1. Information": "Daily Time Series with Splits and Dividend Events", 4 | "2. Symbol": "IBM", 5 | "3. Last Refreshed": "2021-07-20", 6 | "4. Output Size": "Full size", 7 | "5. Time Zone": "US/Eastern" 8 | }, 9 | "Time Series (Daily)": { 10 | "2021-01-11": { 11 | "1. open": "127.95", 12 | "2. high": "129.675", 13 | "3. low": "127.66", 14 | "4. close": "128.58", 15 | "5. adjusted close": "125.471469081", 16 | "6. volume": "5602466", 17 | "7. dividend amount": "0.0000", 18 | "8. split coefficient": "1.0" 19 | }, 20 | "2021-01-08": { 21 | "1. open": "128.57", 22 | "2. high": "129.32", 23 | "3. low": "126.98", 24 | "4. close": "128.53", 25 | "5. adjusted close": "125.422677873", 26 | "6. volume": "4676487", 27 | "7. dividend amount": "0.0000", 28 | "8. split coefficient": "1.0" 29 | }, 30 | "2021-01-07": { 31 | "1. open": "130.04", 32 | "2. high": "130.46", 33 | "3. low": "128.26", 34 | "4. close": "128.99", 35 | "5. adjusted close": "125.871556982", 36 | "6. volume": "4507382", 37 | "7. dividend amount": "0.0000", 38 | "8. split coefficient": "1.0" 39 | }, 40 | "2021-01-06": { 41 | "1. open": "126.9", 42 | "2. high": "131.88", 43 | "3. low": "126.72", 44 | "4. close": "129.29", 45 | "5. adjusted close": "126.164304226", 46 | "6. volume": "7956740", 47 | "7. dividend amount": "0.0000", 48 | "8. split coefficient": "1.0" 49 | }, 50 | "2021-01-05": { 51 | "1. open": "125.01", 52 | "2. high": "126.68", 53 | "3. low": "124.61", 54 | "4. close": "126.14", 55 | "5. adjusted close": "123.090458157", 56 | "6. volume": "6114619", 57 | "7. dividend amount": "0.0000", 58 | "8. split coefficient": "1.0" 59 | }, 60 | "2021-01-04": { 61 | "1. open": "125.85", 62 | "2. high": "125.9174", 63 | "3. low": "123.04", 64 | "4. close": "123.94", 65 | "5. adjusted close": "120.943645029", 66 | "6. volume": "5179161", 67 | "7. dividend amount": "0.0000", 68 | "8. split coefficient": "1.0" 69 | }, 70 | "2020-12-31": { 71 | "1. open": "124.22", 72 | "2. high": "126.03", 73 | "3. low": "123.99", 74 | "4. close": "125.88", 75 | "5. adjusted close": "122.836743878", 76 | "6. volume": "3574696", 77 | "7. dividend amount": "0.0000", 78 | "8. split coefficient": "1.0" 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_bankofcanada/all-partial.json: -------------------------------------------------------------------------------- 1 | { 2 | "terms": { 3 | "url": "https://www.bankofcanada.ca/terms/" 4 | }, 5 | "seriesDetail": { 6 | "FXCADUSD": { 7 | "label": "CAD/USD", 8 | "description": "Canadian dollar to US dollar daily exchange rate", 9 | "dimension": { 10 | "key": "d", 11 | "name": "date" 12 | } 13 | } 14 | }, 15 | "observations": [ 16 | { 17 | "d": "2017-01-03", 18 | "FXCADUSD": { 19 | "v": "0.7443" 20 | } 21 | }, 22 | { 23 | "d": "2017-01-04", 24 | "FXCADUSD": { 25 | "v": "0.7510" 26 | } 27 | }, 28 | { 29 | "d": "2017-01-05", 30 | "FXCADUSD": { 31 | "v": "0.7551" 32 | } 33 | }, 34 | { 35 | "d": "2017-01-06", 36 | "FXCADUSD": { 37 | "v": "0.7568" 38 | } 39 | }, 40 | { 41 | "d": "2017-01-09", 42 | "FXCADUSD": { 43 | "v": "0.7553" 44 | } 45 | }, 46 | { 47 | "d": "2017-01-10", 48 | "FXCADUSD": { 49 | "v": "0.7568" 50 | } 51 | }, 52 | { 53 | "d": "2017-01-11", 54 | "FXCADUSD": { 55 | "v": "0.7547" 56 | } 57 | }, 58 | { 59 | "d": "2020-12-29", 60 | "FXCADUSD": { 61 | "v": "0.7809" 62 | } 63 | }, 64 | { 65 | "d": "2020-12-30", 66 | "FXCADUSD": { 67 | "v": "0.7831" 68 | } 69 | }, 70 | { 71 | "d": "2020-12-31", 72 | "FXCADUSD": { 73 | "v": "0.7854" 74 | } 75 | }, 76 | { 77 | "d": "2021-01-04", 78 | "FXCADUSD": { 79 | "v": "0.7843" 80 | } 81 | }, 82 | { 83 | "d": "2021-01-05", 84 | "FXCADUSD": { 85 | "v": "0.7870" 86 | } 87 | }, 88 | { 89 | "d": "2021-01-06", 90 | "FXCADUSD": { 91 | "v": "0.7883" 92 | } 93 | }, 94 | { 95 | "d": "2021-01-07", 96 | "FXCADUSD": { 97 | "v": "0.7870" 98 | } 99 | } 100 | ] 101 | } 102 | -------------------------------------------------------------------------------- /src/pricehist/beanprice/__init__.py: -------------------------------------------------------------------------------- 1 | import re 2 | from datetime import date, datetime, timedelta, timezone 3 | from decimal import Decimal 4 | from typing import List, NamedTuple, Optional 5 | 6 | from pricehist import exceptions 7 | from pricehist.series import Series 8 | 9 | SourcePrice = NamedTuple( 10 | "SourcePrice", 11 | [ 12 | ("price", Decimal), 13 | ("time", Optional[datetime]), 14 | ("quote_currency", Optional[str]), 15 | ], 16 | ) 17 | 18 | 19 | def source(pricehist_source): 20 | class Source: 21 | def get_latest_price(self, ticker: str) -> Optional[SourcePrice]: 22 | time_end = datetime.combine(date.today(), datetime.min.time()) 23 | time_begin = time_end - timedelta(days=7) 24 | prices = self.get_prices_series(ticker, time_begin, time_end) 25 | if prices: 26 | return prices[-1] 27 | else: 28 | return None 29 | 30 | def get_historical_price( 31 | self, ticker: str, time: datetime 32 | ) -> Optional[SourcePrice]: 33 | prices = self.get_prices_series(ticker, time, time) 34 | if prices: 35 | return prices[-1] 36 | else: 37 | return None 38 | 39 | def get_prices_series( 40 | self, 41 | ticker: str, 42 | time_begin: datetime, 43 | time_end: datetime, 44 | ) -> Optional[List[SourcePrice]]: 45 | base, quote, type = self._decode(ticker) 46 | 47 | start = time_begin.date().isoformat() 48 | end = time_end.date().isoformat() 49 | 50 | local_tz = datetime.now(timezone.utc).astimezone().tzinfo 51 | user_tz = time_begin.tzinfo or local_tz 52 | 53 | try: 54 | series = pricehist_source.fetch(Series(base, quote, type, start, end)) 55 | except exceptions.SourceError: 56 | return None 57 | 58 | return [ 59 | SourcePrice( 60 | price.amount, 61 | datetime.fromisoformat(price.date).replace(tzinfo=user_tz), 62 | series.quote, 63 | ) 64 | for price in series.prices 65 | ] 66 | 67 | def _decode(self, ticker): 68 | # https://github.com/beancount/beanprice/blob/b05203/beanprice/price.py#L166 69 | parts = [ 70 | re.sub(r"_[0-9a-fA-F]{2}", lambda m: chr(int(m.group(0)[1:], 16)), part) 71 | for part in ticker.split(":") 72 | ] 73 | base, quote, candidate_type = (parts + [""] * 3)[0:3] 74 | type = candidate_type or pricehist_source.types()[0] 75 | return (base, quote, type) 76 | 77 | return Source 78 | -------------------------------------------------------------------------------- /tests/pricehist/test_series.py: -------------------------------------------------------------------------------- 1 | from dataclasses import replace 2 | from decimal import Decimal 3 | 4 | import pytest 5 | 6 | from pricehist.price import Price 7 | from pricehist.series import Series 8 | 9 | 10 | @pytest.fixture 11 | def series(): 12 | return Series( 13 | "BASE", 14 | "QUOTE", 15 | "type", 16 | "2021-01-01", 17 | "2021-06-30", 18 | [ 19 | Price("2021-01-01", Decimal("1.0123456789")), 20 | Price("2021-01-02", Decimal("2.01234567890123456789")), 21 | Price("2021-01-03", Decimal("3.012345678901234567890123456789")), 22 | ], 23 | ) 24 | 25 | 26 | def test_invert(series): 27 | result = series.invert() 28 | assert (series.base, series.quote) == ("BASE", "QUOTE") 29 | assert (result.base, result.quote) == ("QUOTE", "BASE") 30 | 31 | 32 | def test_rename_base(series): 33 | result = series.rename_base("NEWBASE") 34 | assert series.base == "BASE" 35 | assert result.base == "NEWBASE" 36 | 37 | 38 | def test_rename_quote(series): 39 | result = series.rename_quote("NEWQUOTE") 40 | assert series.quote == "QUOTE" 41 | assert result.quote == "NEWQUOTE" 42 | 43 | 44 | def test_quantize_rounds_half_even(series): 45 | subject = replace( 46 | series, 47 | prices=[ 48 | Price("2021-01-01", Decimal("1.14")), 49 | Price("2021-01-02", Decimal("2.25")), 50 | Price("2021-01-03", Decimal("3.35")), 51 | Price("2021-01-04", Decimal("4.46")), 52 | ], 53 | ) 54 | amounts = [p.amount for p in subject.quantize(1).prices] 55 | assert amounts == [ 56 | Decimal("1.1"), 57 | Decimal("2.2"), 58 | Decimal("3.4"), 59 | Decimal("4.5"), 60 | ] 61 | 62 | 63 | def test_quantize_does_not_extend(series): 64 | subject = replace( 65 | series, 66 | prices=[ 67 | Price("2021-01-01", Decimal("1.14")), 68 | Price("2021-01-02", Decimal("2.25")), 69 | Price("2021-01-03", Decimal("3.35")), 70 | Price("2021-01-04", Decimal("4.46")), 71 | ], 72 | ) 73 | amounts = [p.amount for p in subject.quantize(3).prices] 74 | assert amounts == [ 75 | Decimal("1.14"), 76 | Decimal("2.25"), 77 | Decimal("3.35"), 78 | Decimal("4.46"), 79 | ] 80 | 81 | 82 | def test_quantize_does_not_go_beyond_context_max_prec(series): 83 | subject = replace( 84 | series, 85 | prices=[ 86 | Price("2021-01-01", Decimal("1.012345678901234567890123456789")), 87 | ], 88 | ) 89 | assert subject.prices[0].amount == Decimal("1.012345678901234567890123456789") 90 | result0 = subject.quantize(26) 91 | result1 = subject.quantize(27) 92 | result2 = subject.quantize(35) 93 | assert result0.prices[0].amount == Decimal("1.01234567890123456789012346") 94 | assert result1.prices[0].amount == Decimal("1.012345678901234567890123457") 95 | assert result2.prices[0].amount == Decimal("1.012345678901234567890123457") 96 | -------------------------------------------------------------------------------- /src/pricehist/fetch.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from datetime import date, datetime, timedelta 3 | 4 | from pricehist import exceptions 5 | 6 | 7 | def fetch(series, source, output, invert: bool, quantize: int, fmt) -> str: 8 | if series.start < source.start(): 9 | logging.warning( 10 | f"The start date {series.start} preceeds the {source.name()} " 11 | f"source start date of {source.start()}." 12 | ) 13 | 14 | with exceptions.handler(): 15 | series = source.fetch(series) 16 | 17 | if len(series.prices) == 0: 18 | logging.warning( 19 | f"No data found for the interval [{series.start}--{series.end}]." 20 | ) 21 | else: 22 | first = series.prices[0].date 23 | last = series.prices[-1].date 24 | message = ( 25 | f"Available data covers the interval [{first}--{last}], " 26 | f"{_cov_description(series.start, series.end, first, last)}." 27 | ) 28 | if first > series.start or last < series.end: 29 | expected_end = _yesterday() if series.end == _today() else series.end 30 | if first == series.start and last == expected_end: 31 | logging.debug(message) # Missing today's price is expected 32 | else: 33 | logging.warning(message) 34 | else: 35 | logging.debug(message) 36 | 37 | if invert: 38 | series = series.invert() 39 | if quantize is not None: 40 | series = series.quantize(quantize) 41 | 42 | return output.format(series, source, fmt=fmt) 43 | 44 | 45 | def _today(): 46 | return date.today().isoformat() 47 | 48 | 49 | def _yesterday(): 50 | return (date.today() - timedelta(days=1)).isoformat() 51 | 52 | 53 | def _cov_description( 54 | requested_start: str, requested_end: str, actual_start: str, actual_end: str 55 | ) -> str: 56 | date_format = "%Y-%m-%d" 57 | r1 = datetime.strptime(requested_start, date_format).date() 58 | r2 = datetime.strptime(requested_end, date_format).date() 59 | a1 = datetime.strptime(actual_start, date_format).date() 60 | a2 = datetime.strptime(actual_end, date_format).date() 61 | start_uncovered = (a1 - r1).days 62 | end_uncovered = (r2 - a2).days 63 | 64 | def s(n): 65 | return "" if n == 1 else "s" 66 | 67 | if start_uncovered == 0 and end_uncovered > 0: 68 | return ( 69 | f"which ends {end_uncovered} day{s(end_uncovered)} earlier than " 70 | f"requested" 71 | ) 72 | elif start_uncovered > 0 and end_uncovered == 0: 73 | return ( 74 | f"which starts {start_uncovered} day{s(start_uncovered)} later " 75 | "than requested" 76 | ) 77 | elif start_uncovered > 0 and end_uncovered > 0: 78 | return ( 79 | f"which starts {start_uncovered} day{s(start_uncovered)} later " 80 | f"and ends {end_uncovered} day{s(end_uncovered)} earlier " 81 | f"than requested" 82 | ) 83 | elif start_uncovered == 0 and end_uncovered == 0: 84 | return "as requested" 85 | else: 86 | return "which doesn't match the request" 87 | -------------------------------------------------------------------------------- /src/pricehist/exceptions.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | from contextlib import contextmanager 4 | 5 | 6 | @contextmanager 7 | def handler(): 8 | try: 9 | yield 10 | except SourceError as e: 11 | logging.debug("Critical exception encountered", exc_info=e) 12 | logging.critical(str(e)) 13 | sys.exit(1) 14 | 15 | 16 | class SourceError(Exception): 17 | """Base exception for errors rased by sources""" 18 | 19 | 20 | class InvalidPair(SourceError, ValueError): 21 | """An invalid pair was requested.""" 22 | 23 | def __init__(self, base, quote, source, message=None): 24 | self.base = base 25 | self.quote = quote 26 | self.source = source 27 | pair = "/".join([s for s in [base, quote] if s]) 28 | insert = message + " " if message else "" 29 | 30 | full_message = ( 31 | f"Invalid pair '{pair}'. {insert}" 32 | f"Run 'pricehist source {source.id()} --symbols' " 33 | f"for information about valid pairs." 34 | ) 35 | super(InvalidPair, self).__init__(full_message) 36 | 37 | 38 | class InvalidType(SourceError, ValueError): 39 | """An invalid price type was requested.""" 40 | 41 | def __init__(self, type, base, quote, source): 42 | self.type = type 43 | self.pair = "/".join([s for s in [base, quote] if s]) 44 | message = ( 45 | f"Invalid price type '{type}' for pair '{self.pair}'. " 46 | f"Run 'pricehist source {source.id()}' " 47 | f"for information about valid types." 48 | ) 49 | super(InvalidType, self).__init__(message) 50 | 51 | 52 | class CredentialsError(SourceError): 53 | """Access credentials are unavailable or invalid.""" 54 | 55 | def __init__(self, keys, source, msg=""): 56 | self.keys = keys 57 | self.source = source 58 | message = ( 59 | f"Access credentials for source '{source.id()}' are unavailable " 60 | f"""or invalid. Set the environment variables '{"', '".join(keys)}' """ 61 | f"correctly. Run 'pricehist source {source.id()}' for more " 62 | f"information about credentials." 63 | ) 64 | if msg: 65 | message += f" {msg}" 66 | super(CredentialsError, self).__init__(message) 67 | 68 | 69 | class RateLimit(SourceError): 70 | """Source request rate limit reached.""" 71 | 72 | def __init__(self, message): 73 | super(RateLimit, self).__init__(f"{self.__doc__} {message}") 74 | 75 | 76 | class RequestError(SourceError): 77 | """An error occured while making a request to the source.""" 78 | 79 | def __init__(self, message): 80 | super(RequestError, self).__init__(f"{self.__doc__} {message}") 81 | 82 | 83 | class BadResponse(SourceError): 84 | """A bad response was received from the source.""" 85 | 86 | def __init__(self, message): 87 | super(BadResponse, self).__init__(f"{self.__doc__} {message}") 88 | 89 | 90 | class ResponseParsingError(SourceError): 91 | """An error occurred while parsing data from the source.""" 92 | 93 | def __init__(self, message): 94 | super(ResponseParsingError, self).__init__(f"{self.__doc__} {message}") 95 | -------------------------------------------------------------------------------- /src/pricehist/sources/ecb.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from datetime import datetime, timedelta 3 | from decimal import Decimal 4 | 5 | import requests 6 | from lxml import etree 7 | 8 | from pricehist import exceptions, isocurrencies 9 | from pricehist.price import Price 10 | 11 | from .basesource import BaseSource 12 | 13 | 14 | class ECB(BaseSource): 15 | def id(self): 16 | return "ecb" 17 | 18 | def name(self): 19 | return "European Central Bank" 20 | 21 | def description(self): 22 | return "European Central Bank Euro foreign exchange reference rates" 23 | 24 | def source_url(self): 25 | return "https://www.ecb.europa.eu/stats/exchange/eurofxref/html/index.en.html" 26 | 27 | def start(self): 28 | return "1999-01-04" 29 | 30 | def types(self): 31 | return ["reference"] 32 | 33 | def notes(self): 34 | return "" 35 | 36 | def symbols(self): 37 | quotes = self._quotes() 38 | iso = isocurrencies.by_code() 39 | return [ 40 | (f"EUR/{c}", f"Euro against {iso[c].name if c in iso else c}") 41 | for c in quotes 42 | ] 43 | 44 | def fetch(self, series): 45 | if series.base != "EUR" or not series.quote: # EUR is the only valid base. 46 | raise exceptions.InvalidPair(series.base, series.quote, self) 47 | 48 | almost_90_days_ago = (datetime.now().date() - timedelta(days=85)).isoformat() 49 | root = self._data(series.start < almost_90_days_ago) 50 | 51 | all_rows = [] 52 | for day in root.cssselect("[time]"): 53 | date = day.attrib["time"] 54 | for row in day.cssselect(f"[currency='{series.quote}']"): 55 | rate = Decimal(row.attrib["rate"]) 56 | all_rows.insert(0, (date, rate)) 57 | 58 | if not all_rows and series.quote not in self._quotes(): 59 | raise exceptions.InvalidPair(series.base, series.quote, self) 60 | 61 | selected = [ 62 | Price(d, r) for d, r in all_rows if d >= series.start and d <= series.end 63 | ] 64 | 65 | return dataclasses.replace(series, prices=selected) 66 | 67 | def _quotes(self): 68 | root = self._data(more_than_90_days=True) 69 | nodes = root.cssselect("[currency]") 70 | quotes = sorted(set([n.attrib["currency"] for n in nodes])) 71 | if not quotes: 72 | raise exceptions.ResponseParsingError("Expected data not found") 73 | return quotes 74 | 75 | def _data(self, more_than_90_days=False): 76 | url_base = "https://www.ecb.europa.eu/stats/eurofxref" 77 | if more_than_90_days: 78 | source_url = f"{url_base}/eurofxref-hist.xml" # since 1999 79 | else: 80 | source_url = f"{url_base}/eurofxref-hist-90d.xml" # last 90 days 81 | 82 | try: 83 | response = self.log_curl(requests.get(source_url)) 84 | except Exception as e: 85 | raise exceptions.RequestError(str(e)) from e 86 | 87 | try: 88 | response.raise_for_status() 89 | except Exception as e: 90 | raise exceptions.BadResponse(str(e)) from e 91 | 92 | try: 93 | root = etree.fromstring(response.content) 94 | except Exception as e: 95 | raise exceptions.ResponseParsingError(str(e)) from e 96 | 97 | return root 98 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_yahoo/inrx-with-null.json: -------------------------------------------------------------------------------- 1 | { 2 | "chart": { 3 | "result": [ 4 | { 5 | "meta": { 6 | "currency": "INR", 7 | "symbol": "INR=X", 8 | "exchangeName": "CCY", 9 | "fullExchangeName": "CCY", 10 | "instrumentType": "CURRENCY", 11 | "firstTradeDate": 1070236800, 12 | "regularMarketTime": 1726284616, 13 | "hasPrePostMarketData": false, 14 | "gmtoffset": 3600, 15 | "timezone": "BST", 16 | "exchangeTimezoneName": "Europe/London", 17 | "regularMarketPrice": 83.89, 18 | "fiftyTwoWeekHigh": 83.89, 19 | "fiftyTwoWeekLow": 83.89, 20 | "regularMarketDayHigh": 83.89, 21 | "regularMarketDayLow": 83.89, 22 | "regularMarketVolume": 0, 23 | "longName": "USD/INR", 24 | "shortName": "USD/INR", 25 | "chartPreviousClose": 64.6117, 26 | "priceHint": 4, 27 | "currentTradingPeriod": { 28 | "pre": { 29 | "timezone": "BST", 30 | "start": 1726182000, 31 | "end": 1726182000, 32 | "gmtoffset": 3600 33 | }, 34 | "regular": { 35 | "timezone": "BST", 36 | "start": 1726182000, 37 | "end": 1726268340, 38 | "gmtoffset": 3600 39 | }, 40 | "post": { 41 | "timezone": "BST", 42 | "start": 1726268340, 43 | "end": 1726268340, 44 | "gmtoffset": 3600 45 | } 46 | }, 47 | "dataGranularity": "1d", 48 | "range": "", 49 | "validRanges": [ 50 | "1d", 51 | "5d", 52 | "1mo", 53 | "3mo", 54 | "6mo", 55 | "1y", 56 | "2y", 57 | "5y", 58 | "10y", 59 | "ytd", 60 | "max" 61 | ] 62 | }, 63 | "timestamp": [ 64 | 1499641200, 65 | 1499727600, 66 | 1499814000, 67 | 1499900400 68 | ], 69 | "indicators": { 70 | "quote": [ 71 | { 72 | "open": [ 73 | 64.6155014038086, 74 | null, 75 | 64.55549621582031, 76 | 64.46800231933594 77 | ], 78 | "volume": [ 79 | 0, 80 | null, 81 | 0, 82 | 0 83 | ], 84 | "low": [ 85 | 64.41000366210938, 86 | null, 87 | 64.3499984741211, 88 | 64.33999633789062 89 | ], 90 | "close": [ 91 | 64.61170196533203, 92 | null, 93 | 64.52559661865234, 94 | 64.36499786376953 95 | ], 96 | "high": [ 97 | 64.6155014038086, 98 | null, 99 | 64.56999969482422, 100 | 64.48419952392578 101 | ] 102 | } 103 | ], 104 | "adjclose": [ 105 | { 106 | "adjclose": [ 107 | 64.61170196533203, 108 | null, 109 | 64.52559661865234, 110 | 64.36499786376953 111 | ] 112 | } 113 | ] 114 | } 115 | } 116 | ], 117 | "error": null 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_alphavantage/search-ibm.json: -------------------------------------------------------------------------------- 1 | { 2 | "bestMatches": [ 3 | { 4 | "1. symbol": "IBM", 5 | "2. name": "International Business Machines Corp", 6 | "3. type": "Equity", 7 | "4. region": "United States", 8 | "5. marketOpen": "09:30", 9 | "6. marketClose": "16:00", 10 | "7. timezone": "UTC-04", 11 | "8. currency": "USD", 12 | "9. matchScore": "1.0000" 13 | }, 14 | { 15 | "1. symbol": "IBMJ", 16 | "2. name": "iShares iBonds Dec 2021 Term Muni Bond ETF", 17 | "3. type": "ETF", 18 | "4. region": "United States", 19 | "5. marketOpen": "09:30", 20 | "6. marketClose": "16:00", 21 | "7. timezone": "UTC-04", 22 | "8. currency": "USD", 23 | "9. matchScore": "0.8571" 24 | }, 25 | { 26 | "1. symbol": "IBMK", 27 | "2. name": "iShares iBonds Dec 2022 Term Muni Bond ETF", 28 | "3. type": "ETF", 29 | "4. region": "United States", 30 | "5. marketOpen": "09:30", 31 | "6. marketClose": "16:00", 32 | "7. timezone": "UTC-04", 33 | "8. currency": "USD", 34 | "9. matchScore": "0.8571" 35 | }, 36 | { 37 | "1. symbol": "IBML", 38 | "2. name": "iShares iBonds Dec 2023 Term Muni Bond ETF", 39 | "3. type": "ETF", 40 | "4. region": "United States", 41 | "5. marketOpen": "09:30", 42 | "6. marketClose": "16:00", 43 | "7. timezone": "UTC-04", 44 | "8. currency": "USD", 45 | "9. matchScore": "0.8571" 46 | }, 47 | { 48 | "1. symbol": "IBMM", 49 | "2. name": "iShares iBonds Dec 2024 Term Muni Bond ETF", 50 | "3. type": "ETF", 51 | "4. region": "United States", 52 | "5. marketOpen": "09:30", 53 | "6. marketClose": "16:00", 54 | "7. timezone": "UTC-04", 55 | "8. currency": "USD", 56 | "9. matchScore": "0.8571" 57 | }, 58 | { 59 | "1. symbol": "IBMN", 60 | "2. name": "iShares iBonds Dec 2025 Term Muni Bond ETF", 61 | "3. type": "ETF", 62 | "4. region": "United States", 63 | "5. marketOpen": "09:30", 64 | "6. marketClose": "16:00", 65 | "7. timezone": "UTC-04", 66 | "8. currency": "USD", 67 | "9. matchScore": "0.8571" 68 | }, 69 | { 70 | "1. symbol": "IBMO", 71 | "2. name": "iShares iBonds Dec 2026 Term Muni Bond ETF", 72 | "3. type": "ETF", 73 | "4. region": "United States", 74 | "5. marketOpen": "09:30", 75 | "6. marketClose": "16:00", 76 | "7. timezone": "UTC-04", 77 | "8. currency": "USD", 78 | "9. matchScore": "0.8571" 79 | }, 80 | { 81 | "1. symbol": "IBM.FRK", 82 | "2. name": "International Business Machines Corporation", 83 | "3. type": "Equity", 84 | "4. region": "Frankfurt", 85 | "5. marketOpen": "08:00", 86 | "6. marketClose": "20:00", 87 | "7. timezone": "UTC+02", 88 | "8. currency": "EUR", 89 | "9. matchScore": "0.7500" 90 | }, 91 | { 92 | "1. symbol": "IBM.LON", 93 | "2. name": "International Business Machines Corporation", 94 | "3. type": "Equity", 95 | "4. region": "United Kingdom", 96 | "5. marketOpen": "08:00", 97 | "6. marketClose": "16:30", 98 | "7. timezone": "UTC+01", 99 | "8. currency": "USD", 100 | "9. matchScore": "0.7500" 101 | }, 102 | { 103 | "1. symbol": "IBM.DEX", 104 | "2. name": "International Business Machines Corporation", 105 | "3. type": "Equity", 106 | "4. region": "XETRA", 107 | "5. marketOpen": "08:00", 108 | "6. marketClose": "20:00", 109 | "7. timezone": "UTC+02", 110 | "8. currency": "EUR", 111 | "9. matchScore": "0.6667" 112 | } 113 | ] 114 | } 115 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_yahoo/tsla-recent.json: -------------------------------------------------------------------------------- 1 | { 2 | "chart": { 3 | "result": [ 4 | { 5 | "meta": { 6 | "currency": "USD", 7 | "symbol": "TSLA", 8 | "exchangeName": "NMS", 9 | "fullExchangeName": "NasdaqGS", 10 | "instrumentType": "EQUITY", 11 | "firstTradeDate": 1277818200, 12 | "regularMarketTime": 1726257600, 13 | "hasPrePostMarketData": true, 14 | "gmtoffset": -14400, 15 | "timezone": "EDT", 16 | "exchangeTimezoneName": "America/New_York", 17 | "regularMarketPrice": 230.29, 18 | "fiftyTwoWeekHigh": 232.664, 19 | "fiftyTwoWeekLow": 226.32, 20 | "regularMarketDayHigh": 232.664, 21 | "regularMarketDayLow": 226.32, 22 | "regularMarketVolume": 59096538, 23 | "longName": "Tesla, Inc.", 24 | "shortName": "Tesla, Inc.", 25 | "chartPreviousClose": 235.223, 26 | "priceHint": 2, 27 | "currentTradingPeriod": { 28 | "pre": { 29 | "timezone": "EDT", 30 | "start": 1726214400, 31 | "end": 1726234200, 32 | "gmtoffset": -14400 33 | }, 34 | "regular": { 35 | "timezone": "EDT", 36 | "start": 1726234200, 37 | "end": 1726257600, 38 | "gmtoffset": -14400 39 | }, 40 | "post": { 41 | "timezone": "EDT", 42 | "start": 1726257600, 43 | "end": 1726272000, 44 | "gmtoffset": -14400 45 | } 46 | }, 47 | "dataGranularity": "1d", 48 | "range": "", 49 | "validRanges": [ 50 | "1d", 51 | "5d", 52 | "1mo", 53 | "3mo", 54 | "6mo", 55 | "1y", 56 | "2y", 57 | "5y", 58 | "10y", 59 | "ytd", 60 | "max" 61 | ] 62 | }, 63 | "timestamp": [ 64 | 1609770600, 65 | 1609857000, 66 | 1609943400, 67 | 1610029800, 68 | 1610116200 69 | ], 70 | "indicators": { 71 | "quote": [ 72 | { 73 | "open": [ 74 | 239.82000732421875, 75 | 241.22000122070312, 76 | 252.8300018310547, 77 | 259.2099914550781, 78 | 285.3333435058594 79 | ], 80 | "close": [ 81 | 243.2566680908203, 82 | 245.0366668701172, 83 | 251.9933319091797, 84 | 272.0133361816406, 85 | 293.3399963378906 86 | ], 87 | "high": [ 88 | 248.163330078125, 89 | 246.94667053222656, 90 | 258.0, 91 | 272.3299865722656, 92 | 294.8299865722656 93 | ], 94 | "low": [ 95 | 239.06333923339844, 96 | 239.73333740234375, 97 | 249.6999969482422, 98 | 258.3999938964844, 99 | 279.46331787109375 100 | ], 101 | "volume": [ 102 | 145914600, 103 | 96735600, 104 | 134100000, 105 | 154496700, 106 | 225166500 107 | ] 108 | } 109 | ], 110 | "adjclose": [ 111 | { 112 | "adjclose": [ 113 | 243.2566680908203, 114 | 245.0366668701172, 115 | 251.9933319091797, 116 | 272.0133361816406, 117 | 293.3399963378906 118 | ] 119 | } 120 | ] 121 | } 122 | } 123 | ], 124 | "error": null 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_coinbasepro/currencies-partial.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": "BTC", 4 | "name": "Bitcoin", 5 | "min_size": "0.00000001", 6 | "status": "online", 7 | "message": "", 8 | "max_precision": "0.00000001", 9 | "convertible_to": [], 10 | "details": { 11 | "type": "crypto", 12 | "symbol": "₿", 13 | "network_confirmations": 3, 14 | "sort_order": 20, 15 | "crypto_address_link": "https://live.blockcypher.com/btc/address/{{address}}", 16 | "crypto_transaction_link": "https://live.blockcypher.com/btc/tx/{{txId}}", 17 | "push_payment_methods": [ 18 | "crypto" 19 | ], 20 | "group_types": [ 21 | "btc", 22 | "crypto" 23 | ], 24 | "display_name": "", 25 | "processing_time_seconds": 0, 26 | "min_withdrawal_amount": 0.0001, 27 | "max_withdrawal_amount": 2400 28 | } 29 | }, 30 | { 31 | "id": "DOGE", 32 | "name": "Dogecoin", 33 | "min_size": "1", 34 | "status": "online", 35 | "message": "", 36 | "max_precision": "0.1", 37 | "convertible_to": [], 38 | "details": { 39 | "type": "crypto", 40 | "symbol": "", 41 | "network_confirmations": 60, 42 | "sort_order": 29, 43 | "crypto_address_link": "https://dogechain.info/address/{{address}}", 44 | "crypto_transaction_link": "", 45 | "push_payment_methods": [ 46 | "crypto" 47 | ], 48 | "group_types": [], 49 | "display_name": "", 50 | "processing_time_seconds": 0, 51 | "min_withdrawal_amount": 1, 52 | "max_withdrawal_amount": 17391300 53 | } 54 | }, 55 | { 56 | "id": "ETH", 57 | "name": "Ether", 58 | "min_size": "0.00000001", 59 | "status": "online", 60 | "message": "", 61 | "max_precision": "0.00000001", 62 | "convertible_to": [], 63 | "details": { 64 | "type": "crypto", 65 | "symbol": "Ξ", 66 | "network_confirmations": 35, 67 | "sort_order": 25, 68 | "crypto_address_link": "https://etherscan.io/address/{{address}}", 69 | "crypto_transaction_link": "https://etherscan.io/tx/0x{{txId}}", 70 | "push_payment_methods": [ 71 | "crypto" 72 | ], 73 | "group_types": [ 74 | "eth", 75 | "crypto" 76 | ], 77 | "display_name": "", 78 | "processing_time_seconds": 0, 79 | "min_withdrawal_amount": 0.001, 80 | "max_withdrawal_amount": 7450 81 | } 82 | }, 83 | { 84 | "id": "EUR", 85 | "name": "Euro", 86 | "min_size": "0.01", 87 | "status": "online", 88 | "message": "", 89 | "max_precision": "0.01", 90 | "convertible_to": [], 91 | "details": { 92 | "type": "fiat", 93 | "symbol": "€", 94 | "network_confirmations": 0, 95 | "sort_order": 2, 96 | "crypto_address_link": "", 97 | "crypto_transaction_link": "", 98 | "push_payment_methods": [ 99 | "sepa_bank_account" 100 | ], 101 | "group_types": [ 102 | "fiat", 103 | "eur" 104 | ], 105 | "display_name": "", 106 | "processing_time_seconds": 0, 107 | "min_withdrawal_amount": 0, 108 | "max_withdrawal_amount": 0 109 | } 110 | }, 111 | { 112 | "id": "GBP", 113 | "name": "British Pound", 114 | "min_size": "0.01", 115 | "status": "online", 116 | "message": "", 117 | "max_precision": "0.01", 118 | "convertible_to": [], 119 | "details": { 120 | "type": "fiat", 121 | "symbol": "£", 122 | "network_confirmations": 0, 123 | "sort_order": 3, 124 | "crypto_address_link": "", 125 | "crypto_transaction_link": "", 126 | "push_payment_methods": [ 127 | "uk_bank_account", 128 | "swift_lhv", 129 | "swift" 130 | ], 131 | "group_types": [ 132 | "fiat", 133 | "gbp" 134 | ], 135 | "display_name": "", 136 | "processing_time_seconds": 0, 137 | "min_withdrawal_amount": 0, 138 | "max_withdrawal_amount": 0 139 | } 140 | } 141 | ] 142 | -------------------------------------------------------------------------------- /src/pricehist/isocurrencies.py: -------------------------------------------------------------------------------- 1 | """ 2 | ISO 4217 Currency data 3 | 4 | Provides `ISO 4217 `_ 5 | currency data in a ready-to-use format, indexed by currency code. Historical 6 | currencies are included and countries with no universal currency are ignored. 7 | 8 | The data is read from vendored copies of the XML files published by the 9 | maintainers of the standard: 10 | 11 | * :file:`list-one.xml` (current currencies & funds) 12 | * :file:`list-three.xml` (historical currencies & funds) 13 | 14 | Classes: 15 | 16 | ISOCurrency 17 | 18 | Functions: 19 | 20 | current_data_date() -> str 21 | historical_data_date() -> str 22 | by_code() -> dict[str, ISOCurrency] 23 | 24 | """ 25 | 26 | from dataclasses import dataclass, field 27 | from importlib.resources import files 28 | from typing import List 29 | 30 | from lxml import etree 31 | 32 | 33 | @dataclass(frozen=False) 34 | class ISOCurrency: 35 | code: str = None 36 | number: int = None 37 | minor_units: int = None 38 | name: str = None 39 | is_fund: bool = False 40 | countries: List[str] = field(default_factory=list) 41 | historical: bool = False 42 | withdrawal_date: str = None 43 | 44 | 45 | def current_data_date(): 46 | one = etree.fromstring( 47 | files("pricehist.resources").joinpath("list-one.xml").read_bytes() 48 | ) 49 | return one.cssselect("ISO_4217")[0].attrib["Pblshd"] 50 | 51 | 52 | def historical_data_date(): 53 | three = etree.fromstring( 54 | files("pricehist.resources").joinpath("list-three.xml").read_bytes() 55 | ) 56 | return three.cssselect("ISO_4217")[0].attrib["Pblshd"] 57 | 58 | 59 | def by_code(): 60 | result = {} 61 | 62 | one = etree.fromstring( 63 | files("pricehist.resources").joinpath("list-one.xml").read_bytes() 64 | ) 65 | three = etree.fromstring( 66 | files("pricehist.resources").joinpath("list-three.xml").read_bytes() 67 | ) 68 | 69 | for entry in three.cssselect("HstrcCcyNtry") + one.cssselect("CcyNtry"): 70 | if currency := _parse(entry): 71 | if existing := result.get(currency.code): 72 | existing.code = currency.code 73 | existing.number = currency.number 74 | existing.minor_units = currency.minor_units 75 | existing.name = currency.name 76 | existing.is_fund = currency.is_fund 77 | existing.countries += currency.countries 78 | existing.historical = currency.historical 79 | existing.withdrawal_date = currency.withdrawal_date 80 | else: 81 | result[currency.code] = currency 82 | 83 | return result 84 | 85 | 86 | def _parse(entry): 87 | try: 88 | code = entry.cssselect("Ccy")[0].text 89 | except IndexError: 90 | return None # Ignore countries without a universal currency 91 | 92 | try: 93 | number = int(entry.cssselect("CcyNbr")[0].text) 94 | except (IndexError, ValueError): 95 | number = None 96 | 97 | try: 98 | minor_units = int(entry.cssselect("CcyMnrUnts")[0].text) 99 | except (IndexError, ValueError): 100 | minor_units = None 101 | 102 | name = None 103 | is_fund = None 104 | if name_tags := entry.cssselect("CcyNm"): 105 | name = name_tags[0].text 106 | is_fund = name_tags[0].attrib.get("IsFund", "").upper() in ["TRUE", "WAHR"] 107 | 108 | countries = [t.text for t in entry.cssselect("CtryNm")] 109 | 110 | try: 111 | withdrawal_date = entry.cssselect("WthdrwlDt")[0].text 112 | historical = True 113 | except IndexError: 114 | withdrawal_date = None 115 | historical = False 116 | 117 | return ISOCurrency( 118 | code=code, 119 | number=number, 120 | minor_units=minor_units, 121 | name=name, 122 | is_fund=is_fund, 123 | countries=countries, 124 | historical=historical, 125 | withdrawal_date=withdrawal_date, 126 | ) 127 | -------------------------------------------------------------------------------- /src/pricehist/sources/bankofcanada.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import json 3 | from decimal import Decimal 4 | 5 | import requests 6 | 7 | from pricehist import exceptions 8 | from pricehist.price import Price 9 | 10 | from .basesource import BaseSource 11 | 12 | 13 | class BankOfCanada(BaseSource): 14 | def id(self): 15 | return "bankofcanada" 16 | 17 | def name(self): 18 | return "Bank of Canada" 19 | 20 | def description(self): 21 | return "Daily exchange rates of the Canadian dollar from the Bank of Canada" 22 | 23 | def source_url(self): 24 | return "https://www.bankofcanada.ca/valet/docs" 25 | 26 | def start(self): 27 | return "2017-01-03" 28 | 29 | def types(self): 30 | return ["default"] 31 | 32 | def notes(self): 33 | return ( 34 | "Currently, only daily exchange rates are supported. They are " 35 | "published once each business day by 16:30 ET. " 36 | "All Bank of Canada exchange rates are indicative rates only.\n" 37 | "To request support for other data provided by the " 38 | "Bank of Canada Valet Web Services, please open an " 39 | "issue in pricehist's Gitlab project. " 40 | ) 41 | 42 | def symbols(self): 43 | url = "https://www.bankofcanada.ca/valet/lists/series/json" 44 | 45 | try: 46 | response = self.log_curl(requests.get(url)) 47 | except Exception as e: 48 | raise exceptions.RequestError(str(e)) from e 49 | 50 | try: 51 | response.raise_for_status() 52 | except Exception as e: 53 | raise exceptions.BadResponse(str(e)) from e 54 | 55 | try: 56 | data = json.loads(response.content, parse_float=Decimal) 57 | series_names = data["series"].keys() 58 | fx_series_names = [ 59 | n for n in series_names if len(n) == 8 and n[0:2] == "FX" 60 | ] 61 | results = [ 62 | (f"{n[2:5]}/{n[5:9]}", data["series"][n]["description"]) 63 | for n in sorted(fx_series_names) 64 | ] 65 | 66 | except Exception as e: 67 | raise exceptions.ResponseParsingError(str(e)) from e 68 | 69 | if not results: 70 | raise exceptions.ResponseParsingError("Expected data not found") 71 | else: 72 | return results 73 | 74 | def fetch(self, series): 75 | if len(series.base) != 3 or len(series.quote) != 3: 76 | raise exceptions.InvalidPair(series.base, series.quote, self) 77 | 78 | series_name = f"FX{series.base}{series.quote}" 79 | data = self._data(series, series_name) 80 | 81 | prices = [] 82 | for o in data.get("observations", []): 83 | prices.append(Price(o["d"], Decimal(o[series_name]["v"]))) 84 | 85 | return dataclasses.replace(series, prices=prices) 86 | 87 | def _data(self, series, series_name): 88 | url = f"https://www.bankofcanada.ca/valet/observations/{series_name}/json" 89 | params = { 90 | "start_date": series.start, 91 | "end_date": series.end, 92 | "order_dir": "asc", 93 | } 94 | 95 | try: 96 | response = self.log_curl(requests.get(url, params=params)) 97 | except Exception as e: 98 | raise exceptions.RequestError(str(e)) from e 99 | 100 | code = response.status_code 101 | text = response.text 102 | 103 | try: 104 | result = json.loads(response.content, parse_float=Decimal) 105 | except Exception as e: 106 | raise exceptions.ResponseParsingError(str(e)) from e 107 | 108 | if code == 404 and "not found" in text: 109 | raise exceptions.InvalidPair(series.base, series.quote, self) 110 | elif code == 400 and "End date must be greater than the Start date" in text: 111 | raise exceptions.BadResponse(result["message"]) 112 | else: 113 | try: 114 | response.raise_for_status() 115 | except Exception as e: 116 | raise exceptions.BadResponse(str(e)) from e 117 | 118 | return result 119 | -------------------------------------------------------------------------------- /src/pricehist/sources/basesource.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from abc import ABC, abstractmethod 3 | from textwrap import TextWrapper 4 | from typing import List, Tuple 5 | 6 | import curlify 7 | 8 | from pricehist import exceptions 9 | from pricehist.series import Series 10 | 11 | 12 | class BaseSource(ABC): 13 | @abstractmethod 14 | def id(self) -> str: 15 | pass # pragma: nocover 16 | 17 | @abstractmethod 18 | def name(self) -> str: 19 | pass # pragma: nocover 20 | 21 | @abstractmethod 22 | def description(self) -> str: 23 | pass # pragma: nocover 24 | 25 | @abstractmethod 26 | def source_url(self) -> str: 27 | pass # pragma: nocover 28 | 29 | @abstractmethod 30 | def start(self) -> str: 31 | pass # pragma: nocover 32 | 33 | @abstractmethod 34 | def types(self) -> List[str]: 35 | pass # pragma: nocover 36 | 37 | @abstractmethod 38 | def notes(self) -> str: 39 | pass # pragma: nocover 40 | 41 | def normalizesymbol(self, str) -> str: 42 | return str.upper() 43 | 44 | @abstractmethod 45 | def symbols(self) -> List[Tuple[str, str]]: 46 | pass # pragma: nocover 47 | 48 | def search(self, query) -> List[Tuple[str, str]]: 49 | pass # pragma: nocover 50 | 51 | @abstractmethod 52 | def fetch(self, series: Series) -> Series: 53 | pass # pragma: nocover 54 | 55 | def log_curl(self, response): 56 | curl = curlify.to_curl(response.request, compressed=True) 57 | logging.debug(curl) 58 | return response 59 | 60 | def format_symbols(self) -> str: 61 | with exceptions.handler(): 62 | symbols = self.symbols() 63 | 64 | width = max([len(sym) for sym, desc in symbols] + [0]) 65 | lines = [sym.ljust(width + 4) + desc + "\n" for sym, desc in symbols] 66 | return "".join(lines) 67 | 68 | def format_search(self, query) -> str: 69 | with exceptions.handler(): 70 | symbols = self.search(query) 71 | 72 | if symbols is None: 73 | logging.error(f"Symbol search is not possible for the {self.id()} source.") 74 | exit(1) 75 | elif symbols == []: 76 | logging.info(f"No results found for query '{query}'.") 77 | return "" 78 | else: 79 | width = max([len(sym) for sym, desc in symbols] + [0]) 80 | lines = [sym.ljust(width + 4) + desc + "\n" for sym, desc in symbols] 81 | return "".join(lines) 82 | 83 | def format_info(self, total_width=80) -> str: 84 | k_width = 11 85 | with exceptions.handler(): 86 | parts = [ 87 | self._fmt_field("ID", self.id(), k_width, total_width), 88 | self._fmt_field("Name", self.name(), k_width, total_width), 89 | self._fmt_field( 90 | "Description", self.description(), k_width, total_width 91 | ), 92 | self._fmt_field("URL", self.source_url(), k_width, total_width, False), 93 | self._fmt_field("Start", self.start(), k_width, total_width), 94 | self._fmt_field("Types", ", ".join(self.types()), k_width, total_width), 95 | self._fmt_field("Notes", self.notes(), k_width, total_width), 96 | ] 97 | return "\n".join(filter(None, parts)) 98 | 99 | def _fmt_field(self, key, value, key_width, total_width, force=True): 100 | separator = " : " 101 | initial_indent = key + (" " * (key_width - len(key))) + separator 102 | subsequent_indent = " " * len(initial_indent) 103 | wrapper = TextWrapper( 104 | width=total_width, 105 | drop_whitespace=True, 106 | initial_indent=initial_indent, 107 | subsequent_indent=subsequent_indent, 108 | break_long_words=force, 109 | ) 110 | first, *rest = value.split("\n") 111 | first_output = wrapper.wrap(first) 112 | wrapper.initial_indent = subsequent_indent 113 | rest_output = sum([wrapper.wrap(line) if line else [""] for line in rest], []) 114 | output = "\n".join(first_output + rest_output) 115 | if output != "": 116 | return output 117 | else: 118 | return None 119 | -------------------------------------------------------------------------------- /src/pricehist/sources/exchangeratehost.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import json 3 | from decimal import Decimal 4 | 5 | import requests 6 | 7 | from pricehist import exceptions 8 | from pricehist.price import Price 9 | 10 | from .basesource import BaseSource 11 | 12 | 13 | class ExchangeRateHost(BaseSource): 14 | def id(self): 15 | return "exchangeratehost" 16 | 17 | def name(self): 18 | return "exchangerate.host Exchange rates API" 19 | 20 | def description(self): 21 | return ( 22 | "Exchange rates API is a simple and lightweight free service for " 23 | "current and historical foreign exchange rates & crypto exchange " 24 | "rates." 25 | ) 26 | 27 | def source_url(self): 28 | return "https://exchangerate.host/" 29 | 30 | def start(self): 31 | return "1999-01-01" 32 | 33 | def types(self): 34 | return ["close"] 35 | 36 | def notes(self): 37 | return "" 38 | 39 | def symbols(self): 40 | url = "https://api.coindesk.com/v1/bpi/supported-currencies.json" 41 | 42 | try: 43 | response = self.log_curl(requests.get(url)) 44 | except Exception as e: 45 | raise exceptions.RequestError(str(e)) from e 46 | 47 | try: 48 | response.raise_for_status() 49 | except Exception as e: 50 | raise exceptions.BadResponse(str(e)) from e 51 | 52 | try: 53 | data = json.loads(response.content, parse_float=Decimal) 54 | relevant = [i for i in data if i["currency"] not in ["BTC", "XBT"]] 55 | results = [ 56 | (f"BTC/{i['currency']}", f"Bitcoin against {i['country']}") 57 | for i in sorted(relevant, key=lambda i: i["currency"]) 58 | ] 59 | except Exception as e: 60 | raise exceptions.ResponseParsingError(str(e)) from e 61 | 62 | if not results: 63 | raise exceptions.ResponseParsingError("Expected data not found") 64 | else: 65 | return results 66 | 67 | def fetch(self, series): 68 | if series.base != "BTC" or series.quote in ["BTC", "XBT"]: 69 | # BTC is the only valid base. 70 | # BTC as the quote will return BTC/USD, which we don't want. 71 | # XBT as the quote will fail with HTTP status 500. 72 | raise exceptions.InvalidPair(series.base, series.quote, self) 73 | 74 | data = self._data(series) 75 | 76 | prices = [] 77 | for (d, v) in data.get("bpi", {}).items(): 78 | prices.append(Price(d, Decimal(str(v)))) 79 | 80 | return dataclasses.replace(series, prices=prices) 81 | 82 | def _data(self, series): 83 | url = "https://api.coindesk.com/v1/bpi/historical/close.json" 84 | params = { 85 | "currency": series.quote, 86 | "start": series.start, 87 | "end": series.end, 88 | } 89 | 90 | try: 91 | response = self.log_curl(requests.get(url, params=params)) 92 | except Exception as e: 93 | raise exceptions.RequestError(str(e)) from e 94 | 95 | code = response.status_code 96 | text = response.text 97 | if code == 404 and "currency was not found" in text: 98 | raise exceptions.InvalidPair(series.base, series.quote, self) 99 | elif code == 404 and "only covers data from" in text: 100 | raise exceptions.BadResponse(text) 101 | elif code == 404 and "end date is before" in text and series.end < series.start: 102 | raise exceptions.BadResponse("End date is before start date.") 103 | elif code == 404 and "end date is before" in text: 104 | raise exceptions.BadResponse("The start date must be in the past.") 105 | elif code == 500 and "No results returned from database" in text: 106 | raise exceptions.BadResponse( 107 | "No results returned from database. This can happen when data " 108 | "for a valid quote currency (e.g. CUP) doesn't go all the way " 109 | "back to the start date, and potentially for other reasons." 110 | ) 111 | else: 112 | try: 113 | response.raise_for_status() 114 | except Exception as e: 115 | raise exceptions.BadResponse(str(e)) from e 116 | 117 | try: 118 | result = json.loads(response.content, parse_float=Decimal) 119 | except Exception as e: 120 | raise exceptions.ResponseParsingError(str(e)) from e 121 | 122 | return result 123 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_coinmarketcap/recent-id1-id2782.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": { 3 | "id": 1, 4 | "name": "Bitcoin", 5 | "symbol": "BTC", 6 | "timeEnd": "1575503999", 7 | "quotes": [ 8 | { 9 | "timeOpen": "2021-01-01T00:00:00.000Z", 10 | "timeClose": "2021-01-01T23:59:59.999Z", 11 | "timeHigh": "2021-01-01T12:38:43.000Z", 12 | "timeLow": "2021-01-01T00:16:43.000Z", 13 | "quote": { 14 | "name": "2782", 15 | "open": 37658.1146368474, 16 | "high": 38417.9137031205, 17 | "low": 37410.7875016392, 18 | "close": 38181.9913330076, 19 | "volume": 52901492931.8344367080, 20 | "marketCap": 709159975413.2388897949, 21 | "timestamp": "2021-01-01T23:59:59.999Z" 22 | } 23 | }, 24 | { 25 | "timeOpen": "2021-01-02T00:00:00.000Z", 26 | "timeClose": "2021-01-02T23:59:59.999Z", 27 | "timeHigh": "2021-01-02T19:49:42.000Z", 28 | "timeLow": "2021-01-02T00:31:44.000Z", 29 | "quote": { 30 | "name": "2782", 31 | "open": 38184.9861160068, 32 | "high": 43096.6811974230, 33 | "low": 37814.1718709653, 34 | "close": 41760.6292307951, 35 | "volume": 88214867181.9830439141, 36 | "marketCap": 776278147177.8037261338, 37 | "timestamp": "2021-01-02T23:59:59.999Z" 38 | } 39 | }, 40 | { 41 | "timeOpen": "2021-01-03T00:00:00.000Z", 42 | "timeClose": "2021-01-03T23:59:59.999Z", 43 | "timeHigh": "2021-01-03T07:47:38.000Z", 44 | "timeLow": "2021-01-03T00:20:45.000Z", 45 | "quote": { 46 | "name": "2782", 47 | "open": 41763.4101511766, 48 | "high": 44985.9324758502, 49 | "low": 41663.2043506016, 50 | "close": 42534.0538859236, 51 | "volume": 102253005977.1115650988, 52 | "marketCap": 792140565709.1701340036, 53 | "timestamp": "2021-01-03T23:59:59.999Z" 54 | } 55 | }, 56 | { 57 | "timeOpen": "2021-01-04T00:00:00.000Z", 58 | "timeClose": "2021-01-04T23:59:59.999Z", 59 | "timeHigh": "2021-01-04T04:07:42.000Z", 60 | "timeLow": "2021-01-04T10:19:42.000Z", 61 | "quote": { 62 | "name": "2782", 63 | "open": 42548.6134964877, 64 | "high": 43347.7527651400, 65 | "low": 37111.8678479690, 66 | "close": 41707.4890765162, 67 | "volume": 105251252720.3013091567, 68 | "marketCap": 770785910830.3801120744, 69 | "timestamp": "2021-01-04T23:59:59.999Z" 70 | } 71 | }, 72 | { 73 | "timeOpen": "2021-01-05T00:00:00.000Z", 74 | "timeClose": "2021-01-05T23:59:59.999Z", 75 | "timeHigh": "2021-01-05T22:44:35.000Z", 76 | "timeLow": "2021-01-05T06:16:41.000Z", 77 | "quote": { 78 | "name": "2782", 79 | "open": 41693.0732180764, 80 | "high": 44406.6531914952, 81 | "low": 39220.9654861842, 82 | "close": 43777.4560620835, 83 | "volume": 88071174132.6445648582, 84 | "marketCap": 824003338903.4613958343, 85 | "timestamp": "2021-01-05T23:59:59.999Z" 86 | } 87 | }, 88 | { 89 | "timeOpen": "2021-01-06T00:00:00.000Z", 90 | "timeClose": "2021-01-06T23:59:59.999Z", 91 | "timeHigh": "2021-01-06T23:57:36.000Z", 92 | "timeLow": "2021-01-06T00:25:38.000Z", 93 | "quote": { 94 | "name": "2782", 95 | "open": 43798.3790529373, 96 | "high": 47185.7303335186, 97 | "low": 43152.6028176424, 98 | "close": 47114.9330444897, 99 | "volume": 96948095813.7503737302, 100 | "marketCap": 881631993096.0701475336, 101 | "timestamp": "2021-01-06T23:59:59.999Z" 102 | } 103 | }, 104 | { 105 | "timeOpen": "2021-01-07T00:00:00.000Z", 106 | "timeClose": "2021-01-07T23:59:59.999Z", 107 | "timeHigh": "2021-01-07T18:17:42.000Z", 108 | "timeLow": "2021-01-07T08:25:51.000Z", 109 | "quote": { 110 | "name": "2782", 111 | "open": 47128.0213932810, 112 | "high": 51832.6746004172, 113 | "low": 46906.6511713961, 114 | "close": 50660.9643451606, 115 | "volume": 108451040396.2660095877, 116 | "marketCap": 936655898949.2177196744, 117 | "timestamp": "2021-01-07T23:59:59.999Z" 118 | } 119 | } 120 | ] 121 | }, 122 | "status": { 123 | "timestamp": "2024-08-02T18:23:21.586Z", 124 | "error_code": "0", 125 | "error_message": "SUCCESS", 126 | "elapsed": "212", 127 | "credit_count": 0 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /src/pricehist/sources/coindesk.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import json 3 | import logging 4 | from decimal import Decimal 5 | 6 | import requests 7 | 8 | from pricehist import exceptions 9 | from pricehist.price import Price 10 | 11 | from .basesource import BaseSource 12 | 13 | 14 | class CoinDesk(BaseSource): 15 | def id(self): 16 | return "coindesk" 17 | 18 | def name(self): 19 | return "CoinDesk Bitcoin Price Index" 20 | 21 | def description(self): 22 | return ( 23 | "WARNING: This source is deprecated. Data stops at 2022-07-10.\n" 24 | "The documentation URL now redirects to the main page.\n" 25 | "An average of Bitcoin prices across leading global exchanges.\n" 26 | "Powered by CoinDesk, https://www.coindesk.com/price/bitcoin" 27 | ) 28 | 29 | def source_url(self): 30 | return "https://www.coindesk.com/coindesk-api" 31 | 32 | def start(self): 33 | return "2010-07-17" 34 | 35 | def types(self): 36 | return ["close"] 37 | 38 | def notes(self): 39 | return "" 40 | 41 | def symbols(self): 42 | url = "https://api.coindesk.com/v1/bpi/supported-currencies.json" 43 | 44 | try: 45 | response = self.log_curl(requests.get(url)) 46 | except Exception as e: 47 | raise exceptions.RequestError(str(e)) from e 48 | 49 | try: 50 | response.raise_for_status() 51 | except Exception as e: 52 | raise exceptions.BadResponse(str(e)) from e 53 | 54 | try: 55 | data = json.loads(response.content, parse_float=Decimal) 56 | relevant = [i for i in data if i["currency"] not in ["BTC", "XBT"]] 57 | results = [ 58 | (f"BTC/{i['currency']}", f"Bitcoin against {i['country']}") 59 | for i in sorted(relevant, key=lambda i: i["currency"]) 60 | ] 61 | except Exception as e: 62 | raise exceptions.ResponseParsingError(str(e)) from e 63 | 64 | if not results: 65 | raise exceptions.ResponseParsingError("Expected data not found") 66 | else: 67 | return results 68 | 69 | def fetch(self, series): 70 | logging.warning("This source is deprecated. Data stops at 2022-07-10.") 71 | 72 | if series.base != "BTC" or series.quote in ["BTC", "XBT"]: 73 | # BTC is the only valid base. 74 | # BTC as the quote will return BTC/USD, which we don't want. 75 | # XBT as the quote will fail with HTTP status 500. 76 | raise exceptions.InvalidPair(series.base, series.quote, self) 77 | 78 | data = self._data(series) 79 | 80 | prices = [] 81 | for (d, v) in data.get("bpi", {}).items(): 82 | prices.append(Price(d, Decimal(str(v)))) 83 | 84 | return dataclasses.replace(series, prices=prices) 85 | 86 | def _data(self, series): 87 | url = "https://api.coindesk.com/v1/bpi/historical/close.json" 88 | params = { 89 | "currency": series.quote, 90 | "start": series.start, 91 | "end": series.end, 92 | } 93 | 94 | try: 95 | response = self.log_curl(requests.get(url, params=params)) 96 | except Exception as e: 97 | raise exceptions.RequestError(str(e)) from e 98 | 99 | code = response.status_code 100 | text = response.text 101 | if code == 404 and "currency was not found" in text: 102 | raise exceptions.InvalidPair(series.base, series.quote, self) 103 | elif code == 404 and "only covers data from" in text: 104 | raise exceptions.BadResponse(text) 105 | elif code == 404 and "end date is before" in text and series.end < series.start: 106 | raise exceptions.BadResponse("End date is before start date.") 107 | elif code == 404 and "end date is before" in text: 108 | raise exceptions.BadResponse("The start date must be in the past.") 109 | elif code == 500 and "No results returned from database" in text: 110 | raise exceptions.BadResponse( 111 | "No results returned from database. This can happen when data " 112 | "for a valid quote currency (e.g. CUP) doesn't go all the way " 113 | "back to the start date, and potentially for other reasons." 114 | ) 115 | else: 116 | try: 117 | response.raise_for_status() 118 | except Exception as e: 119 | raise exceptions.BadResponse(str(e)) from e 120 | 121 | try: 122 | result = json.loads(response.content, parse_float=Decimal) 123 | except Exception as e: 124 | raise exceptions.ResponseParsingError(str(e)) from e 125 | 126 | return result 127 | -------------------------------------------------------------------------------- /tests/pricehist/outputs/test_gnucashsql.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import logging 3 | import re 4 | from decimal import Decimal 5 | 6 | import pytest 7 | 8 | from pricehist.format import Format 9 | from pricehist.outputs.gnucashsql import GnuCashSQL 10 | from pricehist.price import Price 11 | from pricehist.series import Series 12 | 13 | 14 | @pytest.fixture 15 | def out(): 16 | return GnuCashSQL() 17 | 18 | 19 | @pytest.fixture 20 | def series(): 21 | prices = [ 22 | Price("2021-01-01", Decimal("24139.4648")), 23 | Price("2021-01-02", Decimal("26533.576")), 24 | Price("2021-01-03", Decimal("27001.2846")), 25 | ] 26 | return Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03", prices) 27 | 28 | 29 | @pytest.fixture 30 | def src(mocker): 31 | source = mocker.MagicMock() 32 | source.id = mocker.MagicMock(return_value="coindesk") 33 | return source 34 | 35 | 36 | def test_format_base_and_quote(out, series, src): 37 | result = out.format(series, src, Format()) 38 | base, quote = re.findall(r"WHERE mnemonic = (.*) LIMIT", result, re.MULTILINE) 39 | assert base == "'BTC'" 40 | assert quote == "'EUR'" 41 | 42 | 43 | def test_format_new_price_values(out, series, src): 44 | result = out.format(series, src, Format()) 45 | values = re.search( 46 | r"\(guid, date, base, quote, source, type, " 47 | r"value_num, value_denom\) VALUES\n([^;]*);", 48 | result, 49 | re.MULTILINE, 50 | )[1] 51 | assert values == ( 52 | "('0c4c01bd0a252641b806ce46f716f161', '2021-01-01 00:00:00', " 53 | "'BTC', 'EUR', 'coindesk', 'close', 241394648, 10000),\n" 54 | "('47f895ddfcce18e2421387e0e1b636e9', '2021-01-02 00:00:00', " 55 | "'BTC', 'EUR', 'coindesk', 'close', 26533576, 1000),\n" 56 | "('0d81630c4ac50c1b9b7c8211bf99c94e', '2021-01-03 00:00:00', " 57 | "'BTC', 'EUR', 'coindesk', 'close', 270012846, 10000)\n" 58 | ) 59 | 60 | 61 | def test_format_customized(out, series, src): 62 | fmt = Format( 63 | base="XBT", 64 | quote="EURO", 65 | datesep="/", 66 | time="23:59:59", 67 | ) 68 | result = out.format(series, src, fmt) 69 | base, quote = re.findall(r"WHERE mnemonic = (.*) LIMIT", result, re.MULTILINE) 70 | values = re.search( 71 | r"\(guid, date, base, quote, source, type, " 72 | r"value_num, value_denom\) VALUES\n([^;]*);", 73 | result, 74 | re.MULTILINE, 75 | )[1] 76 | assert base == "'XBT'" 77 | assert quote == "'EURO'" 78 | assert values == ( 79 | "('448173eef5dea23cea9ff9d5e8c7b07e', '2021/01/01 23:59:59', " 80 | "'XBT', 'EURO', 'coindesk', 'close', 241394648, 10000),\n" 81 | "('b6c0f4474c91c50e8f65b47767f874ba', '2021/01/02 23:59:59', " 82 | "'XBT', 'EURO', 'coindesk', 'close', 26533576, 1000),\n" 83 | "('2937c872cf0672863e11b9f46ee41e09', '2021/01/03 23:59:59', " 84 | "'XBT', 'EURO', 'coindesk', 'close', 270012846, 10000)\n" 85 | ) 86 | 87 | 88 | def test_format_escaping_of_strings(out, series, src): 89 | result = out.format(series, src, Format(base="B'tc''n")) 90 | base, quote = re.findall(r"WHERE mnemonic = (.*) LIMIT", result, re.MULTILINE) 91 | assert base == "'B''tc''''n'" 92 | 93 | 94 | def test_format_insert_commented_out_if_no_values(out, series, src): 95 | empty_series = dataclasses.replace(series, prices=[]) 96 | result = out.format(empty_series, src, Format()) 97 | ( 98 | "-- INSERT INTO new_prices (guid, date, base, quote, source, type, " 99 | "value_num, value_denom) VALUES\n" 100 | "-- \n" 101 | "-- ;\n" 102 | ) in result 103 | 104 | 105 | def test_format_warns_about_backslash(out, series, src, caplog): 106 | with caplog.at_level(logging.WARNING): 107 | out.format(series, src, Format(quote="EU\\RO")) 108 | r = caplog.records[0] 109 | assert r.levelname == "WARNING" 110 | assert "backslashes in strings" in r.message 111 | 112 | 113 | def test__english_join_other_cases(out): 114 | assert out._english_join([]) == "" 115 | assert out._english_join(["one"]) == "one" 116 | assert out._english_join(["one", "two"]) == "one and two" 117 | assert out._english_join(["one", "two", "three"]) == "one, two and three" 118 | 119 | 120 | def test_format_warns_about_out_of_range_numbers(out, series, src, caplog): 121 | too_big_numerator = Decimal("9223372036854.775808") 122 | s = dataclasses.replace(series, prices=[Price("2021-01-01", too_big_numerator)]) 123 | with caplog.at_level(logging.WARNING): 124 | out.format(s, src, Format()) 125 | r = caplog.records[0] 126 | assert r.levelname == "WARNING" 127 | assert "outside of the int64 range" in r.message 128 | 129 | 130 | def test__rational_other_exponent_cases(out): 131 | assert out._rational(Decimal("9223372036854e6")) == ( 132 | "9223372036854000000", 133 | "1", 134 | True, 135 | ) 136 | assert out._rational(Decimal("9223372036854e-6")) == ( 137 | "9223372036854", 138 | "1000000", 139 | True, 140 | ) 141 | -------------------------------------------------------------------------------- /tests/pricehist/test_beanprice.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | from datetime import date, datetime, timedelta, timezone 3 | from decimal import Decimal 4 | 5 | import pytest 6 | 7 | from pricehist import beanprice, exceptions, sources 8 | from pricehist.price import Price 9 | from pricehist.series import Series 10 | 11 | 12 | @pytest.fixture 13 | def series(): 14 | series = Series( 15 | "BTC", 16 | "USD", 17 | "high", 18 | "2021-01-01", 19 | "2021-01-03", 20 | prices=[ 21 | Price("2021-01-01", Decimal("1.1")), 22 | Price("2021-01-02", Decimal("1.2")), 23 | Price("2021-01-03", Decimal("1.3")), 24 | ], 25 | ) 26 | return series 27 | 28 | 29 | @pytest.fixture 30 | def pricehist_source(mocker, series): 31 | mock = mocker.MagicMock() 32 | mock.types = mocker.MagicMock(return_value=["close", "high", "low"]) 33 | mock.fetch = mocker.MagicMock(return_value=series) 34 | return mock 35 | 36 | 37 | @pytest.fixture 38 | def source(pricehist_source): 39 | return beanprice.source(pricehist_source)() 40 | 41 | 42 | @pytest.fixture 43 | def ltz(): 44 | return datetime.now(timezone.utc).astimezone().tzinfo 45 | 46 | 47 | def test_get_prices_series(pricehist_source, source, ltz): 48 | ticker = "BTC:USD:high" 49 | begin = datetime(2021, 1, 1, tzinfo=ltz) 50 | end = datetime(2021, 1, 3, tzinfo=ltz) 51 | result = source.get_prices_series(ticker, begin, end) 52 | 53 | pricehist_source.fetch.assert_called_once_with( 54 | Series("BTC", "USD", "high", "2021-01-01", "2021-01-03") 55 | ) 56 | 57 | assert result == [ 58 | beanprice.SourcePrice(Decimal("1.1"), datetime(2021, 1, 1, tzinfo=ltz), "USD"), 59 | beanprice.SourcePrice(Decimal("1.2"), datetime(2021, 1, 2, tzinfo=ltz), "USD"), 60 | beanprice.SourcePrice(Decimal("1.3"), datetime(2021, 1, 3, tzinfo=ltz), "USD"), 61 | ] 62 | 63 | 64 | def test_get_prices_series_exception(pricehist_source, source, ltz, mocker): 65 | pricehist_source.fetch = mocker.MagicMock( 66 | side_effect=exceptions.RequestError("Message") 67 | ) 68 | ticker = "_5eDJI::low" 69 | begin = datetime(2021, 1, 1, tzinfo=ltz) 70 | end = datetime(2021, 1, 3, tzinfo=ltz) 71 | result = source.get_prices_series(ticker, begin, end) 72 | assert result is None 73 | 74 | 75 | def test_get_prices_series_special_chars(pricehist_source, source, ltz): 76 | ticker = "_5eDJI::low" 77 | begin = datetime(2021, 1, 1, tzinfo=ltz) 78 | end = datetime(2021, 1, 3, tzinfo=ltz) 79 | source.get_prices_series(ticker, begin, end) 80 | pricehist_source.fetch.assert_called_once_with( 81 | Series("^DJI", "", "low", "2021-01-01", "2021-01-03") 82 | ) 83 | 84 | 85 | def test_get_prices_series_price_type(pricehist_source, source, ltz): 86 | ticker = "TSLA" 87 | begin = datetime(2021, 1, 1, tzinfo=ltz) 88 | end = datetime(2021, 1, 3, tzinfo=ltz) 89 | source.get_prices_series(ticker, begin, end) 90 | pricehist_source.fetch.assert_called_once_with( 91 | Series("TSLA", "", "close", "2021-01-01", "2021-01-03") 92 | ) 93 | 94 | 95 | def test_get_historical_price(pricehist_source, source, ltz): 96 | ticker = "BTC:USD:high" 97 | time = datetime(2021, 1, 3, tzinfo=ltz) 98 | result = source.get_historical_price(ticker, time) 99 | pricehist_source.fetch.assert_called_once_with( 100 | Series("BTC", "USD", "high", "2021-01-03", "2021-01-03") 101 | ) 102 | assert result == beanprice.SourcePrice( 103 | Decimal("1.3"), datetime(2021, 1, 3, tzinfo=ltz), "USD" 104 | ) 105 | 106 | 107 | def test_get_historical_price_none_available(pricehist_source, source, ltz, mocker): 108 | pricehist_source.fetch = mocker.MagicMock( 109 | return_value=Series("BTC", "USD", "high", "2021-01-03", "2021-01-03", prices=[]) 110 | ) 111 | ticker = "BTC:USD:high" 112 | time = datetime(2021, 1, 3, tzinfo=ltz) 113 | result = source.get_historical_price(ticker, time) 114 | assert result is None 115 | 116 | 117 | def test_get_latest_price(pricehist_source, source, ltz): 118 | ticker = "BTC:USD:high" 119 | start = datetime.combine((date.today() - timedelta(days=7)), datetime.min.time()) 120 | today = datetime.combine(date.today(), datetime.min.time()) 121 | result = source.get_latest_price(ticker) 122 | pricehist_source.fetch.assert_called_once_with( 123 | Series("BTC", "USD", "high", start.date().isoformat(), today.date().isoformat()) 124 | ) 125 | assert result == beanprice.SourcePrice( 126 | Decimal("1.3"), datetime(2021, 1, 3, tzinfo=ltz), "USD" 127 | ) 128 | 129 | 130 | def test_get_latest_price_none_available(pricehist_source, source, ltz, mocker): 131 | pricehist_source.fetch = mocker.MagicMock( 132 | return_value=Series("BTC", "USD", "high", "2021-01-01", "2021-01-03", prices=[]) 133 | ) 134 | ticker = "BTC:USD:high" 135 | result = source.get_latest_price(ticker) 136 | assert result is None 137 | 138 | 139 | def test_all_sources_available_for_beanprice(): 140 | for identifier in sources.by_id.keys(): 141 | importlib.import_module(f"pricehist.beanprice.{identifier}").Source() 142 | -------------------------------------------------------------------------------- /tests/pricehist/outputs/test_json.py: -------------------------------------------------------------------------------- 1 | from decimal import Decimal 2 | from textwrap import dedent 3 | 4 | import pytest 5 | 6 | from pricehist.format import Format 7 | from pricehist.outputs.json import JSON 8 | from pricehist.price import Price 9 | from pricehist.series import Series 10 | 11 | 12 | @pytest.fixture 13 | def json_out(): 14 | return JSON() 15 | 16 | 17 | @pytest.fixture 18 | def jsonl_out(): 19 | return JSON(jsonl=True) 20 | 21 | 22 | @pytest.fixture 23 | def series(): 24 | prices = [ 25 | Price("2021-01-01", Decimal("24139.4648")), 26 | Price("2021-01-02", Decimal("26533.576")), 27 | Price("2021-01-03", Decimal("27001.2846")), 28 | ] 29 | return Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03", prices) 30 | 31 | 32 | def test_format_basics(json_out, series, mocker): 33 | source = mocker.MagicMock() 34 | source.id = mocker.MagicMock(return_value="sourceid") 35 | result = json_out.format(series, source, Format()) 36 | assert ( 37 | result 38 | == dedent( 39 | """ 40 | [ 41 | { 42 | "date": "2021-01-01", 43 | "base": "BTC", 44 | "quote": "EUR", 45 | "amount": "24139.4648", 46 | "source": "sourceid", 47 | "type": "close" 48 | }, 49 | { 50 | "date": "2021-01-02", 51 | "base": "BTC", 52 | "quote": "EUR", 53 | "amount": "26533.576", 54 | "source": "sourceid", 55 | "type": "close" 56 | }, 57 | { 58 | "date": "2021-01-03", 59 | "base": "BTC", 60 | "quote": "EUR", 61 | "amount": "27001.2846", 62 | "source": "sourceid", 63 | "type": "close" 64 | } 65 | ] 66 | """ 67 | ).strip() 68 | + "\n" 69 | ) 70 | 71 | 72 | def test_format_basic_jsonl(jsonl_out, series, mocker): 73 | source = mocker.MagicMock() 74 | source.id = mocker.MagicMock(return_value="sourceid") 75 | result = jsonl_out.format(series, source, Format()) 76 | assert ( 77 | result 78 | == dedent( 79 | """ 80 | {"date": "2021-01-01", "base": "BTC", "quote": "EUR", "amount": "24139.4648", "source": "sourceid", "type": "close"} 81 | {"date": "2021-01-02", "base": "BTC", "quote": "EUR", "amount": "26533.576", "source": "sourceid", "type": "close"} 82 | {"date": "2021-01-03", "base": "BTC", "quote": "EUR", "amount": "27001.2846", "source": "sourceid", "type": "close"} 83 | """ # noqa 84 | ).strip() 85 | + "\n" 86 | ) 87 | 88 | 89 | def test_format_custom(json_out, series, mocker): 90 | source = mocker.MagicMock() 91 | source.id = mocker.MagicMock(return_value="sourceid") 92 | fmt = Format(base="XBT", quote="€", thousands=".", decimal=",", datesep="/") 93 | result = json_out.format(series, source, fmt) 94 | assert ( 95 | result 96 | == dedent( 97 | """ 98 | [ 99 | { 100 | "date": "2021/01/01", 101 | "base": "XBT", 102 | "quote": "€", 103 | "amount": "24.139,4648", 104 | "source": "sourceid", 105 | "type": "close" 106 | }, 107 | { 108 | "date": "2021/01/02", 109 | "base": "XBT", 110 | "quote": "€", 111 | "amount": "26.533,576", 112 | "source": "sourceid", 113 | "type": "close" 114 | }, 115 | { 116 | "date": "2021/01/03", 117 | "base": "XBT", 118 | "quote": "€", 119 | "amount": "27.001,2846", 120 | "source": "sourceid", 121 | "type": "close" 122 | } 123 | ] 124 | """ 125 | ).strip() 126 | + "\n" 127 | ) 128 | 129 | 130 | def test_format_numbers(json_out, series, mocker): 131 | source = mocker.MagicMock() 132 | source.id = mocker.MagicMock(return_value="sourceid") 133 | fmt = Format(jsonnums=True) 134 | result = json_out.format(series, source, fmt) 135 | assert ( 136 | result 137 | == dedent( 138 | """ 139 | [ 140 | { 141 | "date": "2021-01-01", 142 | "base": "BTC", 143 | "quote": "EUR", 144 | "amount": 24139.4648, 145 | "source": "sourceid", 146 | "type": "close" 147 | }, 148 | { 149 | "date": "2021-01-02", 150 | "base": "BTC", 151 | "quote": "EUR", 152 | "amount": 26533.576, 153 | "source": "sourceid", 154 | "type": "close" 155 | }, 156 | { 157 | "date": "2021-01-03", 158 | "base": "BTC", 159 | "quote": "EUR", 160 | "amount": 27001.2846, 161 | "source": "sourceid", 162 | "type": "close" 163 | } 164 | ] 165 | """ 166 | ).strip() 167 | + "\n" 168 | ) 169 | -------------------------------------------------------------------------------- /tests/live.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # These are basic happy path tests that run pricehist from the command line and 4 | # confirm that the results come out as expected. They help ensure that the main 5 | # endpoints for each source are still working. 6 | 7 | # Run this from the project root. 8 | 9 | export ALPHAVANTAGE_API_KEY="TEST_KEY_$RANDOM" 10 | cmd_prefix="poetry run" 11 | 12 | passed=0 13 | failed=0 14 | skipped=0 15 | 16 | run_test(){ 17 | name=$1 18 | cmd=$2 19 | expected=$3 20 | echo "TEST: $name" 21 | echo " Action: $cmd" 22 | echo -n " Result: " 23 | full_cmd="$cmd_prefix $cmd" 24 | actual=$($full_cmd 2>&1) 25 | if [[ "$actual" == "$expected" ]]; then 26 | passed=$((passed+1)) 27 | echo "passed, output as expected" 28 | else 29 | failed=$((failed+1)) 30 | echo "failed, output differs as follows..." 31 | echo 32 | diff <(echo "$expected") <(echo "$actual") 33 | fi 34 | echo 35 | } 36 | 37 | skip_test(){ 38 | name=$1 39 | cmd=$2 40 | echo "TEST: $name" 41 | echo " Action: $cmd" 42 | echo " Result: SKIPPED!" 43 | skipped=$((skipped+1)) 44 | echo 45 | } 46 | 47 | report(){ 48 | total=$((passed+failed)) 49 | if [[ "$skipped" -eq "0" ]]; then 50 | skipped_str="none" 51 | else 52 | skipped_str="$skipped" 53 | fi 54 | if [[ "$failed" -eq "0" ]]; then 55 | echo "SUMMARY: $passed tests passed, none failed, $skipped_str skipped" 56 | else 57 | echo "SUMMARY: $failed/$total tests failed, $skipped_str skipped" 58 | exit 1 59 | fi 60 | } 61 | 62 | name="Alpha Vantage stocks" 63 | cmd="pricehist fetch alphavantage TSLA -s 2021-01-04 -e 2021-01-08" 64 | read -r -d '' expected < str: 12 | return "" 13 | 14 | def name(self) -> str: 15 | return "" 16 | 17 | def description(self) -> str: 18 | return "" 19 | 20 | def source_url(self) -> str: 21 | return "" 22 | 23 | def start(self) -> str: 24 | return "" 25 | 26 | def types(self) -> List[str]: 27 | return [] 28 | 29 | def notes(self) -> str: 30 | return "" 31 | 32 | def symbols(self) -> List[Tuple[str, str]]: 33 | return [] 34 | 35 | def fetch(self, series: Series) -> Series: 36 | pass 37 | 38 | 39 | @pytest.fixture 40 | def src(): 41 | return TestSource() 42 | 43 | 44 | def test_normalizesymbol_default_uppercase(src): 45 | assert src.normalizesymbol("eur") == "EUR" 46 | 47 | 48 | def test_format_symbols_one(src, mocker): 49 | src.symbols = mocker.MagicMock(return_value=[("A", "Description")]) 50 | assert src.format_symbols() == "A Description\n" 51 | 52 | 53 | def test_format_symbols_many(src, mocker): 54 | src.symbols = mocker.MagicMock( 55 | return_value=[ 56 | ("A", "Description"), 57 | ("BB", "Description longer"), 58 | ("CCC", "Description longer again"), 59 | ("DDDD", f"Description {'very '*15}long"), 60 | ] 61 | ) 62 | assert src.format_symbols() == ( 63 | "A Description\n" 64 | "BB Description longer\n" 65 | "CCC Description longer again\n" 66 | "DDDD Description very very very very very very very very " 67 | "very very very very very very very long\n" 68 | ) 69 | 70 | 71 | def test_format_search(src, mocker): 72 | src.search = mocker.MagicMock( 73 | return_value=[ 74 | ("A", "Description"), 75 | ("BB", "Description longer"), 76 | ("CCC", "Description longer again"), 77 | ("DDDD", f"Description {'very '*15}long"), 78 | ] 79 | ) 80 | assert src.format_search("some query") == ( 81 | "A Description\n" 82 | "BB Description longer\n" 83 | "CCC Description longer again\n" 84 | "DDDD Description very very very very very very very very " 85 | "very very very very very very very long\n" 86 | ) 87 | 88 | 89 | def test_format_search_not_possible(src, mocker, caplog): 90 | src.search = mocker.MagicMock(return_value=None) 91 | with caplog.at_level(logging.INFO): 92 | with pytest.raises(SystemExit) as e: 93 | src.format_search("some query") 94 | assert e.value.code == 1 95 | r = caplog.records[0] 96 | assert r.levelname == "ERROR" 97 | assert "Symbol search is not possible for" in r.message 98 | 99 | 100 | def test_format_search_no_results(src, mocker, caplog): 101 | src.search = mocker.MagicMock(return_value=[]) 102 | with caplog.at_level(logging.INFO): 103 | results = src.format_search("some query") 104 | r = caplog.records[0] 105 | assert r.levelname == "INFO" 106 | assert "No results found" in r.message 107 | assert results == "" 108 | 109 | 110 | def test_format_info_skips_renderes_all_fields(src, mocker): 111 | src.id = mocker.MagicMock(return_value="sourceid") 112 | src.name = mocker.MagicMock(return_value="Source Name") 113 | src.description = mocker.MagicMock(return_value="Source description.") 114 | src.source_url = mocker.MagicMock(return_value="https://example.com/") 115 | src.start = mocker.MagicMock(return_value="2021-01-01") 116 | src.types = mocker.MagicMock(return_value=["open", "close"]) 117 | src.notes = mocker.MagicMock(return_value="Notes for user.") 118 | output = src.format_info() 119 | assert output == ( 120 | "ID : sourceid\n" 121 | "Name : Source Name\n" 122 | "Description : Source description.\n" 123 | "URL : https://example.com/\n" 124 | "Start : 2021-01-01\n" 125 | "Types : open, close\n" 126 | "Notes : Notes for user." 127 | ) 128 | 129 | 130 | def test_format_info_skips_empty_fields(src, mocker): 131 | src.notes = mocker.MagicMock(return_value="") 132 | output = src.format_info() 133 | assert "Notes" not in output 134 | 135 | 136 | def test_format_info_wraps_long_values_with_indent(src, mocker): 137 | notes = ( 138 | "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do " 139 | "eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim " 140 | "ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut " 141 | "aliquip ex ea commodo consequat." 142 | ) 143 | src.notes = mocker.MagicMock(return_value=notes) 144 | output = src.format_info(total_width=60) 145 | assert output == ( 146 | "Notes : Lorem ipsum dolor sit amet, consectetur\n" 147 | " adipiscing elit, sed do eiusmod tempor\n" 148 | " incididunt ut labore et dolore magna aliqua.\n" 149 | " Ut enim ad minim veniam, quis nostrud\n" 150 | " exercitation ullamco laboris nisi ut aliquip\n" 151 | " ex ea commodo consequat." 152 | ) 153 | 154 | 155 | def test_format_info_newline_handling(src, mocker): 156 | notes = ( 157 | "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do " 158 | "eiusmod tempor incididunt ut labore.\n" 159 | "Ut enim ad minim veniam.\n" 160 | "\n" 161 | "Quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea " 162 | "commodo consequat." 163 | ) 164 | src.notes = mocker.MagicMock(return_value=notes) 165 | output = src.format_info(total_width=60) 166 | assert output == ( 167 | "Notes : Lorem ipsum dolor sit amet, consectetur\n" 168 | " adipiscing elit, sed do eiusmod tempor\n" 169 | " incididunt ut labore.\n" 170 | " Ut enim ad minim veniam.\n" 171 | "\n" 172 | " Quis nostrud exercitation ullamco laboris nisi\n" 173 | " ut aliquip ex ea commodo consequat." 174 | ) 175 | 176 | 177 | def test_format_info_does_not_wrap_source_url(src, mocker): 178 | url = "https://www.example.com/longlonglonglonglonglonglonglong/" 179 | src.source_url = mocker.MagicMock(return_value=url) 180 | output = src.format_info(total_width=60) 181 | assert output == ( 182 | "URL : https://www.example.com/longlonglonglonglonglonglonglong/" 183 | ) 184 | -------------------------------------------------------------------------------- /src/pricehist/sources/yahoo.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import json 3 | import logging 4 | from datetime import datetime, timezone 5 | from decimal import Decimal 6 | 7 | import requests 8 | 9 | from pricehist import __version__, exceptions 10 | from pricehist.price import Price 11 | 12 | from .basesource import BaseSource 13 | 14 | 15 | class Yahoo(BaseSource): 16 | def id(self): 17 | return "yahoo" 18 | 19 | def name(self): 20 | return "Yahoo! Finance" 21 | 22 | def description(self): 23 | return ( 24 | "Historical data for most Yahoo! Finance symbols, " 25 | "as available on the web page" 26 | ) 27 | 28 | def source_url(self): 29 | return "https://finance.yahoo.com/" 30 | 31 | def start(self): 32 | # The "Download historical data in Yahoo Finance" page says 33 | # "Historical prices usually don't go back earlier than 1970", but 34 | # several do. Examples going back to 1962-01-02 include ED and IBM. 35 | return "1962-01-02" 36 | 37 | def types(self): 38 | return ["adjclose", "open", "high", "low", "close", "mid"] 39 | 40 | def notes(self): 41 | return ( 42 | "Yahoo! Finance decommissioned its historical data API in 2017 but " 43 | "some historical data is available via its web page, as described in: " 44 | "https://help.yahoo.com/kb/" 45 | "download-historical-data-yahoo-finance-sln2311.html\n" 46 | f"{self._symbols_message()}\n" 47 | "In output the base and quote will be the Yahoo! symbol and its " 48 | "corresponding currency. Some symbols include the name of the quote " 49 | "currency (e.g. BTC-USD), so you may wish to use --fmt-base to " 50 | "remove the redundant information.\n" 51 | "When a symbol's historical data is unavilable due to data licensing " 52 | "restrictions, its web page will show no download button and " 53 | "pricehist will only find the current day's price." 54 | ) 55 | 56 | def _symbols_message(self): 57 | return ( 58 | "Find the symbol of interest on https://finance.yahoo.com/ and use " 59 | "that as the PAIR in your pricehist command. Prices for each symbol " 60 | "are quoted in its native currency." 61 | ) 62 | 63 | def symbols(self): 64 | logging.info(self._symbols_message()) 65 | return [] 66 | 67 | def fetch(self, series): 68 | if series.quote: 69 | raise exceptions.InvalidPair( 70 | series.base, series.quote, self, "Don't specify the quote currency." 71 | ) 72 | 73 | data = self._data(series) 74 | quote = data["chart"]["result"][0]["meta"]["currency"] 75 | offset = data["chart"]["result"][0]["meta"]["gmtoffset"] 76 | 77 | timestamps = data["chart"]["result"][0]["timestamp"] 78 | adjclose_data = data["chart"]["result"][0]["indicators"]["adjclose"][0] 79 | rest_data = data["chart"]["result"][0]["indicators"]["quote"][0] 80 | amounts = {**adjclose_data, **rest_data} 81 | 82 | prices = [ 83 | Price(date, amount) 84 | for i in range(len(timestamps)) 85 | if (date := self._ts_to_date(timestamps[i] + offset)) <= series.end 86 | if (amount := self._amount(amounts, series.type, i)) is not None 87 | ] 88 | 89 | return dataclasses.replace(series, quote=quote, prices=prices) 90 | 91 | def _ts_to_date(self, ts) -> str: 92 | return datetime.fromtimestamp(ts, tz=timezone.utc).date().isoformat() 93 | 94 | def _amount(self, amounts, type, i): 95 | if ( 96 | type == "mid" 97 | and amounts["high"] != "null" 98 | and amounts["high"] is not None 99 | and amounts["low"] != "null" 100 | and amounts["low"] is not None 101 | ): 102 | return sum([Decimal(amounts["high"][i]), Decimal(amounts["low"][i])]) / 2 103 | elif amounts[type] != "null" and amounts[type][i] is not None: 104 | return Decimal(amounts[type][i]) 105 | else: 106 | return None 107 | 108 | def _data(self, series) -> dict: 109 | base_url = "https://query1.finance.yahoo.com/v8/finance/chart" 110 | headers = {"User-Agent": f"pricehist/{__version__}"} 111 | url = f"{base_url}/{series.base}" 112 | 113 | start_ts = int( 114 | datetime.strptime(series.start, "%Y-%m-%d") 115 | .replace(tzinfo=timezone.utc) 116 | .timestamp() 117 | ) 118 | end_ts = int( 119 | datetime.strptime(series.end, "%Y-%m-%d") 120 | .replace(tzinfo=timezone.utc) 121 | .timestamp() 122 | ) + ( 123 | 24 * 60 * 60 124 | ) # some symbols require padding on the end timestamp 125 | 126 | params = { 127 | "symbol": series.base, 128 | "period1": start_ts, 129 | "period2": end_ts, 130 | "interval": "1d", 131 | "events": "capitalGain%7Cdiv%7Csplit", 132 | "includeAdjustedClose": "true", 133 | "formatted": "true", 134 | "userYfid": "true", 135 | "lang": "en-US", 136 | "region": "US", 137 | } 138 | 139 | try: 140 | response = self.log_curl(requests.get(url, params=params, headers=headers)) 141 | except Exception as e: 142 | raise exceptions.RequestError(str(e)) from e 143 | 144 | code = response.status_code 145 | text = response.text 146 | 147 | if code == 404 and "No data found, symbol may be delisted" in text: 148 | raise exceptions.InvalidPair( 149 | series.base, series.quote, self, "Symbol not found." 150 | ) 151 | elif code == 400 and "Data doesn't exist" in text: 152 | raise exceptions.BadResponse( 153 | "No data for the given interval. Try requesting a larger interval." 154 | ) 155 | elif code == 404 and "Timestamp data missing" in text: 156 | raise exceptions.BadResponse( 157 | "Data missing. The given interval may be for a gap in the data " 158 | "such as a weekend or holiday. Try requesting a larger interval." 159 | ) 160 | 161 | try: 162 | response.raise_for_status() 163 | except Exception as e: 164 | raise exceptions.BadResponse(str(e)) from e 165 | 166 | try: 167 | data = json.loads(response.content, parse_float=Decimal) 168 | except Exception as e: 169 | raise exceptions.ResponseParsingError( 170 | "The data couldn't be parsed. " 171 | ) from e 172 | 173 | if "timestamp" not in data["chart"]["result"][0]: 174 | raise exceptions.BadResponse( 175 | "No data for the given interval. " 176 | "There may be a problem with the symbol or the interval." 177 | ) 178 | 179 | return data 180 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_ecb.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from datetime import datetime, timedelta 4 | from decimal import Decimal 5 | from pathlib import Path 6 | 7 | import pytest 8 | import requests 9 | import responses 10 | 11 | from pricehist import exceptions, isocurrencies 12 | from pricehist.price import Price 13 | from pricehist.series import Series 14 | from pricehist.sources.ecb import ECB 15 | 16 | 17 | @pytest.fixture 18 | def src(): 19 | return ECB() 20 | 21 | 22 | @pytest.fixture 23 | def type(src): 24 | return src.types()[0] 25 | 26 | 27 | @pytest.fixture 28 | def url(): 29 | return "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist.xml" 30 | 31 | 32 | @pytest.fixture 33 | def url_90d(): 34 | return "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist-90d.xml" 35 | 36 | 37 | @pytest.fixture 38 | def xml(): 39 | dir = Path(os.path.splitext(__file__)[0]) 40 | return (dir / "eurofxref-hist-partial.xml").read_text() 41 | 42 | 43 | @pytest.fixture 44 | def requests_mock(): 45 | with responses.RequestsMock() as mock: 46 | yield mock 47 | 48 | 49 | @pytest.fixture 50 | def response_ok(requests_mock, url, xml): 51 | requests_mock.add(responses.GET, url, body=xml, status=200) 52 | yield requests_mock 53 | 54 | 55 | @pytest.fixture 56 | def response_ok_90d(requests_mock, url_90d, xml): 57 | requests_mock.add(responses.GET, url_90d, body=xml, status=200) 58 | yield requests_mock 59 | 60 | 61 | @pytest.fixture 62 | def response_empty_xml(requests_mock, url): 63 | empty_xml = ( 64 | Path(os.path.splitext(__file__)[0]) / "eurofxref-hist-empty.xml" 65 | ).read_text() 66 | requests_mock.add(responses.GET, url, body=empty_xml, status=200) 67 | yield requests_mock 68 | 69 | 70 | def test_normalizesymbol(src): 71 | assert src.normalizesymbol("eur") == "EUR" 72 | assert src.normalizesymbol("symbol") == "SYMBOL" 73 | 74 | 75 | def test_metadata(src): 76 | assert isinstance(src.id(), str) 77 | assert len(src.id()) > 0 78 | 79 | assert isinstance(src.name(), str) 80 | assert len(src.name()) > 0 81 | 82 | assert isinstance(src.description(), str) 83 | assert len(src.description()) > 0 84 | 85 | assert isinstance(src.source_url(), str) 86 | assert src.source_url().startswith("http") 87 | 88 | assert datetime.strptime(src.start(), "%Y-%m-%d") 89 | 90 | assert isinstance(src.types(), list) 91 | assert len(src.types()) > 0 92 | assert isinstance(src.types()[0], str) 93 | assert len(src.types()[0]) > 0 94 | 95 | assert isinstance(src.notes(), str) 96 | 97 | 98 | def test_symbols(src, response_ok): 99 | syms = src.symbols() 100 | assert ("EUR/AUD", "Euro against Australian Dollar") in syms 101 | assert len(syms) > 40 102 | 103 | 104 | def test_symbols_requests_logged_for(src, response_ok, caplog): 105 | with caplog.at_level(logging.DEBUG): 106 | src.symbols() 107 | assert any( 108 | ["DEBUG" == r.levelname and "curl " in r.message for r in caplog.records] 109 | ) 110 | 111 | 112 | def test_symbols_not_in_iso_data(src, response_ok, monkeypatch): 113 | iso = isocurrencies.by_code() 114 | del iso["AUD"] 115 | monkeypatch.setattr(isocurrencies, "by_code", lambda: iso) 116 | syms = src.symbols() 117 | assert ("EUR/AUD", "Euro against AUD") in syms 118 | 119 | 120 | def test_symbols_not_found(src, response_empty_xml): 121 | with pytest.raises(exceptions.ResponseParsingError) as e: 122 | src.symbols() 123 | assert "data not found" in str(e.value) 124 | 125 | 126 | def test_fetch_known_pair(src, type, response_ok): 127 | series = src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) 128 | assert series.prices[0] == Price("2021-01-04", Decimal("1.5928")) 129 | assert series.prices[-1] == Price("2021-01-08", Decimal("1.5758")) 130 | assert len(series.prices) == 5 131 | 132 | 133 | def test_fetch_requests_logged(src, response_ok, caplog): 134 | with caplog.at_level(logging.DEBUG): 135 | src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) 136 | assert any( 137 | ["DEBUG" == r.levelname and "curl " in r.message for r in caplog.records] 138 | ) 139 | 140 | 141 | def test_fetch_recent_interval_uses_90d_data(src, type, response_ok_90d): 142 | today = datetime.now().date() 143 | start = (today - timedelta(days=80)).isoformat() 144 | end = today.isoformat() 145 | src.fetch(Series("EUR", "AUD", type, start, end)) 146 | assert len(response_ok_90d.calls) > 0 147 | 148 | 149 | def test_fetch_long_hist_from_start(src, type, response_ok): 150 | series = src.fetch(Series("EUR", "AUD", type, src.start(), "2021-01-08")) 151 | assert series.prices[0] == Price("1999-01-04", Decimal("1.91")) 152 | assert series.prices[-1] == Price("2021-01-08", Decimal("1.5758")) 153 | assert len(series.prices) > 9 154 | 155 | 156 | def test_fetch_from_before_start(src, type, response_ok): 157 | series = src.fetch(Series("EUR", "AUD", type, "1998-12-01", "1999-01-10")) 158 | assert series.prices[0] == Price("1999-01-04", Decimal("1.91")) 159 | assert series.prices[-1] == Price("1999-01-08", Decimal("1.8406")) 160 | assert len(series.prices) == 5 161 | 162 | 163 | def test_fetch_to_future(src, type, response_ok): 164 | series = src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2100-01-01")) 165 | assert len(series.prices) > 0 166 | 167 | 168 | def test_fetch_known_pair_no_data(src, type, response_ok): 169 | series = src.fetch(Series("EUR", "ROL", type, "2021-01-04", "2021-02-08")) 170 | assert len(series.prices) == 0 171 | 172 | 173 | def test_fetch_non_eur_base(src, type): 174 | with pytest.raises(exceptions.InvalidPair): 175 | src.fetch(Series("USD", "AUD", type, "2021-01-04", "2021-01-08")) 176 | 177 | 178 | def test_fetch_unknown_quote(src, type, response_ok): 179 | with pytest.raises(exceptions.InvalidPair): 180 | src.fetch(Series("EUR", "XZY", type, "2021-01-04", "2021-01-08")) 181 | 182 | 183 | def test_fetch_no_quote(src, type): 184 | with pytest.raises(exceptions.InvalidPair): 185 | src.fetch(Series("EUR", "", type, "2021-01-04", "2021-01-08")) 186 | 187 | 188 | def test_fetch_unknown_pair(src, type): 189 | with pytest.raises(exceptions.InvalidPair): 190 | src.fetch(Series("ABC", "XZY", type, "2021-01-04", "2021-01-08")) 191 | 192 | 193 | def test_fetch_network_issue(src, type, requests_mock, url): 194 | err = requests.exceptions.ConnectionError("Network issue") 195 | requests_mock.add(responses.GET, url, body=err) 196 | with pytest.raises(exceptions.RequestError) as e: 197 | src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) 198 | assert "Network issue" in str(e.value) 199 | 200 | 201 | def test_fetch_bad_status(src, type, requests_mock, url): 202 | requests_mock.add(responses.GET, url, status=500) 203 | with pytest.raises(exceptions.BadResponse) as e: 204 | src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) 205 | assert "Server Error" in str(e.value) 206 | 207 | 208 | def test_fetch_parsing_error(src, type, requests_mock, url): 209 | requests_mock.add(responses.GET, url, body="NOT XML") 210 | with pytest.raises(exceptions.ResponseParsingError) as e: 211 | src.fetch(Series("EUR", "AUD", type, "2021-01-04", "2021-01-08")) 212 | assert "while parsing data" in str(e.value) 213 | -------------------------------------------------------------------------------- /src/pricehist/outputs/gnucashsql.py: -------------------------------------------------------------------------------- 1 | """ 2 | GnuCash SQL output 3 | 4 | Support for the `GnuCash `_ accounting program is 5 | achieved by generating SQL that can later be applied to a GnuCash database. 6 | 7 | This allows pricehist to support GnuCash with simple text output rather than by 8 | depending on GnuCash Python bindings or direct database interaction. 9 | 10 | The generated SQL can be run in SQLite, MariaDB/MySQL or PostgreSQL. 11 | 12 | Rows in GnuCash's prices table must include GUIDs for the related commodities. 13 | The generated SQL selects the relevant GUIDs by mnemonic from the commodities 14 | table and stores them in a temporary table. Another temprary table is populated 15 | with new price data and the two are joined to produce the new rows that are 16 | inserted into the prices table. 17 | 18 | Users need to ensure that the base and quote of the new prices already have 19 | commodities with matching mnemonics in the GnuCash database. If this condition 20 | is not met, the SQL will fail without making changes. The names of the base and 21 | quote can be adjusted with pricehist formatting options in case the source and 22 | GnuCash names don't already match. Other formatting options can adjust date 23 | formatting and the time of day used. 24 | 25 | Each row in the prices table has a GUID of its own. These are generated in 26 | pricehist by hashing the price data, so the same GUID will always be used for a 27 | given date, base, quote, source, type & amount. Existing GUIDs are skipped 28 | during the final insert into the prices table, so there's no problem with 29 | running one SQL file multiple times or running multiple SQL files with 30 | overlapping data. 31 | 32 | Warnings are generated when string escaping or number limit issues are detected 33 | and it should be easy for users to avoid those issues. 34 | 35 | Classes: 36 | 37 | GnuCashSQL 38 | 39 | """ 40 | 41 | import hashlib 42 | import logging 43 | from datetime import datetime, timezone 44 | from decimal import Decimal 45 | from importlib.resources import files 46 | 47 | from pricehist import __version__ 48 | from pricehist.format import Format 49 | 50 | from .baseoutput import BaseOutput 51 | 52 | 53 | class GnuCashSQL(BaseOutput): 54 | def format(self, series, source, fmt=Format()): 55 | base = fmt.base or series.base 56 | quote = fmt.quote or series.quote 57 | src = source.id() 58 | 59 | self._warn_about_backslashes( 60 | { 61 | "date": fmt.format_date("1970-01-01"), 62 | "time": fmt.time, 63 | "base": base, 64 | "quote": quote, 65 | "source": src, 66 | "price type": series.type, 67 | } 68 | ) 69 | 70 | too_big = False 71 | values_parts = [] 72 | for price in series.prices: 73 | date = f"{fmt.format_date(price.date)} {fmt.time}" 74 | m = hashlib.sha256() 75 | m.update( 76 | "".join( 77 | [ 78 | date, 79 | base, 80 | quote, 81 | src, 82 | series.type, 83 | str(price.amount), 84 | ] 85 | ).encode("utf-8") 86 | ) 87 | guid = m.hexdigest()[0:32] 88 | 89 | value_num, value_denom, fit = self._rational(price.amount) 90 | too_big |= not fit 91 | v = ( 92 | "(" 93 | + ", ".join( 94 | [ 95 | self._sql_str(guid), 96 | self._sql_str(date), 97 | self._sql_str(base), 98 | self._sql_str(quote), 99 | self._sql_str(src), 100 | self._sql_str(series.type), 101 | str(value_num), 102 | str(value_denom), 103 | ] 104 | ) 105 | + ")" 106 | ) 107 | values_parts.append(v) 108 | values = ",\n".join(values_parts) 109 | values_comment = "" if values_parts else "-- " 110 | 111 | if too_big: 112 | # https://code.gnucash.org/docs/MAINT/group__Numeric.html 113 | # https://code.gnucash.org/docs/MAINT/structgnc__price__s.html 114 | logging.warning( 115 | "This SQL contains numbers outside of the int64 range required " 116 | "by GnuCash for the numerators and denominators of prices. " 117 | "Using the --quantize option to limit the number of decimal " 118 | "places will usually reduce the size of the rational form as " 119 | "well." 120 | ) 121 | 122 | sql = ( 123 | files("pricehist.resources") 124 | .joinpath("gnucash.sql") 125 | .read_text() 126 | .format( 127 | version=__version__, 128 | timestamp=datetime.now(timezone.utc).isoformat()[:-6] + "Z", 129 | base=self._sql_str(base), 130 | quote=self._sql_str(quote), 131 | values_comment=values_comment, 132 | values=values, 133 | ) 134 | ) 135 | 136 | return sql 137 | 138 | def _warn_about_backslashes(self, fields): 139 | hits = [name for name, value in fields.items() if "\\" in value] 140 | if hits: 141 | logging.warning( 142 | f"Before running this SQL, check the formatting of the " 143 | f"{self._english_join(hits)} strings. " 144 | f"SQLite treats backslashes in strings as plain characters, but " 145 | f"MariaDB/MySQL and PostgreSQL may interpret them as escape " 146 | f"codes." 147 | ) 148 | 149 | def _english_join(self, strings): 150 | if len(strings) == 0: 151 | return "" 152 | elif len(strings) == 1: 153 | return str(strings[0]) 154 | else: 155 | return f"{', '.join(strings[0:-1])} and {strings[-1]}" 156 | 157 | def _sql_str(self, s): 158 | # Documentation regarding SQL string literals: 159 | # * https://www.sqlite.org/lang_expr.html#literal_values_constants_ 160 | # * https://mariadb.com/kb/en/string-literals/ 161 | # * https://dev.mysql.com/doc/refman/8.0/en/string-literals.html 162 | # * https://www.postgresql.org/docs/devel/sql-syntax-lexical.html 163 | escaped = s.replace("'", "''") 164 | quoted = f"'{escaped}'" 165 | return quoted 166 | 167 | def _rational(self, number: Decimal) -> (str, str, bool): 168 | tup = number.as_tuple() 169 | sign = "-" if tup.sign == 1 else "" 170 | if tup.exponent > 0: 171 | numerator = ( 172 | sign + "".join([str(d) for d in tup.digits]) + ("0" * tup.exponent) 173 | ) 174 | denom = str(1) 175 | else: 176 | numerator = sign + "".join([str(d) for d in tup.digits]) 177 | denom = str(10**-tup.exponent) 178 | fit = self._fit_in_int64(Decimal(numerator), Decimal(denom)) 179 | return (numerator, denom, fit) 180 | 181 | def _fit_in_int64(self, *numbers): 182 | return all(n >= -(2**63) and n <= (2**63) - 1 for n in numbers) 183 | -------------------------------------------------------------------------------- /tests/pricehist/test_fetch.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from datetime import date, timedelta 3 | from decimal import Decimal 4 | 5 | import pytest 6 | 7 | from pricehist import exceptions 8 | from pricehist.fetch import fetch 9 | from pricehist.format import Format 10 | from pricehist.price import Price 11 | from pricehist.series import Series 12 | from pricehist.sources.basesource import BaseSource 13 | 14 | 15 | @pytest.fixture 16 | def res_series(mocker): 17 | series = mocker.MagicMock() 18 | series.start = "2021-01-01" 19 | series.end = "2021-01-03" 20 | return series 21 | 22 | 23 | @pytest.fixture 24 | def source(res_series, mocker): 25 | source = mocker.MagicMock(BaseSource) 26 | source.start = mocker.MagicMock(return_value="2021-01-01") 27 | source.fetch = mocker.MagicMock(return_value=res_series) 28 | return source 29 | 30 | 31 | @pytest.fixture 32 | def output(mocker): 33 | output = mocker.MagicMock() 34 | output.format = mocker.MagicMock(return_value="") 35 | return output 36 | 37 | 38 | @pytest.fixture 39 | def fmt(): 40 | return Format() 41 | 42 | 43 | def test_fetch_warns_if_start_before_source_start(source, output, fmt, mocker, caplog): 44 | req_series = Series("BTC", "EUR", "close", "2020-12-31", "2021-01-03") 45 | source.start = mocker.MagicMock(return_value="2021-01-01") 46 | with caplog.at_level(logging.INFO): 47 | fetch(req_series, source, output, invert=False, quantize=None, fmt=fmt) 48 | assert any( 49 | [ 50 | "WARNING" == r.levelname and "start date 2020-12-31 preceeds" in r.message 51 | for r in caplog.records 52 | ] 53 | ) 54 | 55 | 56 | def test_fetch_returns_formatted_output(source, res_series, output, fmt, mocker): 57 | req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") 58 | output.format = mocker.MagicMock(return_value="rendered output") 59 | 60 | result = fetch(req_series, source, output, invert=False, quantize=None, fmt=fmt) 61 | 62 | output.format.assert_called_once_with(res_series, source, fmt=fmt) 63 | assert result == "rendered output" 64 | 65 | 66 | def test_fetch_inverts_if_requested(source, res_series, output, fmt, mocker): 67 | req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") 68 | inv_series = mocker.MagicMock() 69 | res_series.invert = mocker.MagicMock(return_value=inv_series) 70 | 71 | fetch(req_series, source, output, invert=True, quantize=None, fmt=fmt) 72 | 73 | res_series.invert.assert_called_once_with() 74 | output.format.assert_called_once_with(inv_series, source, fmt=fmt) 75 | 76 | 77 | def test_fetch_quantizes_if_requested(source, res_series, output, fmt, mocker): 78 | req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") 79 | qnt_series = mocker.MagicMock() 80 | res_series.quantize = mocker.MagicMock(return_value=qnt_series) 81 | 82 | fetch(req_series, source, output, invert=False, quantize=2, fmt=fmt) 83 | 84 | res_series.quantize.assert_called_once_with(2) 85 | output.format.assert_called_once_with(qnt_series, source, fmt=fmt) 86 | 87 | 88 | def test_fetch_warns_if_no_data(source, res_series, output, fmt, mocker, caplog): 89 | req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") 90 | res_series.prices = mocker.MagicMock(return_value=[]) 91 | with caplog.at_level(logging.INFO): 92 | fetch(req_series, source, output, invert=False, quantize=None, fmt=fmt) 93 | assert any( 94 | [ 95 | "WARNING" == r.levelname and "No data found" in r.message 96 | for r in caplog.records 97 | ] 98 | ) 99 | 100 | 101 | def test_fetch_warns_if_missing_data_at_start(source, res_series, output, fmt, caplog): 102 | req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") 103 | res_series.prices = [ 104 | Price("2021-01-02", Decimal("1.2")), 105 | Price("2021-01-03", Decimal("1.3")), 106 | ] 107 | with caplog.at_level(logging.INFO): 108 | fetch(req_series, source, output, invert=False, quantize=None, fmt=fmt) 109 | r = caplog.records[0] 110 | assert r.levelname == "WARNING" 111 | assert r.message == ( 112 | "Available data covers the interval [2021-01-02--2021-01-03], " 113 | "which starts 1 day later than requested." 114 | ) 115 | 116 | 117 | def test_fetch_warns_if_missing_data_at_end(source, res_series, output, fmt, caplog): 118 | req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") 119 | res_series.prices = [Price("2021-01-01", Decimal("1.1"))] 120 | with caplog.at_level(logging.INFO): 121 | fetch(req_series, source, output, invert=False, quantize=None, fmt=fmt) 122 | r = caplog.records[0] 123 | assert r.levelname == "WARNING" 124 | assert r.message == ( 125 | "Available data covers the interval [2021-01-01--2021-01-01], " 126 | "which ends 2 days earlier than requested." 127 | ) 128 | 129 | 130 | def test_fetch_warns_if_missing_data_at_both_ends( 131 | source, res_series, output, fmt, caplog 132 | ): 133 | req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") 134 | res_series.prices = [Price("2021-01-02", Decimal("1.2"))] 135 | with caplog.at_level(logging.INFO): 136 | fetch(req_series, source, output, invert=False, quantize=None, fmt=fmt) 137 | r = caplog.records[0] 138 | assert r.levelname == "WARNING" 139 | assert r.message == ( 140 | "Available data covers the interval [2021-01-02--2021-01-02], " 141 | "which starts 1 day later and ends 1 day earlier than requested." 142 | ) 143 | 144 | 145 | def test_fetch_debug_not_warning_message_if_only_today_missing( 146 | source, res_series, output, fmt, caplog 147 | ): 148 | start = (date.today() - timedelta(days=2)).isoformat() 149 | yesterday = (date.today() - timedelta(days=1)).isoformat() 150 | today = date.today().isoformat() 151 | req_series = Series("BTC", "EUR", "close", start, today) 152 | res_series.start = start 153 | res_series.end = today 154 | res_series.prices = [Price(start, Decimal("1.1")), Price(yesterday, Decimal("1.2"))] 155 | with caplog.at_level(logging.DEBUG): 156 | fetch(req_series, source, output, invert=False, quantize=None, fmt=fmt) 157 | r = caplog.records[0] 158 | assert r.levelname == "DEBUG" 159 | assert r.message == ( 160 | f"Available data covers the interval [{start}--{yesterday}], " 161 | "which ends 1 day earlier than requested." 162 | ) 163 | 164 | 165 | def test_fetch_debug_not_warning_message_if_as_requested( 166 | source, res_series, output, fmt, caplog 167 | ): 168 | req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") 169 | res_series.prices = [ 170 | Price("2021-01-01", Decimal("1.1")), 171 | Price("2021-01-02", Decimal("1.2")), 172 | Price("2021-01-03", Decimal("1.3")), 173 | ] 174 | with caplog.at_level(logging.DEBUG): 175 | fetch(req_series, source, output, invert=False, quantize=None, fmt=fmt) 176 | r = caplog.records[0] 177 | assert r.levelname == "DEBUG" 178 | assert r.message == ( 179 | "Available data covers the interval [2021-01-01--2021-01-03], as requested." 180 | ) 181 | 182 | 183 | def test_fetch_handles_source_exceptions(source, output, fmt, mocker, caplog): 184 | req_series = Series("BTC", "EUR", "close", "2021-01-01", "2021-01-03") 185 | 186 | def side_effect(_): 187 | raise exceptions.RequestError("something strange") 188 | 189 | source.fetch = mocker.MagicMock(side_effect=side_effect) 190 | 191 | with caplog.at_level(logging.INFO): 192 | with pytest.raises(SystemExit) as e: 193 | fetch(req_series, source, output, invert=False, quantize=None, fmt=fmt) 194 | 195 | r = caplog.records[0] 196 | assert r.levelname == "CRITICAL" 197 | assert "something strange" in r.message 198 | 199 | assert e.value.code == 1 200 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_yahoo/ibm-long-partial.json: -------------------------------------------------------------------------------- 1 | { 2 | "chart": { 3 | "result": [ 4 | { 5 | "meta": { 6 | "currency": "USD", 7 | "symbol": "IBM", 8 | "exchangeName": "NYQ", 9 | "fullExchangeName": "NYSE", 10 | "instrumentType": "EQUITY", 11 | "firstTradeDate": -252322200, 12 | "regularMarketTime": 1726257602, 13 | "hasPrePostMarketData": true, 14 | "gmtoffset": -14400, 15 | "timezone": "EDT", 16 | "exchangeTimezoneName": "America/New_York", 17 | "regularMarketPrice": 214.79, 18 | "fiftyTwoWeekHigh": 216.08, 19 | "fiftyTwoWeekLow": 212.13, 20 | "regularMarketDayHigh": 216.08, 21 | "regularMarketDayLow": 212.13, 22 | "regularMarketVolume": 4553547, 23 | "longName": "International Business Machines Corporation", 24 | "shortName": "International Business Machines", 25 | "chartPreviousClose": 7.291, 26 | "priceHint": 2, 27 | "currentTradingPeriod": { 28 | "pre": { 29 | "timezone": "EDT", 30 | "end": 1726234200, 31 | "start": 1726214400, 32 | "gmtoffset": -14400 33 | }, 34 | "regular": { 35 | "timezone": "EDT", 36 | "end": 1726257600, 37 | "start": 1726234200, 38 | "gmtoffset": -14400 39 | }, 40 | "post": { 41 | "timezone": "EDT", 42 | "end": 1726272000, 43 | "start": 1726257600, 44 | "gmtoffset": -14400 45 | } 46 | }, 47 | "dataGranularity": "1d", 48 | "range": "", 49 | "validRanges": [ 50 | "1d", 51 | "5d", 52 | "1mo", 53 | "3mo", 54 | "6mo", 55 | "1y", 56 | "2y", 57 | "5y", 58 | "10y", 59 | "ytd", 60 | "max" 61 | ] 62 | }, 63 | "timestamp": [ 64 | -252322200, 65 | -252235800, 66 | -252149400, 67 | -252063000, 68 | -251803800, 69 | 1609770600, 70 | 1609857000, 71 | 1609943400, 72 | 1610029800, 73 | 1610116200 74 | ], 75 | "events": { 76 | "dividends": { 77 | "-249298200": { 78 | "amount": 0.000956, 79 | "date": -249298200 80 | }, 81 | "-241439400": { 82 | "amount": 0.000956, 83 | "date": -241439400 84 | }, 85 | "-233577000": { 86 | "amount": 0.000956, 87 | "date": -233577000 88 | }, 89 | "-225797400": { 90 | "amount": 0.000956, 91 | "date": -225797400 92 | }, 93 | "-217848600": { 94 | "amount": 0.001275, 95 | "date": -217848600 96 | }, 97 | "1573137000": { 98 | "amount": 1.548757, 99 | "date": 1573137000 100 | }, 101 | "1581085800": { 102 | "amount": 1.548757, 103 | "date": 1581085800 104 | }, 105 | "1588858200": { 106 | "amount": 1.558317, 107 | "date": 1588858200 108 | }, 109 | "1596807000": { 110 | "amount": 1.558317, 111 | "date": 1596807000 112 | }, 113 | "1604932200": { 114 | "amount": 1.558317, 115 | "date": 1604932200 116 | } 117 | }, 118 | "splits": { 119 | "-177417000": { 120 | "date": -177417000, 121 | "numerator": 5.0, 122 | "denominator": 4.0, 123 | "splitRatio": "5:4" 124 | }, 125 | "-114345000": { 126 | "date": -114345000, 127 | "numerator": 3.0, 128 | "denominator": 2.0, 129 | "splitRatio": "3:2" 130 | }, 131 | "-53343000": { 132 | "date": -53343000, 133 | "numerator": 2.0, 134 | "denominator": 1.0, 135 | "splitRatio": "2:1" 136 | }, 137 | "107530200": { 138 | "date": 107530200, 139 | "numerator": 5.0, 140 | "denominator": 4.0, 141 | "splitRatio": "5:4" 142 | }, 143 | "297091800": { 144 | "date": 297091800, 145 | "numerator": 4.0, 146 | "denominator": 1.0, 147 | "splitRatio": "4:1" 148 | }, 149 | "864826200": { 150 | "date": 864826200, 151 | "numerator": 2.0, 152 | "denominator": 1.0, 153 | "splitRatio": "2:1" 154 | }, 155 | "927811800": { 156 | "date": 927811800, 157 | "numerator": 2.0, 158 | "denominator": 1.0, 159 | "splitRatio": "2:1" 160 | } 161 | } 162 | }, 163 | "indicators": { 164 | "quote": [ 165 | { 166 | "close": [ 167 | 7.2912678718566895, 168 | 7.3550028800964355, 169 | 7.281707763671875, 170 | 7.138305187225342, 171 | 7.00446081161499, 172 | 118.48948669433594, 173 | 120.59273529052734, 174 | 123.60420989990234, 175 | 123.31739807128906, 176 | 122.87763214111328 177 | ], 178 | "low": [ 179 | 7.2912678718566895, 180 | 7.2912678718566895, 181 | 7.2785210609436035, 182 | 7.125557899475098, 183 | 6.9471001625061035, 184 | 117.62906646728516, 185 | 119.13002014160156, 186 | 121.14722442626953, 187 | 122.61949920654297, 188 | 121.39579010009766 189 | ], 190 | "open": [ 191 | 7.374124050140381, 192 | 7.2912678718566895, 193 | 7.3550028800964355, 194 | 7.272148132324219, 195 | 7.131930828094482, 196 | 120.31549072265625, 197 | 119.5124282836914, 198 | 121.3193130493164, 199 | 124.32122039794922, 200 | 122.9158706665039 201 | ], 202 | "high": [ 203 | 7.374124050140381, 204 | 7.3550028800964355, 205 | 7.3550028800964355, 206 | 7.272148132324219, 207 | 7.131930828094482, 208 | 120.38240814208984, 209 | 121.1089859008789, 210 | 126.08030700683594, 211 | 124.7227554321289, 212 | 123.63288879394531 213 | ], 214 | "volume": [ 215 | 407940, 216 | 305955, 217 | 274575, 218 | 384405, 219 | 572685, 220 | 5417443, 221 | 6395872, 222 | 8322708, 223 | 4714740, 224 | 4891305 225 | ] 226 | } 227 | ], 228 | "adjclose": [ 229 | { 230 | "adjclose": [ 231 | 1.5133211612701416, 232 | 1.5265485048294067, 233 | 1.5113375186920166, 234 | 1.4815733432769775, 235 | 1.4537923336029053, 236 | 99.60364532470703, 237 | 101.37164306640625, 238 | 103.90313720703125, 239 | 103.66202545166016, 240 | 103.29237365722656 241 | ] 242 | } 243 | ] 244 | } 245 | } 246 | ], 247 | "error": null 248 | } 249 | } 250 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_bankofcanada.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from datetime import datetime 4 | from decimal import Decimal 5 | from pathlib import Path 6 | 7 | import pytest 8 | import requests 9 | import responses 10 | 11 | from pricehist import exceptions 12 | from pricehist.price import Price 13 | from pricehist.series import Series 14 | from pricehist.sources.bankofcanada import BankOfCanada 15 | 16 | 17 | @pytest.fixture 18 | def src(): 19 | return BankOfCanada() 20 | 21 | 22 | @pytest.fixture 23 | def type(src): 24 | return src.types()[0] 25 | 26 | 27 | @pytest.fixture 28 | def requests_mock(): 29 | with responses.RequestsMock() as mock: 30 | yield mock 31 | 32 | 33 | @pytest.fixture 34 | def series_list_url(): 35 | return "https://www.bankofcanada.ca/valet/lists/series/json" 36 | 37 | 38 | def fetch_url(series_name): 39 | return f"https://www.bankofcanada.ca/valet/observations/{series_name}/json" 40 | 41 | 42 | @pytest.fixture 43 | def series_list_json(): 44 | dir = Path(os.path.splitext(__file__)[0]) 45 | return (dir / "series-partial.json").read_text() 46 | 47 | 48 | @pytest.fixture 49 | def series_list_response_ok(requests_mock, series_list_url, series_list_json): 50 | requests_mock.add(responses.GET, series_list_url, body=series_list_json, status=200) 51 | yield requests_mock 52 | 53 | 54 | @pytest.fixture 55 | def recent_response_ok(requests_mock): 56 | json = (Path(os.path.splitext(__file__)[0]) / "recent.json").read_text() 57 | requests_mock.add(responses.GET, fetch_url("FXCADUSD"), body=json, status=200) 58 | yield requests_mock 59 | 60 | 61 | @pytest.fixture 62 | def all_response_ok(requests_mock): 63 | json = (Path(os.path.splitext(__file__)[0]) / "all-partial.json").read_text() 64 | requests_mock.add(responses.GET, fetch_url("FXCADUSD"), body=json, status=200) 65 | yield requests_mock 66 | 67 | 68 | def test_normalizesymbol(src): 69 | assert src.normalizesymbol("cad") == "CAD" 70 | assert src.normalizesymbol("usd") == "USD" 71 | 72 | 73 | def test_metadata(src): 74 | assert isinstance(src.id(), str) 75 | assert len(src.id()) > 0 76 | 77 | assert isinstance(src.name(), str) 78 | assert len(src.name()) > 0 79 | 80 | assert isinstance(src.description(), str) 81 | assert len(src.description()) > 0 82 | 83 | assert isinstance(src.source_url(), str) 84 | assert src.source_url().startswith("http") 85 | 86 | assert datetime.strptime(src.start(), "%Y-%m-%d") 87 | 88 | assert isinstance(src.types(), list) 89 | assert len(src.types()) > 0 90 | assert isinstance(src.types()[0], str) 91 | assert len(src.types()[0]) > 0 92 | 93 | assert isinstance(src.notes(), str) 94 | 95 | 96 | def test_symbols(src, series_list_response_ok): 97 | syms = src.symbols() 98 | assert ("CAD/USD", "Canadian dollar to US dollar daily exchange rate") in syms 99 | assert len(syms) > 3 100 | 101 | 102 | def test_symbols_requests_logged(src, series_list_response_ok, caplog): 103 | with caplog.at_level(logging.DEBUG): 104 | src.symbols() 105 | assert any( 106 | ["DEBUG" == r.levelname and "curl " in r.message for r in caplog.records] 107 | ) 108 | 109 | 110 | def test_symbols_not_found(src, requests_mock, series_list_url): 111 | requests_mock.add(responses.GET, series_list_url, body='{"series":{}}', status=200) 112 | with pytest.raises(exceptions.ResponseParsingError) as e: 113 | src.symbols() 114 | assert "data not found" in str(e.value) 115 | 116 | 117 | def test_symbols_network_issue(src, requests_mock, series_list_url): 118 | requests_mock.add( 119 | responses.GET, 120 | series_list_url, 121 | body=requests.exceptions.ConnectionError("Network issue"), 122 | ) 123 | with pytest.raises(exceptions.RequestError) as e: 124 | src.symbols() 125 | assert "Network issue" in str(e.value) 126 | 127 | 128 | def test_symbols_bad_status(src, requests_mock, series_list_url): 129 | requests_mock.add(responses.GET, series_list_url, status=500) 130 | with pytest.raises(exceptions.BadResponse) as e: 131 | src.symbols() 132 | assert "Server Error" in str(e.value) 133 | 134 | 135 | def test_symbols_parsing_error(src, requests_mock, series_list_url): 136 | requests_mock.add(responses.GET, series_list_url, body="NOT JSON") 137 | with pytest.raises(exceptions.ResponseParsingError) as e: 138 | src.symbols() 139 | assert "while parsing data" in str(e.value) 140 | 141 | 142 | def test_fetch_known_pair(src, type, recent_response_ok): 143 | series = src.fetch(Series("CAD", "USD", type, "2021-01-01", "2021-01-07")) 144 | req = recent_response_ok.calls[0].request 145 | assert req.params["order_dir"] == "asc" 146 | assert req.params["start_date"] == "2021-01-01" 147 | assert req.params["end_date"] == "2021-01-07" 148 | assert series.prices[0] == Price("2021-01-04", Decimal("0.7843")) 149 | assert series.prices[-1] == Price("2021-01-07", Decimal("0.7870")) 150 | assert len(series.prices) == 4 151 | 152 | 153 | def test_fetch_requests_logged(src, type, recent_response_ok, caplog): 154 | with caplog.at_level(logging.DEBUG): 155 | src.fetch(Series("CAD", "USD", type, "2021-01-01", "2021-01-07")) 156 | assert any( 157 | ["DEBUG" == r.levelname and "curl " in r.message for r in caplog.records] 158 | ) 159 | 160 | 161 | def test_fetch_long_hist_from_start(src, type, all_response_ok): 162 | series = src.fetch(Series("CAD", "USD", type, src.start(), "2021-01-07")) 163 | assert series.prices[0] == Price("2017-01-03", Decimal("0.7443")) 164 | assert series.prices[-1] == Price("2021-01-07", Decimal("0.7870")) 165 | assert len(series.prices) > 13 166 | 167 | 168 | def test_fetch_from_before_start(src, type, requests_mock): 169 | body = """{ "observations": [] }""" 170 | requests_mock.add(responses.GET, fetch_url("FXCADUSD"), status=200, body=body) 171 | series = src.fetch(Series("CAD", "USD", type, "2000-01-01", "2017-01-01")) 172 | assert len(series.prices) == 0 173 | 174 | 175 | def test_fetch_to_future(src, type, all_response_ok): 176 | series = src.fetch(Series("CAD", "USD", type, "2021-01-01", "2100-01-01")) 177 | assert len(series.prices) > 0 178 | 179 | 180 | def test_wrong_dates_order(src, type, requests_mock): 181 | body = """{ "message": "The End date must be greater than the Start date." }""" 182 | requests_mock.add(responses.GET, fetch_url("FXCADUSD"), status=400, body=body) 183 | with pytest.raises(exceptions.BadResponse) as e: 184 | src.fetch(Series("CAD", "USD", type, "2021-01-07", "2021-01-01")) 185 | assert "End date must be greater" in str(e.value) 186 | 187 | 188 | def test_fetch_in_future(src, type, requests_mock): 189 | body = """{ "observations": [] }""" 190 | requests_mock.add(responses.GET, fetch_url("FXCADUSD"), status=200, body=body) 191 | series = src.fetch(Series("CAD", "USD", type, "2030-01-01", "2030-01-07")) 192 | assert len(series.prices) == 0 193 | 194 | 195 | def test_fetch_empty(src, type, requests_mock): 196 | requests_mock.add( 197 | responses.GET, fetch_url("FXCADUSD"), body="""{"observations":{}}""" 198 | ) 199 | series = src.fetch(Series("CAD", "USD", type, "2021-01-03", "2021-01-03")) 200 | assert len(series.prices) == 0 201 | 202 | 203 | def test_fetch_no_quote(src, type): 204 | with pytest.raises(exceptions.InvalidPair): 205 | src.fetch(Series("CAD", "", type, "2021-01-01", "2021-01-07")) 206 | 207 | 208 | def test_fetch_unknown_pair(src, type, requests_mock): 209 | requests_mock.add( 210 | responses.GET, 211 | fetch_url("FXCADAFN"), 212 | status=404, 213 | body="""{ 214 | "message": "Series FXCADAFN not found.", 215 | "docs": "https://www.bankofcanada.ca/valet/docs" 216 | }""", 217 | ) 218 | with pytest.raises(exceptions.InvalidPair): 219 | src.fetch(Series("CAD", "AFN", type, "2021-01-01", "2021-01-07")) 220 | 221 | 222 | def test_fetch_network_issue(src, type, requests_mock): 223 | body = requests.exceptions.ConnectionError("Network issue") 224 | requests_mock.add(responses.GET, fetch_url("FXCADUSD"), body=body) 225 | with pytest.raises(exceptions.RequestError) as e: 226 | src.fetch(Series("CAD", "USD", type, "2021-01-01", "2021-01-07")) 227 | assert "Network issue" in str(e.value) 228 | 229 | 230 | def test_fetch_bad_status(src, type, requests_mock): 231 | requests_mock.add( 232 | responses.GET, 233 | fetch_url("FXCADUSD"), 234 | status=500, 235 | body="""{"message": "Some other reason"}""", 236 | ) 237 | with pytest.raises(exceptions.BadResponse) as e: 238 | src.fetch(Series("CAD", "USD", type, "2021-01-01", "2021-01-07")) 239 | assert "Internal Server Error" in str(e.value) 240 | 241 | 242 | def test_fetch_parsing_error(src, type, requests_mock): 243 | requests_mock.add(responses.GET, fetch_url("FXCADUSD"), body="NOT JSON") 244 | with pytest.raises(exceptions.ResponseParsingError) as e: 245 | src.fetch(Series("CAD", "USD", type, "2021-01-01", "2021-01-07")) 246 | assert "while parsing data" in str(e.value) 247 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_coindesk.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from datetime import datetime 4 | from decimal import Decimal 5 | from pathlib import Path 6 | 7 | import pytest 8 | import requests 9 | import responses 10 | 11 | from pricehist import exceptions 12 | from pricehist.price import Price 13 | from pricehist.series import Series 14 | from pricehist.sources.coindesk import CoinDesk 15 | 16 | 17 | @pytest.fixture 18 | def src(): 19 | return CoinDesk() 20 | 21 | 22 | @pytest.fixture 23 | def type(src): 24 | return src.types()[0] 25 | 26 | 27 | @pytest.fixture 28 | def requests_mock(): 29 | with responses.RequestsMock() as mock: 30 | yield mock 31 | 32 | 33 | @pytest.fixture 34 | def currencies_url(): 35 | return "https://api.coindesk.com/v1/bpi/supported-currencies.json" 36 | 37 | 38 | @pytest.fixture 39 | def fetch_url(): 40 | return "https://api.coindesk.com/v1/bpi/historical/close.json" 41 | 42 | 43 | @pytest.fixture 44 | def currencies_json(): 45 | dir = Path(os.path.splitext(__file__)[0]) 46 | return (dir / "supported-currencies-partial.json").read_text() 47 | 48 | 49 | @pytest.fixture 50 | def currencies_response_ok(requests_mock, currencies_url, currencies_json): 51 | requests_mock.add(responses.GET, currencies_url, body=currencies_json, status=200) 52 | yield requests_mock 53 | 54 | 55 | @pytest.fixture 56 | def recent_response_ok(requests_mock, fetch_url): 57 | json = (Path(os.path.splitext(__file__)[0]) / "recent.json").read_text() 58 | requests_mock.add(responses.GET, fetch_url, body=json, status=200) 59 | yield requests_mock 60 | 61 | 62 | @pytest.fixture 63 | def all_response_ok(requests_mock, fetch_url): 64 | json = (Path(os.path.splitext(__file__)[0]) / "all-partial.json").read_text() 65 | requests_mock.add(responses.GET, fetch_url, body=json, status=200) 66 | yield requests_mock 67 | 68 | 69 | @pytest.fixture 70 | def not_found_response(requests_mock, fetch_url): 71 | requests_mock.add( 72 | responses.GET, 73 | fetch_url, 74 | status=404, 75 | body="Sorry, that currency was not found", 76 | ) 77 | 78 | 79 | def test_normalizesymbol(src): 80 | assert src.normalizesymbol("btc") == "BTC" 81 | assert src.normalizesymbol("usd") == "USD" 82 | 83 | 84 | def test_metadata(src): 85 | assert isinstance(src.id(), str) 86 | assert len(src.id()) > 0 87 | 88 | assert isinstance(src.name(), str) 89 | assert len(src.name()) > 0 90 | 91 | assert isinstance(src.description(), str) 92 | assert len(src.description()) > 0 93 | 94 | assert isinstance(src.source_url(), str) 95 | assert src.source_url().startswith("http") 96 | 97 | assert datetime.strptime(src.start(), "%Y-%m-%d") 98 | 99 | assert isinstance(src.types(), list) 100 | assert len(src.types()) > 0 101 | assert isinstance(src.types()[0], str) 102 | assert len(src.types()[0]) > 0 103 | 104 | assert isinstance(src.notes(), str) 105 | 106 | 107 | def test_symbols(src, currencies_response_ok): 108 | syms = src.symbols() 109 | assert ("BTC/AUD", "Bitcoin against Australian Dollar") in syms 110 | assert len(syms) > 3 111 | 112 | 113 | def test_symbols_requests_logged(src, currencies_response_ok, caplog): 114 | with caplog.at_level(logging.DEBUG): 115 | src.symbols() 116 | assert any( 117 | ["DEBUG" == r.levelname and "curl " in r.message for r in caplog.records] 118 | ) 119 | 120 | 121 | def test_symbols_not_found(src, requests_mock, currencies_url): 122 | requests_mock.add(responses.GET, currencies_url, body="[]", status=200) 123 | with pytest.raises(exceptions.ResponseParsingError) as e: 124 | src.symbols() 125 | assert "data not found" in str(e.value) 126 | 127 | 128 | def test_symbols_network_issue(src, requests_mock, currencies_url): 129 | requests_mock.add( 130 | responses.GET, 131 | currencies_url, 132 | body=requests.exceptions.ConnectionError("Network issue"), 133 | ) 134 | with pytest.raises(exceptions.RequestError) as e: 135 | src.symbols() 136 | assert "Network issue" in str(e.value) 137 | 138 | 139 | def test_symbols_bad_status(src, requests_mock, currencies_url): 140 | requests_mock.add(responses.GET, currencies_url, status=500) 141 | with pytest.raises(exceptions.BadResponse) as e: 142 | src.symbols() 143 | assert "Server Error" in str(e.value) 144 | 145 | 146 | def test_symbols_parsing_error(src, requests_mock, currencies_url): 147 | requests_mock.add(responses.GET, currencies_url, body="NOT JSON") 148 | with pytest.raises(exceptions.ResponseParsingError) as e: 149 | src.symbols() 150 | assert "while parsing data" in str(e.value) 151 | 152 | 153 | def test_fetch_known_pair(src, type, recent_response_ok): 154 | series = src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) 155 | req = recent_response_ok.calls[0].request 156 | assert req.params["currency"] == "AUD" 157 | assert req.params["start"] == "2021-01-01" 158 | assert req.params["end"] == "2021-01-07" 159 | assert series.prices[0] == Price("2021-01-01", Decimal("38204.8987")) 160 | assert series.prices[-1] == Price("2021-01-07", Decimal("50862.227")) 161 | assert len(series.prices) == 7 162 | 163 | 164 | def test_fetch_requests_logged(src, type, recent_response_ok, caplog): 165 | with caplog.at_level(logging.DEBUG): 166 | src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) 167 | assert any( 168 | ["DEBUG" == r.levelname and "curl " in r.message for r in caplog.records] 169 | ) 170 | 171 | 172 | def test_fetch_long_hist_from_start(src, type, all_response_ok): 173 | series = src.fetch(Series("BTC", "AUD", type, src.start(), "2021-01-07")) 174 | assert series.prices[0] == Price("2010-07-18", Decimal("0.0984")) 175 | assert series.prices[-1] == Price("2021-01-07", Decimal("50862.227")) 176 | assert len(series.prices) > 13 177 | 178 | 179 | def test_fetch_from_before_start(src, type, requests_mock, fetch_url): 180 | body = "Sorry, the CoinDesk BPI only covers data from 2010-07-17 onwards." 181 | requests_mock.add(responses.GET, fetch_url, status=404, body=body) 182 | with pytest.raises(exceptions.BadResponse) as e: 183 | src.fetch(Series("BTC", "AUD", type, "2010-01-01", "2010-07-24")) 184 | assert "only covers data from" in str(e.value) 185 | 186 | 187 | def test_fetch_to_future(src, type, all_response_ok): 188 | series = src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2100-01-01")) 189 | assert len(series.prices) > 0 190 | 191 | 192 | def test_wrong_dates_order(src, type, requests_mock, fetch_url): 193 | body = "Sorry, but your specified end date is before your start date." 194 | requests_mock.add(responses.GET, fetch_url, status=404, body=body) 195 | with pytest.raises(exceptions.BadResponse) as e: 196 | src.fetch(Series("BTC", "AUD", type, "2021-01-07", "2021-01-01")) 197 | assert "End date is before start date." in str(e.value) 198 | 199 | 200 | def test_fetch_in_future(src, type, requests_mock, fetch_url): 201 | body = "Sorry, but your specified end date is before your start date." 202 | requests_mock.add(responses.GET, fetch_url, status=404, body=body) 203 | with pytest.raises(exceptions.BadResponse) as e: 204 | src.fetch(Series("BTC", "AUD", type, "2030-01-01", "2030-01-07")) 205 | assert "start date must be in the past" in str(e.value) 206 | 207 | 208 | def test_fetch_empty(src, type, requests_mock, fetch_url): 209 | requests_mock.add(responses.GET, fetch_url, body="{}") 210 | series = src.fetch(Series("BTC", "AUD", type, "2010-07-17", "2010-07-17")) 211 | assert len(series.prices) == 0 212 | 213 | 214 | def test_fetch_known_pair_no_data(src, type, requests_mock, fetch_url): 215 | body = "No results returned from database" 216 | requests_mock.add(responses.GET, fetch_url, status=500, body=body) 217 | with pytest.raises(exceptions.BadResponse) as e: 218 | src.fetch(Series("BTC", "CUP", type, "2010-07-17", "2010-07-23")) 219 | assert "No results returned from database" in str(e.value) 220 | 221 | 222 | def test_fetch_non_btc_base(src, type): 223 | with pytest.raises(exceptions.InvalidPair): 224 | src.fetch(Series("USD", "AUD", type, "2021-01-01", "2021-01-07")) 225 | 226 | 227 | def test_fetch_unknown_quote(src, type, not_found_response): 228 | with pytest.raises(exceptions.InvalidPair): 229 | src.fetch(Series("BTC", "XZY", type, "2021-01-01", "2021-01-07")) 230 | 231 | 232 | def test_fetch_no_quote(src, type, not_found_response): 233 | with pytest.raises(exceptions.InvalidPair): 234 | src.fetch(Series("BTC", "", type, "2021-01-01", "2021-01-07")) 235 | 236 | 237 | def test_fetch_unknown_pair(src, type): 238 | with pytest.raises(exceptions.InvalidPair): 239 | src.fetch(Series("ABC", "XZY", type, "2021-01-01", "2021-01-07")) 240 | 241 | 242 | def test_fetch_network_issue(src, type, requests_mock, fetch_url): 243 | body = requests.exceptions.ConnectionError("Network issue") 244 | requests_mock.add(responses.GET, fetch_url, body=body) 245 | with pytest.raises(exceptions.RequestError) as e: 246 | src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) 247 | assert "Network issue" in str(e.value) 248 | 249 | 250 | def test_fetch_bad_status(src, type, requests_mock, fetch_url): 251 | requests_mock.add(responses.GET, fetch_url, status=500, body="Some other reason") 252 | with pytest.raises(exceptions.BadResponse) as e: 253 | src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) 254 | assert "Internal Server Error" in str(e.value) 255 | 256 | 257 | def test_fetch_parsing_error(src, type, requests_mock, fetch_url): 258 | requests_mock.add(responses.GET, fetch_url, body="NOT JSON") 259 | with pytest.raises(exceptions.ResponseParsingError) as e: 260 | src.fetch(Series("BTC", "AUD", type, "2021-01-01", "2021-01-07")) 261 | assert "while parsing data" in str(e.value) 262 | -------------------------------------------------------------------------------- /tests/pricehist/sources/test_yahoo.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from datetime import datetime, timezone 4 | from decimal import Decimal 5 | from pathlib import Path 6 | 7 | import pytest 8 | import requests 9 | import responses 10 | 11 | from pricehist import exceptions 12 | from pricehist.price import Price 13 | from pricehist.series import Series 14 | from pricehist.sources.yahoo import Yahoo 15 | 16 | 17 | def timestamp(date): 18 | return int( 19 | datetime.strptime(date, "%Y-%m-%d").replace(tzinfo=timezone.utc).timestamp() 20 | ) 21 | 22 | 23 | @pytest.fixture 24 | def src(): 25 | return Yahoo() 26 | 27 | 28 | @pytest.fixture 29 | def type(src): 30 | return src.types()[0] 31 | 32 | 33 | @pytest.fixture 34 | def requests_mock(): 35 | with responses.RequestsMock() as mock: 36 | yield mock 37 | 38 | 39 | def url(base): 40 | return f"https://query1.finance.yahoo.com/v8/finance/chart/{base}" 41 | 42 | 43 | @pytest.fixture 44 | def recent_ok(requests_mock): 45 | json = (Path(os.path.splitext(__file__)[0]) / "tsla-recent.json").read_text() 46 | requests_mock.add(responses.GET, url("TSLA"), body=json, status=200) 47 | yield requests_mock 48 | 49 | 50 | @pytest.fixture 51 | def long_ok(requests_mock): 52 | json = (Path(os.path.splitext(__file__)[0]) / "ibm-long-partial.json").read_text() 53 | requests_mock.add(responses.GET, url("IBM"), body=json, status=200) 54 | yield requests_mock 55 | 56 | 57 | @pytest.fixture 58 | def with_null_ok(requests_mock): 59 | json = (Path(os.path.splitext(__file__)[0]) / "inrx-with-null.json").read_text() 60 | requests_mock.add(responses.GET, url("INR=X"), body=json, status=200) 61 | yield requests_mock 62 | 63 | 64 | def test_normalizesymbol(src): 65 | assert src.normalizesymbol("tsla") == "TSLA" 66 | 67 | 68 | def test_metadata(src): 69 | assert isinstance(src.id(), str) 70 | assert len(src.id()) > 0 71 | 72 | assert isinstance(src.name(), str) 73 | assert len(src.name()) > 0 74 | 75 | assert isinstance(src.description(), str) 76 | assert len(src.description()) > 0 77 | 78 | assert isinstance(src.source_url(), str) 79 | assert src.source_url().startswith("http") 80 | 81 | assert datetime.strptime(src.start(), "%Y-%m-%d") 82 | 83 | assert isinstance(src.types(), list) 84 | assert len(src.types()) > 0 85 | assert isinstance(src.types()[0], str) 86 | assert len(src.types()[0]) > 0 87 | 88 | assert isinstance(src.notes(), str) 89 | 90 | 91 | def test_symbols(src, caplog): 92 | with caplog.at_level(logging.INFO): 93 | symbols = src.symbols() 94 | assert symbols == [] 95 | assert any(["Find the symbol of interest on" in r.message for r in caplog.records]) 96 | 97 | 98 | def test_fetch_known(src, type, recent_ok): 99 | series = src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) 100 | req = recent_ok.calls[0].request 101 | assert req.params["events"] == "capitalGain%7Cdiv%7Csplit" 102 | assert req.params["includeAdjustedClose"] == "true" 103 | assert (series.base, series.quote) == ("TSLA", "USD") 104 | assert len(series.prices) == 5 105 | 106 | 107 | def test_fetch_requests_and_receives_correct_times(src, type, recent_ok): 108 | series = src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) 109 | req = recent_ok.calls[0].request 110 | assert req.params["period1"] == str(timestamp("2021-01-04")) 111 | assert req.params["period2"] == str(timestamp("2021-01-09")) # rounded up one 112 | assert req.params["interval"] == "1d" 113 | assert series.prices[0] == Price("2021-01-04", Decimal("243.2566680908203")) 114 | assert series.prices[-1] == Price("2021-01-08", Decimal("293.3399963378906")) 115 | 116 | 117 | def test_fetch_ignores_any_extra_row(src, type, recent_ok): 118 | series = src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-07")) 119 | assert series.prices[0] == Price("2021-01-04", Decimal("243.2566680908203")) 120 | assert series.prices[-1] == Price("2021-01-07", Decimal("272.0133361816406")) 121 | 122 | 123 | def test_fetch_requests_logged(src, type, recent_ok, caplog): 124 | with caplog.at_level(logging.DEBUG): 125 | src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) 126 | logged_requests = 0 127 | for r in caplog.records: 128 | if r.levelname == "DEBUG" and "curl " in r.message: 129 | logged_requests += 1 130 | assert logged_requests == 1 131 | 132 | 133 | def test_fetch_types_all_available(src, recent_ok): 134 | adj = src.fetch(Series("TSLA", "", "adjclose", "2021-01-04", "2021-01-08")) 135 | opn = src.fetch(Series("TSLA", "", "open", "2021-01-04", "2021-01-08")) 136 | hgh = src.fetch(Series("TSLA", "", "high", "2021-01-04", "2021-01-08")) 137 | low = src.fetch(Series("TSLA", "", "low", "2021-01-04", "2021-01-08")) 138 | cls = src.fetch(Series("TSLA", "", "close", "2021-01-04", "2021-01-08")) 139 | mid = src.fetch(Series("TSLA", "", "mid", "2021-01-04", "2021-01-08")) 140 | assert adj.prices[0].amount == Decimal("243.2566680908203") 141 | assert opn.prices[0].amount == Decimal("239.82000732421875") 142 | assert hgh.prices[0].amount == Decimal("248.163330078125") 143 | assert low.prices[0].amount == Decimal("239.06333923339844") 144 | assert cls.prices[0].amount == Decimal("243.2566680908203") 145 | assert mid.prices[0].amount == Decimal("243.61333465576172") 146 | 147 | 148 | def test_fetch_type_mid_is_mean_of_low_and_high(src, recent_ok): 149 | mid = src.fetch(Series("TSLA", "", "mid", "2021-01-04", "2021-01-08")).prices 150 | hgh = src.fetch(Series("TSLA", "", "high", "2021-01-04", "2021-01-08")).prices 151 | low = src.fetch(Series("TSLA", "", "low", "2021-01-04", "2021-01-08")).prices 152 | assert all( 153 | [ 154 | mid[i].amount == (sum([low[i].amount, hgh[i].amount]) / 2) 155 | for i in range(0, 5) 156 | ] 157 | ) 158 | 159 | 160 | def test_fetch_from_before_start(src, type, long_ok): 161 | series = src.fetch(Series("IBM", "", type, "1900-01-01", "2021-01-08")) 162 | assert series.prices[0] == Price("1962-01-02", Decimal("1.5133211612701416")) 163 | assert series.prices[-1] == Price("2021-01-08", Decimal("103.29237365722656")) 164 | assert len(series.prices) > 9 165 | 166 | 167 | def test_fetch_skips_dates_with_nulls(src, type, with_null_ok): 168 | series = src.fetch(Series("INR=X", "", type, "2017-07-10", "2017-07-12")) 169 | assert series.prices[0] == Price("2017-07-10", Decimal("64.61170196533203")) 170 | assert series.prices[1] == Price("2017-07-12", Decimal("64.52559661865234")) 171 | assert len(series.prices) == 2 172 | 173 | 174 | def test_fetch_to_future(src, type, recent_ok): 175 | series = src.fetch(Series("TSLA", "", type, "2021-01-04", "2100-01-08")) 176 | assert len(series.prices) > 0 177 | 178 | 179 | def test_fetch_no_data_in_past(src, type, requests_mock): 180 | requests_mock.add( 181 | responses.GET, 182 | url("TSLA"), 183 | status=400, 184 | body=( 185 | "400 Bad Request: Data doesn't exist for " 186 | "startDate = 1262304000, endDate = 1262995200" 187 | ), 188 | ) 189 | with pytest.raises(exceptions.BadResponse) as e: 190 | src.fetch(Series("TSLA", "", type, "2010-01-04", "2010-01-08")) 191 | assert "No data for the given interval" in str(e.value) 192 | 193 | 194 | def test_fetch_no_data_in_future(src, type, requests_mock): 195 | requests_mock.add( 196 | responses.GET, 197 | url("TSLA"), 198 | status=400, 199 | body=( 200 | "400 Bad Request: Data doesn't exist for " 201 | "startDate = 1893715200, endDate = 1894147200" 202 | ), 203 | ) 204 | with pytest.raises(exceptions.BadResponse) as e: 205 | src.fetch(Series("TSLA", "", type, "2030-01-04", "2030-01-08")) 206 | assert "No data for the given interval" in str(e.value) 207 | 208 | 209 | def test_fetch_no_data_on_weekend(src, type, requests_mock): 210 | requests_mock.add( 211 | responses.GET, 212 | url("TSLA"), 213 | status=404, 214 | body="404 Not Found: Timestamp data missing.", 215 | ) 216 | with pytest.raises(exceptions.BadResponse) as e: 217 | src.fetch(Series("TSLA", "", type, "2021-01-09", "2021-01-10")) 218 | assert "may be for a gap in the data" in str(e.value) 219 | 220 | 221 | def test_fetch_bad_sym(src, type, requests_mock): 222 | requests_mock.add( 223 | responses.GET, 224 | url("NOTABASE"), 225 | status=404, 226 | body="404 Not Found: No data found, symbol may be delisted", 227 | ) 228 | with pytest.raises(exceptions.InvalidPair) as e: 229 | src.fetch(Series("NOTABASE", "", type, "2021-01-04", "2021-01-08")) 230 | assert "Symbol not found" in str(e.value) 231 | 232 | 233 | def test_fetch_giving_quote(src, type): 234 | with pytest.raises(exceptions.InvalidPair) as e: 235 | src.fetch(Series("TSLA", "USD", type, "2021-01-04", "2021-01-08")) 236 | assert "quote currency" in str(e.value) 237 | 238 | 239 | def test_fetch_network_issue(src, type, requests_mock): 240 | body = requests.exceptions.ConnectionError("Network issue") 241 | requests_mock.add(responses.GET, url("TSLA"), body=body) 242 | with pytest.raises(exceptions.RequestError) as e: 243 | src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) 244 | assert "Network issue" in str(e.value) 245 | 246 | 247 | def test_fetch_bad_status(src, type, requests_mock): 248 | requests_mock.add(responses.GET, url("TSLA"), status=500, body="Some other reason") 249 | with pytest.raises(exceptions.BadResponse) as e: 250 | src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) 251 | assert "Internal Server Error" in str(e.value) 252 | 253 | 254 | def test_fetch_parsing_error(src, type, requests_mock): 255 | requests_mock.add(responses.GET, url("TSLA"), body="") 256 | with pytest.raises(exceptions.ResponseParsingError) as e: 257 | src.fetch(Series("TSLA", "", type, "2021-01-04", "2021-01-08")) 258 | assert "error occurred while parsing data from the source" in str(e.value) 259 | --------------------------------------------------------------------------------