├── .editorconfig ├── .github └── workflows │ └── beancount.yml ├── .gitignore ├── .pylintrc ├── COPYING ├── README.md ├── beanprice ├── BUILD ├── __init__.py ├── date_utils.py ├── date_utils_test.py ├── net_utils.py ├── net_utils_test.py ├── price.py ├── price_test.py ├── source.py └── sources │ ├── BUILD │ ├── __init__.py │ ├── alphavantage.py │ ├── alphavantage_test.py │ ├── coinbase.py │ ├── coinbase_test.py │ ├── coincap.py │ ├── coincap_test.py │ ├── coinmarketcap.py │ ├── coinmarketcap_test.py │ ├── eastmoneyfund.py │ ├── eastmoneyfund_test.py │ ├── ecbrates.py │ ├── ecbrates_test.py │ ├── iex.py │ ├── iex_test.py │ ├── oanda.py │ ├── oanda_test.py │ ├── quandl.py │ ├── quandl_test.py │ ├── ratesapi.py │ ├── ratesapi_test.py │ ├── tsp.py │ ├── tsp_test.py │ ├── yahoo.py │ └── yahoo_test.py ├── bin ├── BUILD └── bean-price ├── etc └── env ├── experiments └── dividends │ └── download_dividends.py ├── pyproject.toml ├── requirements.txt ├── requirements_dev.txt └── setup.py /.editorconfig: -------------------------------------------------------------------------------- 1 | # Editor configuration, see http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | charset = utf-8 6 | indent_style = space 7 | insert_final_newline = true 8 | trim_trailing_whitespace = true 9 | 10 | [*.py] 11 | indent_size = 4 12 | 13 | [*.yml] 14 | indent_size = 2 15 | -------------------------------------------------------------------------------- /.github/workflows/beancount.yml: -------------------------------------------------------------------------------- 1 | name: beancount 2 | 3 | on: 4 | [push, pull_request] 5 | 6 | jobs: 7 | test: 8 | runs-on: ubuntu-latest 9 | strategy: 10 | fail-fast: false 11 | steps: 12 | - uses: actions/checkout@v2 13 | - uses: actions/setup-python@v2 14 | with: 15 | python-version: '3.9' 16 | - run: pip install -r requirements_dev.txt 17 | - run: pylint beanprice 18 | - run: pytest beanprice 19 | - run: mypy beanprice 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.DS_Store 2 | *.a 3 | *.pyc 4 | *~ 5 | .cache 6 | .idea 7 | .noseids 8 | .pytest_cache 9 | TAGS 10 | venv 11 | build 12 | dist 13 | beanprice.egg-info 14 | __pycache__ 15 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # Specify a configuration file. 4 | #rcfile= 5 | 6 | # Python code to execute, usually for sys.path manipulation such as 7 | # pygtk.require(). 8 | #init-hook= 9 | 10 | # Add files or directories to the blacklist. They should be base names, not 11 | # paths. 12 | ignore=.git,BUILD 13 | 14 | # Pickle collected data for later comparisons. 15 | persistent=yes 16 | 17 | # List of plugins (as comma separated values of python modules names) to load, 18 | # usually to register additional checkers. 19 | load-plugins= 20 | 21 | # DEPRECATED 22 | #include-ids=no 23 | 24 | # DEPRECATED 25 | #symbols=no 26 | 27 | 28 | [REPORTS] 29 | 30 | # Set the output format. Available formats are text, parseable, colorized, msvs 31 | # (visual studio) and html. You can also give a reporter class, eg 32 | # mypackage.mymodule.MyReporterClass. 33 | output-format=text 34 | 35 | # Tells whether to display a full report or only the messages 36 | reports=no 37 | 38 | # Python expression which should return a note less than 10 (10 is the highest 39 | # note). You have access to the variables errors warning, statement which 40 | # respectively contain the number of errors / warnings messages and the total 41 | # number of statements analyzed. This is used by the global evaluation report 42 | # (RP0004). 43 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 44 | 45 | # Template used to display messages. This is a python new-style format string 46 | # used to format the message information. See doc for all details 47 | msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" 48 | 49 | 50 | [MESSAGES CONTROL] 51 | 52 | # Enable the message, report, category or checker with the given id(s). You can 53 | # either give multiple identifier separated by comma (,) or put this option 54 | # multiple time. See also the "--disable" option for examples. 55 | 56 | enable=all 57 | 58 | 59 | # Disable the message, report, category or checker with the given id(s). You 60 | # can either give multiple identifiers separated by comma (,) or put this 61 | # option multiple times (only on the command line, not in the configuration 62 | # file where it should appear only once).You can also use "--disable=all" to 63 | # disable everything first and then reenable specific checks. For example, if 64 | # you want to run only the similarities checker, you can use "--disable=all 65 | # --enable=similarities". If you want to run only the classes checker, but have 66 | # no Warning level messages displayed, use"--disable=all --enable=classes 67 | # --disable=W" 68 | 69 | disable=locally-disabled, 70 | suppressed-message, 71 | missing-docstring, 72 | too-many-lines, 73 | multiple-statements, 74 | superfluous-parens, 75 | ungrouped-imports, 76 | wrong-import-position, 77 | no-self-argument, 78 | no-member, 79 | no-value-for-parameter, 80 | too-many-function-args, 81 | unsubscriptable-object, 82 | too-many-nested-blocks, 83 | duplicate-code, 84 | too-few-public-methods, 85 | too-many-public-methods, 86 | too-many-branches, 87 | too-many-arguments, 88 | too-many-locals, 89 | too-many-statements, 90 | attribute-defined-outside-init, 91 | protected-access, 92 | arguments-differ, 93 | abstract-method, 94 | fixme, 95 | global-variable-undefined, 96 | global-statement, 97 | unused-variable, 98 | unused-argument, 99 | redefined-outer-name, 100 | redefined-builtin, 101 | undefined-loop-variable, 102 | broad-except, 103 | logging-format-interpolation, 104 | anomalous-backslash-in-string, 105 | len-as-condition, 106 | no-else-return, 107 | invalid-unary-operand-type, 108 | no-name-in-module, 109 | inconsistent-return-statements, 110 | not-callable, 111 | stop-iteration-return, 112 | assignment-from-no-return, 113 | c-extension-no-member, 114 | cyclic-import, 115 | isinstance-second-argument-not-valid-type, 116 | missing-timeout, 117 | consider-using-f-string, 118 | consider-using-with, 119 | use-implicit-booleaness-not-comparison-to-string, 120 | use-implicit-booleaness-not-comparison-to-zero, 121 | too-many-positional-arguments, 122 | possibly-used-before-assignment, 123 | arguments-renamed 124 | 125 | # Notes: 126 | # bad-continuation: Is buggy, see https://github.com/PyCQA/pylint/issues/3512 127 | 128 | 129 | [VARIABLES] 130 | 131 | # Tells whether we should check for unused import in __init__ files. 132 | init-import=no 133 | 134 | # A regular expression matching the name of dummy variables (i.e. expectedly 135 | # not used). 136 | dummy-variables-rgx=_$|dummy 137 | 138 | # List of additional names supposed to be defined in builtins. Remember that 139 | # you should avoid to define new builtins when possible. 140 | additional-builtins= 141 | 142 | 143 | [TYPECHECK] 144 | 145 | # Tells whether missing members accessed in mixin class should be ignored. A 146 | # mixin class is detected if its name ends with "mixin" (case insensitive). 147 | ignore-mixin-members=yes 148 | 149 | # List of module names for which member attributes should not be checked 150 | # (useful for modules/projects where namespaces are manipulated during runtime 151 | # and thus existing member attributes cannot be deduced by static analysis 152 | ignored-modules= 153 | 154 | # List of classes names for which member attributes should not be checked 155 | # (useful for classes with attributes dynamically set). 156 | ignored-classes= 157 | 158 | # List of members which are set dynamically and missed by pylint inference 159 | # system, and so shouldn't trigger E0201 when accessed. Python regular 160 | # expressions are accepted. 161 | #generated-members=REQUEST,acl_users,aq_parent 162 | generated-members= 163 | 164 | 165 | [LOGGING] 166 | 167 | # Logging modules to check that the string format arguments are in logging 168 | # function parameter format 169 | logging-modules=logging 170 | 171 | 172 | [MISCELLANEOUS] 173 | 174 | # List of note tags to take in consideration, separated by a comma. 175 | notes=FIXME,XXX,TODO 176 | 177 | 178 | [BASIC] 179 | 180 | # Good variable names which should always be accepted, separated by a comma 181 | good-names=f,i,j,k,ex,Run,_ 182 | 183 | # Bad variable names which should always be refused, separated by a comma 184 | bad-names=foo,bar,baz,toto,tutu,tata 185 | 186 | # Colon-delimited sets of names that determine each other's naming style when 187 | # the name regexes allow several styles. 188 | name-group= 189 | 190 | # Include a hint for the correct naming format with invalid-name 191 | include-naming-hint=no 192 | 193 | # Regular expression matching correct module names 194 | module-rgx=(([a-z_][a-z0-9_\-]*)|([A-Z][a-zA-Z0-9]+))$ 195 | 196 | # Regular expression matching correct method names 197 | method-rgx=[a-z_][a-zA-Z0-9_]{2,72}$ 198 | 199 | # Regular expression matching correct class names 200 | class-rgx=[A-Z_][a-zA-Z0-9]+$ 201 | 202 | # Regular expression matching correct class attribute names 203 | class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 204 | 205 | # Regular expression matching correct inline iteration names 206 | inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ 207 | 208 | # Regular expression matching correct variable names 209 | variable-rgx=(_?[a-z_][a-z0-9_]{2,30}|__|mu|no)$ 210 | 211 | # Regular expression matching correct constant names 212 | const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 213 | 214 | # Regular expression matching correct attribute names 215 | attr-rgx=[a-z_][a-z0-9_]{2,30}$ 216 | 217 | # Regular expression matching correct argument names 218 | argument-rgx=(_?[a-z_][a-z0-9_]{2,30}|__|mu)$ 219 | 220 | # Regular expression matching correct function names 221 | function-rgx=_?[a-z_][a-zA-Z0-9_]{2,64}$ 222 | 223 | # Regular expression which should only match function or class names that do 224 | # not require a docstring. 225 | no-docstring-rgx=__.*__ 226 | 227 | # Minimum line length for functions/classes that require docstrings, shorter 228 | # ones are exempt. 229 | docstring-min-length=-1 230 | 231 | 232 | [FORMAT] 233 | 234 | # Maximum number of characters on a single line. 235 | max-line-length=92 236 | 237 | # Regexp for a line that is allowed to be longer than the limit. 238 | ignore-long-lines=^\s*(# )??$ 239 | 240 | # Allow the body of an if to be on the same line as the test if there is no 241 | # else. 242 | single-line-if-stmt=no 243 | 244 | # Maximum number of lines in a module 245 | max-module-lines=1000 246 | 247 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 248 | # tab). 249 | indent-string=' ' 250 | 251 | # Number of spaces of indent required inside a hanging or continued line. 252 | indent-after-paren=4 253 | 254 | 255 | [SIMILARITIES] 256 | 257 | # Minimum lines number of a similarity. 258 | min-similarity-lines=4 259 | 260 | # Ignore comments when computing similarities. 261 | ignore-comments=yes 262 | 263 | # Ignore docstrings when computing similarities. 264 | ignore-docstrings=yes 265 | 266 | # Ignore imports when computing similarities. 267 | ignore-imports=no 268 | 269 | 270 | [DESIGN] 271 | 272 | # Maximum number of arguments for function / method 273 | max-args=5 274 | 275 | # Argument names that match this expression will be ignored. Default to name 276 | # with leading underscore 277 | ignored-argument-names=_.* 278 | 279 | # Maximum number of locals for function / method body 280 | max-locals=20 281 | 282 | # Maximum number of return / yield for function / method body 283 | max-returns=6 284 | 285 | # Maximum number of branch for function / method body 286 | max-branches=12 287 | 288 | # Maximum number of statements in function / method body 289 | max-statements=50 290 | 291 | # Maximum number of parents for a class (see R0901). 292 | max-parents=7 293 | 294 | # Maximum number of attributes for a class (see R0902). 295 | max-attributes=7 296 | 297 | # Minimum number of public methods for a class (see R0903). 298 | min-public-methods=2 299 | 300 | # Maximum number of public methods for a class (see R0904). 301 | max-public-methods=20 302 | 303 | 304 | [IMPORTS] 305 | 306 | # Deprecated modules which should not be used, separated by a comma 307 | deprecated-modules=stringprep,optparse 308 | 309 | # Create a graph of every (i.e. internal and external) dependencies in the 310 | # given file (report RP0402 must not be disabled) 311 | import-graph= 312 | 313 | # Create a graph of external dependencies in the given file (report RP0402 must 314 | # not be disabled) 315 | ext-import-graph= 316 | 317 | # Create a graph of internal dependencies in the given file (report RP0402 must 318 | # not be disabled) 319 | int-import-graph= 320 | 321 | 322 | [CLASSES] 323 | 324 | # List of interface methods to ignore, separated by a comma. This is used for 325 | # instance to not check methods defines in Zope's Interface base class. 326 | ## ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by 327 | 328 | # List of method names used to declare (i.e. assign) instance attributes. 329 | defining-attr-methods=__init__,__new__,setUp 330 | 331 | # List of valid names for the first argument in a class method. 332 | valid-classmethod-first-arg=cls 333 | 334 | # List of valid names for the first argument in a metaclass class method. 335 | valid-metaclass-classmethod-first-arg=mcs 336 | 337 | 338 | [EXCEPTIONS] 339 | 340 | # Exceptions that will emit a warning when being caught. Defaults to 341 | # "Exception" 342 | overgeneral-exceptions=builtins.Exception 343 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | 341 | 342 | ------------------------------------------------------------ 343 | 344 | Note: 345 | This is a GNU GPL "v2 only" license. 346 | This is not a GNU GPL "v2 or any later version" license. 347 | ---Martin Blais 348 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # beanprice: Price quotes fetcher for Beancount 2 | 3 | ## Description 4 | 5 | A script to fetch market data prices from various sources on the internet 6 | and render them for plain text accounting price syntax (and Beancount). 7 | 8 | This used to be located within Beancount itself (at v2) under beancount.prices. 9 | This repo will contain all future updates to that script and to those price 10 | sources. 11 | 12 | ## Documentation 13 | 14 | Some documentation is still part of Beancount. More about how to use this can be 15 | found on that [mailing-list](https://groups.google.com/forum/#!forum/beancount). 16 | Otherwise read the source. 17 | 18 | ## Quick start 19 | 20 | To install beanprice, run: 21 | 22 | ```shell 23 | pip install git+https://github.com/beancount/beanprice.git 24 | ``` 25 | 26 | You can fetch the latest price of a stock by running: 27 | 28 | ```shell 29 | bean-price -e 'USD:yahoo/AAPL' 30 | ``` 31 | 32 | To fetch the latest prices from your beancount file, first ensure that commodities have price metadata, e.g. 33 | 34 | ``` 35 | 2000-01-01 commodity AAPL 36 | price: "USD:yahoo/AAPL" 37 | ``` 38 | 39 | Then run: 40 | 41 | ```shell 42 | bean-price ledger.beancount 43 | ``` 44 | 45 | To update prices up to the present day, run: 46 | 47 | ```shell 48 | bean-price --update ledger.beancount 49 | ``` 50 | 51 | For more detailed guide for price fetching, read . 52 | 53 | 54 | ## Price source info 55 | The following price sources are available: 56 | 57 | | Name | Module | Provides prices for | Base currency | Latest price? | Historical price? | 58 | |-------------------------|---------------------------|-----------------------------------------------------------------------------------|----------------------------------------------------------------------------------|---------------|-------------------| 59 | | Alphavantage | `beanprice.alphavantage` | [Stocks, FX, Crypto](http://alphavantage.co) | Many currencies | ✓ | ✕ | 60 | | Coinbase | `beanprice.coinbase` | [Most common (crypto)currencies](https://api.coinbase.com/v2/exchange-rates) | [Many currencies](https://api.coinbase.com/v2/currencies) | ✓ | ✓ | 61 | | Coincap | `beanprice.coincap` | [Most common (crypto)currencies](https://docs.coincap.io) | USD | ✓ | ✓ | 62 | | Coinmarketcap | `beanprice.coinmarketcap` | [Most common (crypto)currencies](https://coinmarketcap.com/api/documentation/v1/) | Many Currencies | ✓ | ✕ | 63 | | European Central Bank API| `beanprice.ecbrates` | [Many currencies](https://data.ecb.europa.eu/search-results?searchTerm=exchange%20rates) | [Many currencies](https://data.ecb.europa.eu/search-results?searchTerm=exchange%20rates) (Derived from EUR rates)| ✓ | ✓ | 64 | | IEX | `beanprice.iex` | [Trading symbols](https://iextrading.com/trading/eligible-symbols/) | USD | ✓ | 🚧 (Not yet!) | 65 | | OANDA | `beanprice.oanda` | [Many currencies](https://developer.oanda.com/exchange-rates-api/v1/currencies/) | [Many currencies](https://developer.oanda.com/exchange-rates-api/v1/currencies/) | ✓ | ✓ | 66 | | Quandl | `beanprice.quandl` | [Various datasets](https://www.quandl.com/search) | [Various datasets](https://www.quandl.com/search) | ✓ | ✓ | 67 | | Rates API | `beanprice.ratesapi` | [Many currencies](https://api.exchangerate.host/symbols) | [Many currencies](https://api.exchangerate.host/symbols) | ✓ | ✓ | 68 | | Thrift Savings Plan | `beanprice.tsp` | TSP Funds | USD | ✓ | ✓ | 69 | | Yahoo | `beanprice.yahoo` | Many currencies | Many currencies | ✓ | ✓ | 70 | | EastMoneyFund(天天基金) | `beanprice.eastmoneyfund` | [Chinese Funds](http://fund.eastmoney.com/js/fundcode_search.js) | CNY | ✓ | ✓ | 71 | 72 | 73 | More price sources can be found at [awesome-beancount.com](https://awesome-beancount.com/#price-sources) website. 74 | 75 | ## Creating a custom price source 76 | 77 | To create a price source, create a package (i.e. `my_package`) with a module (i.e. `my_module`) that contains the Source class which inherits from the `beanprice.Source` class: 78 | 79 | ```python 80 | from beanprice import source 81 | 82 | class Source(source.Source): 83 | def get_latest_price(self, ticker) -> source.SourcePrice | None: 84 | pass 85 | 86 | def get_historical_price(self, ticker, time): 87 | pass 88 | ``` 89 | Implement the logic for fetching the prices. At a minimum, the `get_latest_price()` is required. 90 | 91 | Then use your price source in the commodities 92 | 93 | ```beancount 94 | 1900-01-01 commodity XYZ 95 | price: "AUD:my_package.my_module/XYZ" 96 | ``` 97 | `AUD` just being an example of a currency specification. 98 | 99 | ## Testing 100 | 101 | Run tests: 102 | 103 | ``` 104 | pytest beanprice 105 | ``` 106 | 107 | Lint: 108 | 109 | ``` 110 | pylint beanprice 111 | ``` 112 | 113 | Type checker: 114 | 115 | ``` 116 | mypy beanprice 117 | ``` 118 | 119 | ## Copyright and License 120 | 121 | Copyright (C) 2007-2020 Martin Blais. All Rights Reserved. 122 | 123 | This code is distributed under the terms of the "GNU GPLv2 only". 124 | See COPYING file for details. 125 | -------------------------------------------------------------------------------- /beanprice/BUILD: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | py_library( 4 | name = "__init__", 5 | srcs = ["__init__.py"], 6 | ) 7 | 8 | 9 | py_library( 10 | name = "net_utils", 11 | srcs = ["net_utils.py"], 12 | ) 13 | 14 | py_test( 15 | name = "net_utils_test", 16 | srcs = ["net_utils_test.py"], 17 | deps = [ 18 | ":net_utils", 19 | ], 20 | ) 21 | 22 | 23 | py_library( 24 | name = "price", 25 | srcs = ["price.py"], 26 | deps = [ 27 | ":date_utils", 28 | "//beancount/core:amount", 29 | "//beancount/core:data", 30 | "//beancount/core:number", 31 | "//beancount/core:prices", 32 | "//beancount/core:getters", 33 | "//beancount/parser:printer", 34 | "//beancount/parser:version", 35 | "//beancount/ops:find_prices", 36 | "//beancount/ops:lifetimes", 37 | "//beancount/prices:__init__", 38 | "//beancount:loader", 39 | ], 40 | ) 41 | 42 | # Again, the problem here is that this code calls the binary. 43 | py_test( 44 | name = "price_test", 45 | srcs = ["price_test.py"], 46 | deps = [ 47 | "//beancount/core:number", 48 | "//beancount:loader", 49 | "//beancount/parser:cmptest", 50 | "//beancount/ops:find_prices", 51 | "//beancount/prices:price", 52 | "//beancount/prices:source", 53 | "//beancount/prices/sources:oanda", 54 | "//beancount/prices/sources:yahoo", 55 | "//beancount/utils:test_utils", 56 | "//beancount:plugins_for_tests", 57 | ], 58 | ) 59 | 60 | py_library( 61 | name = "source", 62 | srcs = ["source.py"], 63 | deps = [ 64 | "//beancount/core:number", 65 | ], 66 | ) 67 | 68 | py_library( 69 | name = "date_utils", 70 | srcs = ["date_utils.py"], 71 | ) 72 | 73 | py_test( 74 | name = "date_utils_test", 75 | srcs = ["date_utils_test.py"], 76 | deps = [ 77 | ":date_utils", 78 | ], 79 | ) 80 | -------------------------------------------------------------------------------- /beanprice/__init__.py: -------------------------------------------------------------------------------- 1 | """Fetch prices from the internet and output them as Beancount price directives. 2 | 3 | This script accepts a list of Beancount input filenames, and fetches prices 4 | required to compute market values for current positions: 5 | 6 | bean-price /home/joe/finances/joe.beancount 7 | 8 | The list of fetching jobs to carry out is derived automatically from the input 9 | file (see section below for full details). It is also possible to provide a list 10 | of specific price fetching jobs to run, e.g., 11 | 12 | bean-price -e google/TSE:XUS yahoo/AAPL mysources.morningstar/RBF1005 13 | 14 | The general format of each of these "source strings" is 15 | 16 | /[^] 17 | 18 | The "module" is the name of a Python module that contains a Source class which 19 | can be instantiated and connect to a data source to extract price data. These 20 | modules are automatically imported by name and instantiated in order to pull the 21 | price from a particular data source. This allows you to write your own 22 | supplementary fetcher codes without having to modify this script. 23 | 24 | Default implementations are provided to provide access to prices from Yahoo! 25 | Finance or Google Finance, which cover a large universe of common public 26 | investment types (e.g. stock tickers). As a convenience, the module name is 27 | always first searched under the "beanprice.sources" package, where those 28 | default source implementations live. This is how, for example, in order to use 29 | the provided Google Finance data fetcher you don't have to write 30 | "beanprice.sources.yahoo/AAPL" but simply "yahoo/AAPL". 31 | 32 | Date 33 | ---- 34 | 35 | By default, this script will fetch prices at the latest available date & time. 36 | You can use an option to fetch historical prices for a desired date instead: 37 | 38 | bean-price --date=2015-02-03 39 | 40 | Inverse 41 | ------- 42 | 43 | Sometimes, prices are available for the inverse of an instrument. This is often 44 | the case for currencies. For example, the price of "CAD" in USD" is provided by 45 | the USD/CAD market, which gives the price of a US dollar in Canadian dollars. In 46 | order specify this, you can prepend "^" to the instrument to instruct the driver 47 | to compute the inverse of the given price: 48 | 49 | bean-price -e USD:google/^CURRENCY:USDCAD 50 | 51 | If a source price is to be inverted, like this, the precision could be different 52 | than what is fetched. For instance, if the price of USD/CAD is 1.32759, it would 53 | output be this from the above directive: 54 | 55 | 2015-10-28 price CAD 0.753244601119 USD 56 | 57 | By default, inverted rates will be rounded similarly to how other Price 58 | directives were rounding those numbers. 59 | 60 | 61 | Swap Inverted 62 | ------------- 63 | 64 | If you prefer to have the output Price entries with swapped currencies instead 65 | of inverting the rate itself, you can use the --swap-inverted option. In the 66 | previous example for the price of CAD, it would output this: 67 | 68 | 2015-10-28 price USD 1.32759 CAD 69 | 70 | This works since the Beancount price database computes and interpolates the 71 | reciprocals automatically for all pairs of commodities in its database. 72 | 73 | 74 | Prices Needed for a Beancount File 75 | ---------------------------------- 76 | 77 | You can also provide a filename to extract the list of tickers to fetch from a 78 | Beancount input file, e.g.: 79 | 80 | bean-price /home/joe/finances/joe.beancount 81 | 82 | There are many ways to extract a list of commodities with needed prices from a 83 | Beancount input file: 84 | 85 | - Prices for all the holdings that were seen held-at-cost at a particular date. 86 | 87 | - Prices for holdings held at a particular date which were price converted from 88 | some other commodity in the past (i.e., for currencies). 89 | 90 | - The list of all Commodity directives present in the file. For each of those 91 | holdings, the corresponding Commodity directive is consulted and its "price" 92 | metadata field is used to specify where to attempt to fetch prices. You should 93 | have directives like this in your input file: 94 | 95 | 2007-07-20 commodity VEA 96 | price: "google/NYSEARCA:VEA" 97 | 98 | The "price" metadata can be a comma-separated list of sources to try out, in 99 | which case each of the sources will be looked at : 100 | 101 | 2007-07-20 commodity VEA 102 | price: "google/CURRENCY:USDCAD,yahoo/USDCAD" 103 | 104 | - Existing price directives for the same data are excluded by default, since the 105 | price is already in the file. 106 | 107 | By default, the list of tickers to be fetched includes only the intersection of 108 | these lists. The general intent of the user of this script is to fetch missing 109 | prices, and only needed ones, for a particular date. 110 | 111 | * Use the --date option to change the applied date. 112 | * Use the --all option to fetch the entire set of prices, regardless 113 | of holdings and date. 114 | * Use --clobber to ignore existing price directives. 115 | 116 | You can also print the list of prices to be fetched with the --dry-run option, 117 | which stops short of actually fetching the missing prices (it just prints the 118 | list of fetches it would otherwise attempt). 119 | 120 | Caching 121 | ------- 122 | 123 | Prices are automatically cached. You can disable the cache with an option: 124 | 125 | bean-price --no-cache 126 | 127 | You can also instruct the script to clear the cache before fetching its prices: 128 | 129 | bean-price --clear-cache 130 | 131 | About Sources and Data Availability 132 | ----------------------------------- 133 | 134 | IMPORTANT: Note that each source may support a different routine for getting its 135 | latest data and for fetching historical/dated data, and that each of these may 136 | differ in their support. For example, Google Finance does not support fetching 137 | historical data for its CURRENCY:* instruments. 138 | 139 | """ 140 | 141 | __copyright__ = "Copyright (C) 2015-2020 Martin Blais" 142 | __license__ = "GNU GPLv2" 143 | -------------------------------------------------------------------------------- /beanprice/date_utils.py: -------------------------------------------------------------------------------- 1 | """Date utilities.""" 2 | 3 | __copyright__ = "Copyright (C) 2020 Martin Blais" 4 | __license__ = "GNU GPLv2" 5 | 6 | import contextlib 7 | import os 8 | import time 9 | 10 | import dateutil.parser 11 | 12 | 13 | def parse_date_liberally(string, parse_kwargs_dict=None): 14 | """Parse arbitrary strings to dates. 15 | 16 | This function is intended to support liberal inputs, so that we can use it 17 | in accepting user-specified dates on command-line scripts. 18 | 19 | Args: 20 | string: A string to parse. 21 | parse_kwargs_dict: Dict of kwargs to pass to dateutil parser. 22 | Returns: 23 | A datetime.date object. 24 | """ 25 | # At the moment, rely on the most excellent dateutil. 26 | if parse_kwargs_dict is None: 27 | parse_kwargs_dict = {} 28 | return dateutil.parser.parse(string, **parse_kwargs_dict).date() 29 | 30 | 31 | @contextlib.contextmanager 32 | def intimezone(tz_value: str): 33 | """Temporarily reset the value of TZ. 34 | 35 | This is used for testing. 36 | 37 | Args: 38 | tz_value: The value of TZ to set for the duration of this context. 39 | Returns: 40 | A contextmanager in the given timezone locale. 41 | """ 42 | tz_old = os.environ.get("TZ", None) 43 | os.environ["TZ"] = tz_value 44 | time.tzset() 45 | try: 46 | yield 47 | finally: 48 | if tz_old is None: 49 | del os.environ["TZ"] 50 | else: 51 | os.environ["TZ"] = tz_old 52 | time.tzset() 53 | -------------------------------------------------------------------------------- /beanprice/date_utils_test.py: -------------------------------------------------------------------------------- 1 | __copyright__ = "Copyright (C) 2020 Martin Blais" 2 | __license__ = "GNU GPLv2" 3 | 4 | import unittest 5 | import datetime 6 | import dateutil 7 | 8 | from beanprice import date_utils 9 | 10 | 11 | class TestDateUtils(unittest.TestCase): 12 | def test_parse_date_liberally(self): 13 | const_date = datetime.date(2014, 12, 7) 14 | test_cases = ( 15 | ("12/7/2014",), 16 | ("7-Dec-2014",), 17 | ("7/12/2014", {"parserinfo": dateutil.parser.parserinfo(dayfirst=True)}), 18 | ("12/7", {"default": datetime.datetime(2014, 1, 1)}), 19 | ("7.12.2014", {"dayfirst": True}), 20 | ("14 12 7", {"yearfirst": True}), 21 | ("Transaction of 7th December 2014", {"fuzzy": True}), 22 | ) 23 | for case in test_cases: 24 | if len(case) == 2: 25 | parse_date = date_utils.parse_date_liberally(case[0], case[1]) 26 | else: 27 | parse_date = date_utils.parse_date_liberally(case[0]) 28 | self.assertEqual(const_date, parse_date) 29 | 30 | def test_intimezone(self): 31 | with date_utils.intimezone("America/New_York"): 32 | now_nyc = datetime.datetime.now() 33 | with date_utils.intimezone("Europe/Berlin"): 34 | now_berlin = datetime.datetime.now() 35 | with date_utils.intimezone("Asia/Tokyo"): 36 | now_tokyo = datetime.datetime.now() 37 | self.assertNotEqual(now_nyc, now_berlin) 38 | self.assertNotEqual(now_berlin, now_tokyo) 39 | self.assertNotEqual(now_tokyo, now_nyc) 40 | 41 | 42 | if __name__ == "__main__": 43 | unittest.main() 44 | -------------------------------------------------------------------------------- /beanprice/net_utils.py: -------------------------------------------------------------------------------- 1 | """Network utilities.""" 2 | 3 | __copyright__ = "Copyright (C) 2015-2016 Martin Blais" 4 | __license__ = "GNU GPLv2" 5 | 6 | import logging 7 | from urllib import request 8 | from urllib import error 9 | 10 | 11 | def retrying_urlopen(url, timeout=5, max_retry=5): 12 | """Open and download the given URL, retrying if it times out. 13 | 14 | Args: 15 | url: A string, the URL to fetch. 16 | timeout: A timeout after which to stop waiting for a response and return an 17 | error. 18 | max_retry: The maximum number of times to retry. 19 | Returns: 20 | The contents of the fetched URL. 21 | """ 22 | for _ in range(max_retry): 23 | logging.debug("Reading %s", url) 24 | try: 25 | response = request.urlopen(url, timeout=timeout) 26 | if response: 27 | break 28 | except error.URLError: 29 | return None 30 | if response and response.getcode() != 200: 31 | return None 32 | return response 33 | -------------------------------------------------------------------------------- /beanprice/net_utils_test.py: -------------------------------------------------------------------------------- 1 | __copyright__ = "Copyright (C) 2015-2016 Martin Blais" 2 | __license__ = "GNU GPLv2" 3 | 4 | import http.client 5 | import unittest 6 | from unittest import mock 7 | 8 | from beanprice import net_utils 9 | 10 | 11 | class TestRetryingUrlopen(unittest.TestCase): 12 | def test_success_200(self): 13 | response = http.client.HTTPResponse(mock.MagicMock()) 14 | response.status = 200 15 | with mock.patch("urllib.request.urlopen", return_value=response): 16 | self.assertIs(net_utils.retrying_urlopen("http://nowhere.com"), response) 17 | 18 | def test_success_other(self): 19 | response = http.client.HTTPResponse(mock.MagicMock()) 20 | with mock.patch("urllib.request.urlopen", return_value=response): 21 | self.assertIsNone(net_utils.retrying_urlopen("http://nowhere.com")) 22 | 23 | def test_timeout_once(self): 24 | response = http.client.HTTPResponse(mock.MagicMock()) 25 | response.status = 200 26 | with mock.patch("urllib.request.urlopen", side_effect=[None, response]): 27 | self.assertIs(net_utils.retrying_urlopen("http://nowhere.com"), response) 28 | 29 | def test_max_retry(self): 30 | with mock.patch( 31 | "urllib.request.urlopen", side_effect=[None, None, None, None, None, None] 32 | ): 33 | self.assertIsNone(net_utils.retrying_urlopen("http://nowhere.com")) 34 | 35 | 36 | if __name__ == "__main__": 37 | unittest.main() 38 | -------------------------------------------------------------------------------- /beanprice/price_test.py: -------------------------------------------------------------------------------- 1 | """Tests for main driver for price fetching.""" 2 | 3 | __copyright__ = "Copyright (C) 2015-2020 Martin Blais" 4 | __license__ = "GNU GPLv2" 5 | 6 | import datetime 7 | import logging 8 | import shutil 9 | import sys 10 | import tempfile 11 | import types 12 | import unittest 13 | from os import path 14 | from unittest import mock 15 | from decimal import Decimal 16 | 17 | from dateutil import tz 18 | 19 | from beancount.utils import test_utils 20 | from beancount.parser import cmptest 21 | from beancount import loader 22 | 23 | from beanprice.source import SourcePrice 24 | from beanprice import price 25 | from beanprice.sources import yahoo 26 | 27 | 28 | PS = price.PriceSource 29 | 30 | 31 | def run_with_args(function, args, runner_file=None): 32 | """Run the given function with sys.argv set to argv. The first argument is 33 | automatically inferred to be where the function object was defined. sys.argv 34 | is restored after the function is called. 35 | Args: 36 | function: A function object to call with no arguments. 37 | argv: A list of arguments, excluding the script name, to be temporarily 38 | set on sys.argv. 39 | runner_file: An optional name of the top-level file being run. 40 | Returns: 41 | The return value of the function run. 42 | """ 43 | saved_argv = sys.argv 44 | saved_handlers = logging.root.handlers 45 | 46 | try: 47 | if runner_file is None: 48 | module = sys.modules[function.__module__] 49 | runner_file = module.__file__ 50 | sys.argv = [runner_file] + args 51 | logging.root.handlers = [] 52 | return function() 53 | finally: 54 | sys.argv = saved_argv 55 | logging.root.handlers = saved_handlers 56 | 57 | 58 | class TestCache(unittest.TestCase): 59 | def test_fetch_cached_price__disabled(self): 60 | # Latest. 61 | with mock.patch("beanprice.price._CACHE", None): 62 | self.assertIsNone(price._CACHE) 63 | source = mock.MagicMock() 64 | price.fetch_cached_price(source, "HOOL", None) 65 | self.assertTrue(source.get_latest_price.called) 66 | 67 | # Historical. 68 | with mock.patch("beanprice.price._CACHE", None): 69 | self.assertIsNone(price._CACHE) 70 | source = mock.MagicMock() 71 | price.fetch_cached_price(source, "HOOL", datetime.date.today()) 72 | self.assertTrue(source.get_historical_price.called) 73 | 74 | def test_fetch_cached_price__latest(self): 75 | tmpdir = tempfile.mkdtemp() 76 | tmpfile = path.join(tmpdir, "prices.cache") 77 | try: 78 | price.setup_cache(tmpfile, False) 79 | 80 | srcprice = SourcePrice( 81 | Decimal("1.723"), datetime.datetime.now(tz.tzutc()), "USD" 82 | ) 83 | source = mock.MagicMock() 84 | source.get_latest_price.return_value = srcprice 85 | source.__file__ = "" 86 | 87 | # Cache miss. 88 | result = price.fetch_cached_price(source, "HOOL", None) 89 | self.assertTrue(source.get_latest_price.called) 90 | self.assertEqual(1, len(price._CACHE)) 91 | self.assertEqual(srcprice, result) 92 | 93 | source.get_latest_price.reset_mock() 94 | 95 | # Cache hit. 96 | result = price.fetch_cached_price(source, "HOOL", None) 97 | self.assertFalse(source.get_latest_price.called) 98 | self.assertEqual(1, len(price._CACHE)) 99 | self.assertEqual(srcprice, result) 100 | 101 | srcprice2 = SourcePrice( 102 | Decimal("1.894"), datetime.datetime.now(tz.tzutc()), "USD" 103 | ) 104 | source.get_latest_price.reset_mock() 105 | source.get_latest_price.return_value = srcprice2 106 | 107 | # Cache expired. 108 | time_beyond = datetime.datetime.now() + price._CACHE.expiration * 2 109 | with mock.patch("beanprice.price.now", return_value=time_beyond): 110 | result = price.fetch_cached_price(source, "HOOL", None) 111 | self.assertTrue(source.get_latest_price.called) 112 | self.assertEqual(1, len(price._CACHE)) 113 | self.assertEqual(srcprice2, result) 114 | finally: 115 | price.reset_cache() 116 | if path.exists(tmpdir): 117 | shutil.rmtree(tmpdir) 118 | 119 | def test_fetch_cached_price__clear_cache(self): 120 | tmpdir = tempfile.mkdtemp() 121 | tmpfile = path.join(tmpdir, "prices.cache") 122 | try: 123 | price.setup_cache(tmpfile, False) 124 | 125 | srcprice = SourcePrice( 126 | Decimal("1.723"), datetime.datetime.now(tz.tzutc()), "USD" 127 | ) 128 | source = mock.MagicMock() 129 | source.get_latest_price.return_value = srcprice 130 | source.__file__ = "" 131 | 132 | # Cache miss. 133 | result = price.fetch_cached_price(source, "HOOL", None) 134 | self.assertTrue(source.get_latest_price.called) 135 | self.assertEqual(1, len(price._CACHE)) 136 | self.assertEqual(srcprice, result) 137 | 138 | source.get_latest_price.reset_mock() 139 | 140 | # Cache hit. 141 | result = price.fetch_cached_price(source, "HOOL", None) 142 | self.assertFalse(source.get_latest_price.called) 143 | self.assertEqual(1, len(price._CACHE)) 144 | self.assertEqual(srcprice, result) 145 | 146 | srcprice2 = SourcePrice( 147 | Decimal("1.894"), datetime.datetime.now(tz.tzutc()), "USD" 148 | ) 149 | source.get_latest_price.reset_mock() 150 | source.get_latest_price.return_value = srcprice2 151 | 152 | # Open cache again, but clear it. 153 | price.reset_cache() 154 | price.setup_cache(tmpfile, True) 155 | 156 | # Cache cleared. 157 | result = price.fetch_cached_price(source, "HOOL", None) 158 | self.assertTrue(source.get_latest_price.called) 159 | self.assertEqual(1, len(price._CACHE)) 160 | self.assertEqual(srcprice2, result) 161 | finally: 162 | price.reset_cache() 163 | if path.exists(tmpdir): 164 | shutil.rmtree(tmpdir) 165 | 166 | def test_fetch_cached_price__historical(self): 167 | tmpdir = tempfile.mkdtemp() 168 | tmpfile = path.join(tmpdir, "prices.cache") 169 | try: 170 | price.setup_cache(tmpfile, False) 171 | 172 | srcprice = SourcePrice( 173 | Decimal("1.723"), datetime.datetime.now(tz.tzutc()), "USD" 174 | ) 175 | source = mock.MagicMock() 176 | source.get_historical_price.return_value = srcprice 177 | source.__file__ = "" 178 | 179 | # Cache miss. 180 | day = datetime.date(2006, 1, 2) 181 | result = price.fetch_cached_price(source, "HOOL", day) 182 | self.assertTrue(source.get_historical_price.called) 183 | self.assertEqual(1, len(price._CACHE)) 184 | self.assertEqual(srcprice, result) 185 | 186 | source.get_historical_price.reset_mock() 187 | 188 | # Cache hit. 189 | result = price.fetch_cached_price(source, "HOOL", day) 190 | self.assertFalse(source.get_historical_price.called) 191 | self.assertEqual(1, len(price._CACHE)) 192 | self.assertEqual(srcprice, result) 193 | finally: 194 | price.reset_cache() 195 | if path.exists(tmpdir): 196 | shutil.rmtree(tmpdir) 197 | 198 | 199 | class TestProcessArguments(unittest.TestCase): 200 | def test_filename_not_exists(self): 201 | with test_utils.capture("stderr"): 202 | with self.assertRaises(SystemExit): 203 | run_with_args(price.process_args, ["--no-cache", "/some/file.beancount"]) 204 | 205 | @test_utils.docfile 206 | def test_explicit_file__badcontents(self, filename): 207 | """ 208 | 2015-01-01 open Assets:Invest 209 | 2015-01-01 open USD ;; Error 210 | """ 211 | with test_utils.capture("stderr"): 212 | args, jobs, _, __ = run_with_args(price.process_args, ["--no-cache", filename]) 213 | self.assertEqual([], jobs) 214 | 215 | def test_filename_exists(self): 216 | with tempfile.NamedTemporaryFile("w") as tmpfile: 217 | with test_utils.capture("stderr"): 218 | args, jobs, _, __ = run_with_args( 219 | price.process_args, ["--no-cache", tmpfile.name] 220 | ) 221 | self.assertEqual([], jobs) # Empty file. 222 | 223 | def test_expressions(self): 224 | with test_utils.capture("stderr"): 225 | args, jobs, _, __ = run_with_args( 226 | price.process_args, ["--no-cache", "-e", "USD:yahoo/AAPL"] 227 | ) 228 | self.assertEqual( 229 | [ 230 | price.DatedPrice( 231 | "AAPL", "USD", None, [price.PriceSource(yahoo, "AAPL", False)] 232 | ) 233 | ], 234 | jobs, 235 | ) 236 | 237 | 238 | class TestClobber(cmptest.TestCase): 239 | @loader.load_doc() 240 | def setUp(self, entries, _, __): 241 | """ 242 | ;; Existing file. 243 | 2015-01-05 price HDV 75.56 USD 244 | 2015-01-23 price HDV 77.34 USD 245 | 2015-02-06 price HDV 77.16 USD 246 | 2015-02-12 price HDV 78.17 USD 247 | 2015-05-01 price HDV 77.48 USD 248 | 2015-06-02 price HDV 76.33 USD 249 | 2015-06-29 price HDV 73.74 USD 250 | 2015-07-06 price HDV 73.79 USD 251 | 2015-08-11 price HDV 74.19 USD 252 | 2015-09-04 price HDV 68.98 USD 253 | """ 254 | self.entries = entries 255 | 256 | # New entries. 257 | self.price_entries, _, __ = loader.load_string( 258 | """ 259 | 2015-01-27 price HDV 76.83 USD 260 | 2015-02-06 price HDV 77.16 USD 261 | 2015-02-19 price HDV 77.5 USD 262 | 2015-06-02 price HDV 76.33 USD 263 | 2015-06-19 price HDV 76 USD 264 | 2015-07-06 price HDV 73.79 USD 265 | 2015-07-31 price HDV 74.64 USD 266 | 2015-08-11 price HDV 74.20 USD ;; Different 267 | """, 268 | dedent=True, 269 | ) 270 | 271 | def test_clobber_nodiffs(self): 272 | new_price_entries, _ = price.filter_redundant_prices( 273 | self.price_entries, self.entries, diffs=False 274 | ) 275 | self.assertEqualEntries( 276 | """ 277 | 2015-01-27 price HDV 76.83 USD 278 | 2015-02-19 price HDV 77.5 USD 279 | 2015-06-19 price HDV 76 USD 280 | 2015-07-31 price HDV 74.64 USD 281 | """, 282 | new_price_entries, 283 | ) 284 | 285 | def test_clobber_diffs(self): 286 | new_price_entries, _ = price.filter_redundant_prices( 287 | self.price_entries, self.entries, diffs=True 288 | ) 289 | self.assertEqualEntries( 290 | """ 291 | 2015-01-27 price HDV 76.83 USD 292 | 2015-02-19 price HDV 77.5 USD 293 | 2015-06-19 price HDV 76 USD 294 | 2015-07-31 price HDV 74.64 USD 295 | 2015-08-11 price HDV 74.20 USD ;; Different 296 | """, 297 | new_price_entries, 298 | ) 299 | 300 | 301 | class TestTimezone(unittest.TestCase): 302 | @mock.patch.object(price, "fetch_cached_price") 303 | def test_fetch_price__naive_time_no_timeozne(self, fetch_cached): 304 | fetch_cached.return_value = SourcePrice( 305 | Decimal("125.00"), datetime.datetime(2015, 11, 22, 16, 0, 0), "JPY" 306 | ) 307 | dprice = price.DatedPrice("JPY", "USD", datetime.date(2015, 11, 22), None) 308 | with self.assertRaises(ValueError): 309 | price.fetch_price( 310 | dprice._replace(sources=[price.PriceSource(yahoo, "USDJPY", False)]), False 311 | ) 312 | 313 | 314 | class TestInverted(unittest.TestCase): 315 | def setUp(self): 316 | fetch_cached = mock.patch("beanprice.price.fetch_cached_price").start() 317 | fetch_cached.return_value = SourcePrice( 318 | Decimal("125.00"), 319 | datetime.datetime(2015, 11, 22, 16, 0, 0, tzinfo=tz.tzlocal()), 320 | "JPY", 321 | ) 322 | self.dprice = price.DatedPrice("JPY", "USD", datetime.date(2015, 11, 22), None) 323 | self.addCleanup(mock.patch.stopall) 324 | 325 | def test_fetch_price__normal(self): 326 | entry = price.fetch_price( 327 | self.dprice._replace(sources=[price.PriceSource(yahoo, "USDJPY", False)]), False 328 | ) 329 | self.assertEqual(("JPY", "USD"), (entry.currency, entry.amount.currency)) 330 | self.assertEqual(Decimal("125.00"), entry.amount.number) 331 | 332 | def test_fetch_price__inverted(self): 333 | entry = price.fetch_price( 334 | self.dprice._replace(sources=[price.PriceSource(yahoo, "USDJPY", True)]), False 335 | ) 336 | self.assertEqual(("JPY", "USD"), (entry.currency, entry.amount.currency)) 337 | self.assertEqual(Decimal("0.008"), entry.amount.number) 338 | 339 | def test_fetch_price__swapped(self): 340 | entry = price.fetch_price( 341 | self.dprice._replace(sources=[price.PriceSource(yahoo, "USDJPY", True)]), True 342 | ) 343 | self.assertEqual(("USD", "JPY"), (entry.currency, entry.amount.currency)) 344 | self.assertEqual(Decimal("125.00"), entry.amount.number) 345 | 346 | 347 | class TestImportSource(unittest.TestCase): 348 | def test_import_source_valid(self): 349 | for name in "oanda", "yahoo": 350 | module = price.import_source(name) 351 | self.assertIsInstance(module, types.ModuleType) 352 | module = price.import_source("beanprice.sources.yahoo") 353 | self.assertIsInstance(module, types.ModuleType) 354 | 355 | def test_import_source_invalid(self): 356 | with self.assertRaises(ImportError): 357 | price.import_source("non.existing.module") 358 | 359 | 360 | class TestParseSource(unittest.TestCase): 361 | def test_source_invalid(self): 362 | with self.assertRaises(ValueError): 363 | price.parse_single_source("AAPL") 364 | with self.assertRaises(ValueError): 365 | price.parse_single_source("***//--") 366 | 367 | # The module gets imported at this stage. 368 | with self.assertRaises(ImportError): 369 | price.parse_single_source("invalid.module.name/NASDAQ:AAPL") 370 | 371 | def test_source_valid(self): 372 | psource = price.parse_single_source("yahoo/CNYUSD=X") 373 | self.assertEqual(PS(yahoo, "CNYUSD=X", False), psource) 374 | 375 | # Make sure that an invalid name at the tail doesn't succeed. 376 | with self.assertRaises(ValueError): 377 | psource = price.parse_single_source("yahoo/CNYUSD&X") 378 | 379 | psource = price.parse_single_source("beanprice.sources.yahoo/AAPL") 380 | self.assertEqual(PS(yahoo, "AAPL", False), psource) 381 | 382 | 383 | class TestParseSourceMap(unittest.TestCase): 384 | def _clean_source_map(self, smap): 385 | return { 386 | currency: [PS(s[0].__name__, s[1], s[2]) for s in sources] 387 | for currency, sources in smap.items() 388 | } 389 | 390 | def test_source_map_invalid(self): 391 | for expr in "USD", "something else", "USD:NASDAQ:AAPL": 392 | with self.assertRaises(ValueError): 393 | price.parse_source_map(expr) 394 | 395 | def test_source_map_onecur_single(self): 396 | smap = price.parse_source_map("USD:yahoo/AAPL") 397 | self.assertEqual( 398 | {"USD": [PS("beanprice.sources.yahoo", "AAPL", False)]}, 399 | self._clean_source_map(smap), 400 | ) 401 | 402 | def test_source_map_onecur_multiple(self): 403 | smap = price.parse_source_map("USD:oanda/USDCAD,yahoo/CAD=X") 404 | self.assertEqual( 405 | { 406 | "USD": [ 407 | PS("beanprice.sources.oanda", "USDCAD", False), 408 | PS("beanprice.sources.yahoo", "CAD=X", False), 409 | ] 410 | }, 411 | self._clean_source_map(smap), 412 | ) 413 | 414 | def test_source_map_manycur_single(self): 415 | smap = price.parse_source_map("USD:yahoo/USDCAD CAD:yahoo/CAD=X") 416 | self.assertEqual( 417 | { 418 | "USD": [PS("beanprice.sources.yahoo", "USDCAD", False)], 419 | "CAD": [PS("beanprice.sources.yahoo", "CAD=X", False)], 420 | }, 421 | self._clean_source_map(smap), 422 | ) 423 | 424 | def test_source_map_manycur_multiple(self): 425 | smap = price.parse_source_map("USD:yahoo/GBPUSD,oanda/GBPUSD CAD:yahoo/GBPCAD") 426 | self.assertEqual( 427 | { 428 | "USD": [ 429 | PS("beanprice.sources.yahoo", "GBPUSD", False), 430 | PS("beanprice.sources.oanda", "GBPUSD", False), 431 | ], 432 | "CAD": [PS("beanprice.sources.yahoo", "GBPCAD", False)], 433 | }, 434 | self._clean_source_map(smap), 435 | ) 436 | 437 | def test_source_map_inverse(self): 438 | smap = price.parse_source_map("USD:yahoo/^GBPUSD") 439 | self.assertEqual( 440 | {"USD": [PS("beanprice.sources.yahoo", "GBPUSD", True)]}, 441 | self._clean_source_map(smap), 442 | ) 443 | 444 | 445 | class TestFilters(unittest.TestCase): 446 | @loader.load_doc() 447 | def test_get_price_jobs__date(self, entries, _, __): 448 | """ 449 | 2000-01-10 open Assets:US:Invest:QQQ 450 | 2000-01-10 open Assets:US:Invest:VEA 451 | 2000-01-10 open Assets:US:Invest:Margin 452 | 453 | 2014-01-01 commodity QQQ 454 | price: "USD:yahoo/NASDAQ:QQQ" 455 | 456 | 2014-01-01 commodity VEA 457 | price: "USD:yahoo/NASDAQ:VEA" 458 | 459 | 2014-02-06 * 460 | Assets:US:Invest:QQQ 100 QQQ {86.23 USD} 461 | Assets:US:Invest:VEA 200 VEA {43.22 USD} 462 | Assets:US:Invest:Margin 463 | 464 | 2014-08-07 * 465 | Assets:US:Invest:QQQ -100 QQQ {86.23 USD} @ 91.23 USD 466 | Assets:US:Invest:Margin 467 | 468 | 2015-01-15 * 469 | Assets:US:Invest:QQQ 10 QQQ {92.32 USD} 470 | Assets:US:Invest:VEA -200 VEA {43.22 USD} @ 41.01 USD 471 | Assets:US:Invest:Margin 472 | """ 473 | jobs = price.get_price_jobs_at_date(entries, datetime.date(2014, 1, 1), False, None) 474 | self.assertEqual(set(), {(job.base, job.quote) for job in jobs}) 475 | 476 | jobs = price.get_price_jobs_at_date(entries, datetime.date(2014, 6, 1), False, None) 477 | self.assertEqual( 478 | {("QQQ", "USD"), ("VEA", "USD")}, {(job.base, job.quote) for job in jobs} 479 | ) 480 | 481 | jobs = price.get_price_jobs_at_date( 482 | entries, datetime.date(2014, 10, 1), False, None 483 | ) 484 | self.assertEqual({("VEA", "USD")}, {(job.base, job.quote) for job in jobs}) 485 | 486 | jobs = price.get_price_jobs_at_date(entries, None, False, None) 487 | self.assertEqual({("QQQ", "USD")}, {(job.base, job.quote) for job in jobs}) 488 | 489 | @loader.load_doc() 490 | def test_get_price_jobs__inactive(self, entries, _, __): 491 | """ 492 | 2000-01-10 open Assets:US:Invest:QQQ 493 | 2000-01-10 open Assets:US:Invest:VEA 494 | 2000-01-10 open Assets:US:Invest:Margin 495 | 496 | 2014-01-01 commodity QQQ 497 | price: "USD:yahoo/NASDAQ:QQQ" 498 | 499 | 2014-01-01 commodity VEA 500 | price: "USD:yahoo/NASDAQ:VEA" 501 | 502 | 2014-02-06 * 503 | Assets:US:Invest:QQQ 100 QQQ {86.23 USD} 504 | Assets:US:Invest:VEA 200 VEA {43.22 USD} 505 | Assets:US:Invest:Margin 506 | 507 | 2014-08-07 * 508 | Assets:US:Invest:QQQ -100 QQQ {86.23 USD} @ 91.23 USD 509 | Assets:US:Invest:Margin 510 | """ 511 | jobs = price.get_price_jobs_at_date(entries, None, False, None) 512 | self.assertEqual({("VEA", "USD")}, {(job.base, job.quote) for job in jobs}) 513 | 514 | jobs = price.get_price_jobs_at_date(entries, None, True, None) 515 | self.assertEqual( 516 | {("VEA", "USD"), ("QQQ", "USD")}, {(job.base, job.quote) for job in jobs} 517 | ) 518 | 519 | @loader.load_doc() 520 | def test_get_price_jobs__undeclared(self, entries, _, __): 521 | """ 522 | 2000-01-10 open Assets:US:Invest:QQQ 523 | 2000-01-10 open Assets:US:Invest:VEA 524 | 2000-01-10 open Assets:US:Invest:Margin 525 | 526 | 2014-01-01 commodity QQQ 527 | price: "USD:yahoo/NASDAQ:QQQ" 528 | 529 | 2014-02-06 * 530 | Assets:US:Invest:QQQ 100 QQQ {86.23 USD} 531 | Assets:US:Invest:VEA 200 VEA {43.22 USD} 532 | Assets:US:Invest:Margin 533 | """ 534 | jobs = price.get_price_jobs_at_date(entries, None, False, None) 535 | self.assertEqual({("QQQ", "USD")}, {(job.base, job.quote) for job in jobs}) 536 | 537 | jobs = price.get_price_jobs_at_date(entries, None, False, "yahoo") 538 | self.assertEqual( 539 | {("QQQ", "USD"), ("VEA", "USD")}, {(job.base, job.quote) for job in jobs} 540 | ) 541 | 542 | @loader.load_doc() 543 | def test_get_price_jobs__default_source(self, entries, _, __): 544 | """ 545 | 2000-01-10 open Assets:US:Invest:QQQ 546 | 2000-01-10 open Assets:US:Invest:Margin 547 | 548 | 2014-01-01 commodity QQQ 549 | price: "NASDAQ:QQQ" 550 | 551 | 2014-02-06 * 552 | Assets:US:Invest:QQQ 100 QQQ {86.23 USD} 553 | Assets:US:Invest:Margin 554 | """ 555 | jobs = price.get_price_jobs_at_date(entries, None, False, "yahoo") 556 | self.assertEqual(1, len(jobs[0].sources)) 557 | self.assertIsInstance(jobs[0].sources[0], price.PriceSource) 558 | 559 | @loader.load_doc() 560 | def test_get_price_jobs__currencies_not_at_cost(self, entries, _, __): 561 | """ 562 | 2000-01-10 open Assets:US:BofA:Checking 563 | 2000-01-10 open Assets:US:BofA:CHF 564 | 565 | 2014-01-01 commodity USD 566 | 2014-01-01 commodity CHF 567 | price: "USD:yahoo/CHFUSD=X" 568 | 569 | 2021-01-04 * 570 | Assets:US:BofA:Checking 100 USD 571 | Assets:US:BofA:CHF -110 CHF @@ 100 USD 572 | """ 573 | # TODO: Shouldn't we actually return (CHF, USD) here? 574 | jobs = price.get_price_jobs_at_date(entries, datetime.date(2021, 1, 4), False, None) 575 | self.assertEqual(set(), {(job.base, job.quote) for job in jobs}) 576 | 577 | jobs = price.get_price_jobs_at_date(entries, datetime.date(2021, 1, 6), False, None) 578 | self.assertEqual({("CHF", "USD")}, {(job.base, job.quote) for job in jobs}) 579 | 580 | # TODO: Shouldn't we return (CHF, USD) here, as above? 581 | jobs = price.get_price_jobs_up_to_date( 582 | entries, datetime.date(2021, 1, 6), False, None 583 | ) 584 | self.assertEqual(set(), {(job.base, job.quote) for job in jobs}) 585 | 586 | @loader.load_doc() 587 | def test_get_price_jobs_up_to_date(self, entries, _, __): 588 | """ 589 | 2000-01-10 open Assets:US:Invest:QQQ 590 | 2000-01-10 open Assets:US:Invest:VEA 591 | 2000-01-10 open Assets:US:Invest:Margin 592 | 593 | 2021-01-01 commodity QQQ 594 | price: "USD:yahoo/NASDAQ:QQQ" 595 | 596 | 2021-01-01 commodity VEA 597 | price: "USD:yahoo/NASDAQ:VEA" 598 | 599 | 2021-01-04 * 600 | Assets:US:Invest:QQQ 100 QQQ {86.23 USD} 601 | Assets:US:Invest:VEA 200 VEA {43.22 USD} 602 | Assets:US:Invest:Margin 603 | 604 | 2021-01-05 * 605 | Assets:US:Invest:QQQ -100 QQQ {86.23 USD} @ 91.23 USD 606 | Assets:US:Invest:Margin 607 | 608 | 2021-01-07 * 609 | Assets:US:Invest:QQQ 10 QQQ {92.32 USD} 610 | Assets:US:Invest:VEA -200 VEA {43.22 USD} @ 41.01 USD 611 | Assets:US:Invest:Margin 612 | """ 613 | jobs = price.get_price_jobs_up_to_date(entries, datetime.date(2021, 1, 8)) 614 | self.assertEqual( 615 | { 616 | ("QQQ", "USD", datetime.date(2021, 1, 4)), 617 | ("QQQ", "USD", datetime.date(2021, 1, 5)), 618 | ("QQQ", "USD", datetime.date(2021, 1, 7)), 619 | ("VEA", "USD", datetime.date(2021, 1, 4)), 620 | ("VEA", "USD", datetime.date(2021, 1, 5)), 621 | ("VEA", "USD", datetime.date(2021, 1, 6)), 622 | ("VEA", "USD", datetime.date(2021, 1, 7)), 623 | }, 624 | {(job.base, job.quote, job.date) for job in jobs}, 625 | ) 626 | 627 | 628 | class TestFromFile(unittest.TestCase): 629 | @loader.load_doc() 630 | def setUp(self, entries, _, __): 631 | """ 632 | 2000-01-10 open Assets:US:Investments:QQQ 633 | 2000-01-10 open Assets:CA:Investments:XSP 634 | 2000-01-10 open Assets:Cash 635 | 2000-01-10 open Assets:External 636 | 2000-01-10 open Expenses:Foreign 637 | 638 | 2010-01-01 commodity USD 639 | 640 | 2010-01-01 commodity QQQ 641 | name: "PowerShares QQQ Trust, Series 1 (ETF)" 642 | price: "USD:yahoo/NASDAQ:QQQ" 643 | 644 | 2010-01-01 commodity XSP 645 | name: "iShares S&P 500 Index Fund (CAD Hedged)" 646 | quote: CAD 647 | 648 | 2010-01-01 commodity AMTKPTS 649 | quote: USD 650 | price: "" 651 | 652 | """ 653 | self.entries = entries 654 | 655 | def test_find_currencies_declared(self): 656 | currencies = price.find_currencies_declared(self.entries, None) 657 | currencies2 = [(base, quote) for base, quote, _ in currencies] 658 | self.assertEqual([("QQQ", "USD")], currencies2) 659 | 660 | 661 | if __name__ == "__main__": 662 | unittest.main() 663 | -------------------------------------------------------------------------------- /beanprice/source.py: -------------------------------------------------------------------------------- 1 | """Interface definition for all price sources. 2 | 3 | This module describes the contract to be fulfilled by all implementations of 4 | price sources. 5 | 6 | TODO(blais): It would be an improvement if the interfaces here return an 7 | indication of why fetching failed and leave the responsibility to the caller to 8 | decide whether to share this with the user or to ignore and continue with other 9 | sources. 10 | """ 11 | 12 | __copyright__ = "Copyright (C) 2015-2020 Martin Blais" 13 | __license__ = "GNU GPLv2" 14 | 15 | import datetime 16 | from decimal import Decimal 17 | from typing import List, Optional, NamedTuple 18 | 19 | 20 | # A record that contains data for a price fetched from a source. 21 | # 22 | # A triple of 23 | # price: A Decimal instance, the price or rate. 24 | # time: A datetime.time instance at which that price or rate was available. 25 | # Note that this instance is REQUIRED to be timezone aware, as this is 26 | # used to compute a corresponding date in the user's timezone. 27 | # quote-currency: A string, the quote currency of the given price, if 28 | # available. 29 | SourcePrice = NamedTuple( 30 | "SourcePrice", 31 | [ 32 | ("price", Decimal), 33 | ("time", Optional[datetime.datetime]), 34 | ("quote_currency", Optional[str]), 35 | ], 36 | ) 37 | 38 | 39 | class Source: 40 | """Interface to be implemented by all price sources. 41 | 42 | Notes about arguments below: 43 | `ticker` arguments: A string, the ticker to be fetched by the source. This 44 | ticker may include structure, such as the exchange code. Also note that 45 | this ticker is source-specified, and is not necessarily the same value 46 | as the commodity symbol used in the Beancount file. 47 | time arguments: A `datetime.datetime` instance. This is a timezone-aware 48 | `datetime` you can convert to any timezone. For past dates we query for 49 | a time that is equivalent to 4pm in the user's timezone. 50 | 51 | About return values: 52 | If the price could not be fetched, None is returned and another source 53 | should be consulted. There is never any guarantee that a price source will 54 | be able to fetch its value and failure to fetch is more frequent than one 55 | might assume in practice; client code must be able to handle this and try 56 | again with another price source until all sources are exhausted. 57 | 58 | Also, note in the case we were able to fetch, the price's returned time 59 | must be timezone-aware (not naive). 60 | """ 61 | 62 | def get_latest_price(self, ticker: str) -> Optional[SourcePrice]: 63 | """Fetch the current latest price. The date may differ. 64 | 65 | This routine attempts to fetch the most recent available price, and 66 | returns the actual date of the quoted price, which may differ from the 67 | date this call is made at. {1cfa25e37fc1} 68 | 69 | Args: 70 | ticker: A string, the ticker to be fetched by the source. 71 | Returns: 72 | A SourcePrice instance, or None if we failed to fetch. 73 | """ 74 | 75 | def get_historical_price( 76 | self, ticker: str, time: datetime.datetime 77 | ) -> Optional[SourcePrice]: 78 | """Return the lastest historical price found for the symbol at the given date. 79 | 80 | This could be the price of the close of the day, for instance. We assume 81 | that there is some single price representative of the day. Also note 82 | that if you're querying for a weekend or holiday (closed market) date, 83 | the price returned may have a date earlier than the one you requested 84 | (the latest available market price for that instrument is from a prior 85 | date). 86 | 87 | Args: 88 | ticker: A string, the ticker to be fetched by the source. 89 | time: The timestamp at which to query for the price. 90 | Returns: 91 | A SourcePrice instance, or None if we failed to fetch. 92 | """ 93 | 94 | def get_prices_series( 95 | self, ticker: str, time_begin: datetime.datetime, time_end: datetime.datetime 96 | ) -> Optional[List[SourcePrice]]: 97 | """Return the historical daily price series between two dates. 98 | 99 | Note that weekends don't have any prices, so there's no guarantee that 100 | this returns a contiguous series of prices for each day in the requested 101 | interval. 102 | 103 | Args: 104 | ticker: A string, the ticker to be fetched by the source. 105 | time_begin: The earliest timestamp whose prices to include. 106 | time_end: The latest timestamp whose prices to include. 107 | Returns: 108 | A list of SourcePrice instances, sorted by date/time, or None if we 109 | failed to fetch. An empty list signals success fetching but no data in 110 | the requested interval. 111 | """ 112 | -------------------------------------------------------------------------------- /beanprice/sources/BUILD: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | py_library( 4 | name = "iex", 5 | srcs = ["iex.py"], 6 | deps = [ 7 | "//beancount/core:number", 8 | "//beancount/prices:source", 9 | ], 10 | ) 11 | 12 | py_test( 13 | name = "iex_test", 14 | srcs = ["iex_test.py"], 15 | deps = [ 16 | "//beancount/core:number", 17 | "//beancount/prices:source", 18 | "//beancount/prices/sources:iex", 19 | "//beanprice:date_utils", 20 | ], 21 | ) 22 | 23 | py_library( 24 | name = "oanda", 25 | srcs = ["oanda.py"], 26 | deps = [ 27 | "//beancount/core:number", 28 | "//beancount/prices:source", 29 | "//beancount/utils:net_utils", 30 | ], 31 | ) 32 | 33 | py_test( 34 | name = "oanda_test", 35 | srcs = ["oanda_test.py"], 36 | deps = [ 37 | "//beanprice:date_utils", 38 | "//beancount/core:number", 39 | "//beancount/prices:source", 40 | "//beancount/prices/sources:oanda", 41 | "//beancount/utils:net_utils", 42 | ], 43 | ) 44 | 45 | py_library( 46 | name = "quandl", 47 | srcs = ["quandl.py"], 48 | deps = [ 49 | "//beancount/core:number", 50 | "//beancount/prices:source", 51 | ], 52 | ) 53 | 54 | py_test( 55 | name = "quandl_test", 56 | srcs = ["quandl_test.py"], 57 | deps = [ 58 | "//beancount/core:number", 59 | "//beancount/prices:source", 60 | "//beancount/prices/sources:quandl", 61 | "//beanprice:date_utils", 62 | ], 63 | ) 64 | 65 | py_library( 66 | name = "yahoo", 67 | srcs = ["yahoo.py"], 68 | deps = [ 69 | "//beancount/core:number", 70 | "//beancount/prices:source", 71 | ], 72 | ) 73 | 74 | py_test( 75 | name = "yahoo_test", 76 | srcs = ["yahoo_test.py"], 77 | deps = [ 78 | "//beancount/core:number", 79 | "//beancount/prices/sources:yahoo", 80 | "//beanprice:date_utils", 81 | ], 82 | ) 83 | 84 | py_library( 85 | name = "tsp", 86 | srcs = ["tsp.py"], 87 | deps = [ 88 | "//beancount/core:number", 89 | "//beancount/prices:source", 90 | ], 91 | ) 92 | 93 | py_test( 94 | name = "tsp_test", 95 | srcs = ["tsp_test.py"], 96 | deps = [ 97 | "//beancount/core:number", 98 | "//beancount/prices/sources:tsp", 99 | ], 100 | ) 101 | 102 | py_library( 103 | name = "coincap", 104 | srcs = ["coincap.py"], 105 | deps = [ 106 | "//beancount/core:number", 107 | "//beancount/prices:source", 108 | ], 109 | ) 110 | 111 | py_test( 112 | name = "coincap_test", 113 | srcs = ["coincap_test.py"], 114 | deps = [ 115 | "//beancount/core:number", 116 | "//beancount/prices/sources:tsp", 117 | ], 118 | ) 119 | -------------------------------------------------------------------------------- /beanprice/sources/__init__.py: -------------------------------------------------------------------------------- 1 | """Implementation of various price extractors. 2 | 3 | This package is looked up by the driver script to figure out which extractor to 4 | use. 5 | """ 6 | 7 | __copyright__ = "Copyright (C) 2015-2020 Martin Blais" 8 | __license__ = "GNU GPLv2" 9 | -------------------------------------------------------------------------------- /beanprice/sources/alphavantage.py: -------------------------------------------------------------------------------- 1 | """A source fetching prices and exchangerates from https://www.alphavantage.co. 2 | 3 | It requires a free api key which needs to be set in the 4 | environment variable "ALPHAVANTAGE_API_KEY" 5 | 6 | Valid tickers for prices are in the form "price:XXX:YYY", such as "price:IBM:USD" 7 | where XXX is the symbol and YYY is the expected quote currency in which the data 8 | is returned. The api currently does not support converting to a specific ccy and 9 | does unfortunately not return in which ccy the result is. 10 | 11 | Valid tickers for exchangerates are in the form "fx:XXX:YYY", such as "fx:USD:CHF". 12 | 13 | Here is the API documentation: 14 | https://www.alphavantage.co/documentation/ 15 | 16 | For example: 17 | 18 | 19 | https://www.alphavantage.co/query?function=GLOBAL_QUOTE&symbol=IBM&apikey=demo 20 | 21 | https://www.alphavantage.co/query?function=CURRENCY_EXCHANGE_RATE&from_currency=USD&to_currency=JPY&apikey=demo 22 | 23 | """ 24 | 25 | from decimal import Decimal 26 | 27 | import re 28 | from os import environ 29 | from time import sleep 30 | import requests 31 | from dateutil.tz import tz 32 | from dateutil.parser import parse 33 | 34 | from beanprice import source 35 | 36 | 37 | class AlphavantageApiError(ValueError): 38 | "An error from the Alphavantage API." 39 | 40 | 41 | def _parse_ticker(ticker): 42 | """Parse the base and quote currencies from the ticker. 43 | 44 | Args: 45 | ticker: A string, the symbol in kind-XXX-YYY format. 46 | Returns: 47 | A (kind, symbol, base) tuple. 48 | """ 49 | match = re.match(r"^(?Pprice|fx):(?P[^:]+):(?P\w+)$", ticker) 50 | if not match: 51 | raise ValueError('Invalid ticker. Use "price:SYMBOL:BASE" or "fx:CCY:BASE" format.') 52 | return match.groups() 53 | 54 | 55 | def _do_fetch(params): 56 | params["apikey"] = environ["ALPHAVANTAGE_API_KEY"] 57 | 58 | resp = requests.get(url="https://www.alphavantage.co/query", params=params) 59 | data = resp.json() 60 | # This is for dealing with the rate limit, sleep for 60 seconds and then retry 61 | if "Note" in data: 62 | sleep(60) 63 | resp = requests.get(url="https://www.alphavantage.co/query", params=params) 64 | data = resp.json() 65 | 66 | if resp.status_code != requests.codes.ok: 67 | raise AlphavantageApiError( 68 | "Invalid response ({}): {}".format(resp.status_code, resp.text) 69 | ) 70 | 71 | if "Error Message" in data: 72 | raise AlphavantageApiError("Invalid response: {}".format(data["Error Message"])) 73 | 74 | return data 75 | 76 | 77 | class Source(source.Source): 78 | def get_latest_price(self, ticker): 79 | kind, symbol, base = _parse_ticker(ticker) 80 | 81 | if kind == "price": 82 | params = { 83 | "function": "GLOBAL_QUOTE", 84 | "symbol": symbol, 85 | } 86 | data = _do_fetch(params) 87 | 88 | price_data = data["Global Quote"] 89 | price = Decimal(price_data["05. price"]) 90 | date = parse(price_data["07. latest trading day"]).replace(tzinfo=tz.tzutc()) 91 | else: 92 | params = { 93 | "function": "CURRENCY_EXCHANGE_RATE", 94 | "from_currency": symbol, 95 | "to_currency": base, 96 | } 97 | data = _do_fetch(params) 98 | 99 | price_data = data["Realtime Currency Exchange Rate"] 100 | price = Decimal(price_data["5. Exchange Rate"]) 101 | date = parse(price_data["6. Last Refreshed"]).replace( 102 | tzinfo=tz.gettz(price_data["7. Time Zone"]) 103 | ) 104 | 105 | return source.SourcePrice(price, date, base) 106 | 107 | def get_historical_price(self, ticker, time): 108 | return None 109 | -------------------------------------------------------------------------------- /beanprice/sources/alphavantage_test.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import unittest 3 | from os import environ 4 | from decimal import Decimal 5 | 6 | from unittest import mock 7 | from dateutil import tz 8 | 9 | import requests 10 | 11 | from beanprice import source 12 | from beanprice.sources import alphavantage 13 | 14 | 15 | def response(contents, status_code=requests.codes.ok): 16 | """Return a context manager to patch a JSON response.""" 17 | response = mock.Mock() 18 | response.status_code = status_code 19 | response.text = "" 20 | response.json.return_value = contents 21 | return mock.patch("requests.get", return_value=response) 22 | 23 | 24 | class AlphavantagePriceFetcher(unittest.TestCase): 25 | def setUp(self): 26 | environ["ALPHAVANTAGE_API_KEY"] = "foo" 27 | 28 | def tearDown(self): 29 | del environ["ALPHAVANTAGE_API_KEY"] 30 | 31 | def test_error_invalid_ticker(self): 32 | with self.assertRaises(ValueError): 33 | alphavantage.Source().get_latest_price("INVALID") 34 | 35 | def test_error_network(self): 36 | with response("Foobar", 404): 37 | with self.assertRaises(alphavantage.AlphavantageApiError): 38 | alphavantage.Source().get_latest_price("price:IBM:USD") 39 | 40 | def test_error_response(self): 41 | contents = {"Error Message": "Something wrong"} 42 | with response(contents): 43 | with self.assertRaises(alphavantage.AlphavantageApiError): 44 | alphavantage.Source().get_latest_price("price:IBM:USD") 45 | 46 | def test_valid_response_price(self): 47 | contents = { 48 | "Global Quote": { 49 | "05. price": "144.7400", 50 | "07. latest trading day": "2021-01-21", 51 | } 52 | } 53 | with response(contents): 54 | srcprice = alphavantage.Source().get_latest_price("price:FOO:USD") 55 | self.assertIsInstance(srcprice, source.SourcePrice) 56 | self.assertEqual(Decimal("144.7400"), srcprice.price) 57 | self.assertEqual("USD", srcprice.quote_currency) 58 | self.assertEqual( 59 | datetime.datetime(2021, 1, 21, 0, 0, 0, tzinfo=tz.tzutc()), srcprice.time 60 | ) 61 | 62 | def test_valid_response_fx(self): 63 | contents = { 64 | "Realtime Currency Exchange Rate": { 65 | "5. Exchange Rate": "108.94000000", 66 | "6. Last Refreshed": "2021-02-21 20:32:25", 67 | "7. Time Zone": "UTC", 68 | } 69 | } 70 | with response(contents): 71 | srcprice = alphavantage.Source().get_latest_price("fx:USD:CHF") 72 | self.assertIsInstance(srcprice, source.SourcePrice) 73 | self.assertEqual(Decimal("108.94000000"), srcprice.price) 74 | self.assertEqual("CHF", srcprice.quote_currency) 75 | self.assertEqual( 76 | datetime.datetime(2021, 2, 21, 20, 32, 25, tzinfo=tz.tzutc()), srcprice.time 77 | ) 78 | 79 | 80 | if __name__ == "__main__": 81 | unittest.main() 82 | -------------------------------------------------------------------------------- /beanprice/sources/coinbase.py: -------------------------------------------------------------------------------- 1 | """A source fetching cryptocurrency prices from Coinbase. 2 | 3 | Valid tickers are in the form "XXX-YYY", such as "BTC-USD". 4 | 5 | Here is the API documentation: 6 | https://developers.coinbase.com/api/v2 7 | 8 | For example: 9 | https://api.coinbase.com/v2/prices/BTC-GBP/spot 10 | 11 | Timezone information: Input and output datetimes are specified via UTC 12 | timestamps. 13 | """ 14 | 15 | import datetime 16 | from decimal import Decimal 17 | 18 | import requests 19 | from dateutil.tz import tz 20 | 21 | from beanprice import source 22 | 23 | 24 | class CoinbaseError(ValueError): 25 | "An error from the Coinbase API." 26 | 27 | 28 | def fetch_quote(ticker, time=None): 29 | """Fetch a quote from Coinbase.""" 30 | url = "https://api.coinbase.com/v2/prices/{}/spot".format(ticker.lower()) 31 | options = {} 32 | if time is not None: 33 | options["date"] = time.astimezone(tz.tzutc()).date().isoformat() 34 | 35 | response = requests.get(url, options) 36 | if response.status_code != requests.codes.ok: 37 | raise CoinbaseError( 38 | "Invalid response ({}): {}".format(response.status_code, response.text) 39 | ) 40 | result = response.json() 41 | 42 | price = Decimal(result["data"]["amount"]) 43 | if time is None: 44 | time = datetime.datetime.now(tz.tzutc()) 45 | currency = result["data"]["currency"] 46 | 47 | return source.SourcePrice(price, time, currency) 48 | 49 | 50 | class Source(source.Source): 51 | "Coinbase API price extractor." 52 | 53 | def get_latest_price(self, ticker): 54 | """See contract in beanprice.source.Source.""" 55 | return fetch_quote(ticker) 56 | 57 | def get_historical_price(self, ticker, time): 58 | """See contract in beanprice.source.Source.""" 59 | return fetch_quote(ticker, time) 60 | -------------------------------------------------------------------------------- /beanprice/sources/coinbase_test.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import unittest 3 | from decimal import Decimal 4 | 5 | from unittest import mock 6 | from dateutil import tz 7 | 8 | import requests 9 | 10 | from beanprice import source 11 | from beanprice.sources import coinbase 12 | 13 | 14 | def response(contents, status_code=requests.codes.ok): 15 | """Return a context manager to patch a JSON response.""" 16 | response = mock.Mock() 17 | response.status_code = status_code 18 | response.text = "" 19 | response.json.return_value = contents 20 | return mock.patch("requests.get", return_value=response) 21 | 22 | 23 | class CoinbasePriceFetcher(unittest.TestCase): 24 | def test_error_network(self): 25 | with response(None, 404): 26 | with self.assertRaises(ValueError) as exc: 27 | coinbase.fetch_quote("AAPL") 28 | self.assertRegex(exc.message, "premium") 29 | 30 | def test_valid_response(self): 31 | contents = {"data": {"base": "BTC", "currency": "USD", "amount": "101.23456"}} 32 | with response(contents): 33 | srcprice = coinbase.Source().get_latest_price("BTC-GBP") 34 | self.assertIsInstance(srcprice, source.SourcePrice) 35 | self.assertEqual(Decimal("101.23456"), srcprice.price) 36 | self.assertEqual("USD", srcprice.quote_currency) 37 | 38 | def test_historical_price(self): 39 | contents = {"data": {"base": "BTC", "currency": "USD", "amount": "101.23456"}} 40 | with response(contents): 41 | time = datetime.datetime(2018, 3, 27, 0, 0, 0, tzinfo=tz.tzutc()) 42 | srcprice = coinbase.Source().get_historical_price("BTC-GBP", time) 43 | self.assertIsInstance(srcprice, source.SourcePrice) 44 | self.assertEqual(Decimal("101.23456"), srcprice.price) 45 | self.assertEqual("USD", srcprice.quote_currency) 46 | self.assertEqual( 47 | datetime.datetime(2018, 3, 27, 0, 0, 0, tzinfo=tz.tzutc()), srcprice.time 48 | ) 49 | 50 | 51 | if __name__ == "__main__": 52 | unittest.main() 53 | -------------------------------------------------------------------------------- /beanprice/sources/coincap.py: -------------------------------------------------------------------------------- 1 | """ 2 | A source fetching cryptocurrency prices from Coincap. 3 | 4 | Tickers can be in two formats: Either using the coincap currency id (bitcoin, 5 | ethereum, etc.), or using a currency ticker (BTC, ETH, etc.). In the latter 6 | case, any ambiguity will be resolved using the coin ranking. 7 | 8 | Prices are denoted in USD. 9 | 10 | The documentation can be found here: 11 | https://docs.coincap.io/ 12 | 13 | """ 14 | 15 | from datetime import datetime, timezone, timedelta 16 | import math 17 | from decimal import Decimal 18 | from typing import List, Optional, Dict 19 | import requests 20 | from beanprice import source 21 | 22 | API_BASE_URL = "https://api.coincap.io/v2/" 23 | 24 | 25 | class CoincapError(ValueError): 26 | "An error from the Coincap importer." 27 | 28 | 29 | def get_asset_list() -> List[Dict[str, str]]: 30 | """ 31 | Get list of currencies supported by Coincap. Returned is a list with 32 | elements with many properties, including "id", representing the Coincap id, 33 | and "symbol", representing the ticker symbol. 34 | """ 35 | path = "assets/" 36 | url = API_BASE_URL + path 37 | response = requests.get(url) 38 | data = response.json()["data"] 39 | return data 40 | 41 | 42 | def get_currency_id(currency: str) -> Optional[str]: 43 | """ 44 | Find currency ID by its symbol. 45 | If results are ambiguous, select currency with the highest market cap 46 | """ 47 | # Array is already sorted based on market cap 48 | for coin in get_asset_list(): 49 | if coin["symbol"] == currency: 50 | return coin["id"] 51 | return None 52 | 53 | 54 | def resolve_currency_id(base_currency: str) -> str: 55 | """ 56 | Obtain the currency ID from the ticker, which can either already be a 57 | currency id (bitcoin), or a coin ticker (BTC). 58 | """ 59 | if base_currency.isupper(): 60 | # Try to find currency ID by its symbol 61 | base_currency_id = get_currency_id(base_currency) 62 | if not isinstance(base_currency_id, str): 63 | raise CoincapError( 64 | f"Could not find currency id with ticker '{base_currency}'" 65 | ) 66 | return base_currency_id 67 | else: 68 | return base_currency 69 | 70 | 71 | def get_latest_price(base_currency: str) -> source.SourcePrice: 72 | """ 73 | Get the latest available price for a given currency. 74 | """ 75 | path = "assets/" 76 | url = f"{API_BASE_URL}{path}{resolve_currency_id(base_currency)}" 77 | response = requests.get(url) 78 | data = response.json() 79 | time = datetime.fromtimestamp(data["timestamp"] / 1000.0).replace( 80 | tzinfo=timezone.utc 81 | ) 82 | price = Decimal(data["data"]["priceUsd"]) 83 | return source.SourcePrice(price, time, "USD") 84 | 85 | 86 | def get_price_series( 87 | base_currency_id: str, time_begin: datetime, time_end: datetime 88 | ) -> List[source.SourcePrice]: 89 | path = f"assets/{base_currency_id}/history" 90 | params = { 91 | "interval": "d1", 92 | "start": str(math.floor(time_begin.timestamp() * 1000.0)), 93 | "end": str(math.ceil(time_end.timestamp() * 1000.0)), 94 | } 95 | url = API_BASE_URL + path 96 | response = requests.get(url, params=params) 97 | return [ 98 | source.SourcePrice( 99 | Decimal(item["priceUsd"]), 100 | datetime.fromtimestamp(item["time"] / 1000.0).replace(tzinfo=timezone.utc), 101 | "USD", 102 | ) 103 | for item in response.json()["data"] 104 | ] 105 | 106 | 107 | class Source(source.Source): 108 | """A price source for the Coincap API v2. Supports only prices denoted in USD. 109 | There are two ways of expressing a ticker, either by their coincap id (bitcoin) 110 | or by their ticker (BTC), in which case the highest ranked coin will be picked.""" 111 | 112 | def get_latest_price(self, ticker) -> source.SourcePrice: 113 | return get_latest_price(ticker) 114 | 115 | def get_historical_price( 116 | self, ticker: str, time: datetime 117 | ) -> Optional[source.SourcePrice]: 118 | for datapoint in self.get_prices_series( 119 | ticker, 120 | time + timedelta(days=-1), 121 | time + timedelta(days=1), 122 | ): 123 | # TODO(blais): This is poorly thought out, the date may not match 124 | # that in the differing timezone. You probably want the last price 125 | # before the datapoint time. 126 | if datapoint.time is not None and datapoint.time.date() == time.date(): 127 | return datapoint 128 | return None 129 | 130 | def get_prices_series( 131 | self, ticker: str, time_begin: datetime, time_end: datetime 132 | ) -> List[source.SourcePrice]: 133 | return get_price_series(resolve_currency_id(ticker), time_begin, time_end) 134 | -------------------------------------------------------------------------------- /beanprice/sources/coincap_test.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import unittest 3 | from decimal import Decimal 4 | 5 | from unittest import mock 6 | from dateutil import tz 7 | 8 | import requests 9 | 10 | from beanprice import source 11 | from beanprice.sources import coincap 12 | 13 | timezone = tz.gettz("Europe/Amsterdam") 14 | 15 | response_assets_bitcoin_historical = { 16 | "data": [ 17 | { 18 | "priceUsd": "32263.2648195597839546", 19 | "time": 1609804800000, 20 | "date": "2021-01-05T00:00:00.000Z", 21 | }, 22 | { 23 | "priceUsd": "34869.7692419204775049", 24 | "time": 1609891200000, 25 | "date": "2021-01-06T00:00:00.000Z", 26 | }, 27 | ], 28 | "timestamp": 1618220568799, 29 | } 30 | 31 | response_assets_bitcoin = { 32 | "data": { 33 | "id": "bitcoin", 34 | "rank": "1", 35 | "symbol": "BTC", 36 | "name": "Bitcoin", 37 | "supply": "18672456.0000000000000000", 38 | "maxSupply": "21000000.0000000000000000", 39 | "marketCapUsd": "1134320211245.9295410753733840", 40 | "volumeUsd24Hr": "16998481452.4370929843940509", 41 | "priceUsd": "60748.3135183678858890", 42 | "changePercent24Hr": "1.3457951950518293", 43 | "vwap24Hr": "59970.0332730340881967", 44 | "explorer": "https://blockchain.info/", 45 | }, 46 | "timestamp": 1618218375359, 47 | } 48 | 49 | response_bitcoin_history = { 50 | "data": [ 51 | { 52 | "priceUsd": "29232.6707650537687673", 53 | "time": 1609459200000, 54 | "date": "2021-01-01T00:00:00.000Z", 55 | }, 56 | { 57 | "priceUsd": "30688.0967118388768791", 58 | "time": 1609545600000, 59 | "date": "2021-01-02T00:00:00.000Z", 60 | }, 61 | { 62 | "priceUsd": "33373.7277104175704785", 63 | "time": 1609632000000, 64 | "date": "2021-01-03T00:00:00.000Z", 65 | }, 66 | { 67 | "priceUsd": "31832.6862288485383625", 68 | "time": 1609718400000, 69 | "date": "2021-01-04T00:00:00.000Z", 70 | }, 71 | { 72 | "priceUsd": "32263.2648195597839546", 73 | "time": 1609804800000, 74 | "date": "2021-01-05T00:00:00.000Z", 75 | }, 76 | { 77 | "priceUsd": "34869.7692419204775049", 78 | "time": 1609891200000, 79 | "date": "2021-01-06T00:00:00.000Z", 80 | }, 81 | { 82 | "priceUsd": "38041.0026368820979411", 83 | "time": 1609977600000, 84 | "date": "2021-01-07T00:00:00.000Z", 85 | }, 86 | { 87 | "priceUsd": "39821.5432664411153366", 88 | "time": 1610064000000, 89 | "date": "2021-01-08T00:00:00.000Z", 90 | }, 91 | ], 92 | "timestamp": 1618219315479, 93 | } 94 | 95 | 96 | def response(content, status_code=requests.codes.ok): 97 | """Return a context manager to patch a JSON response.""" 98 | response = mock.Mock() 99 | response.status_code = status_code 100 | response.text = "" 101 | response.json.return_value = content 102 | return mock.patch("requests.get", return_value=response) 103 | 104 | 105 | class Source(unittest.TestCase): 106 | def test_get_latest_price(self): 107 | with response(content=response_assets_bitcoin): 108 | srcprice = coincap.Source().get_latest_price("bitcoin") 109 | self.assertIsInstance(srcprice, source.SourcePrice) 110 | self.assertEqual(Decimal("60748.3135183678858890"), srcprice.price) 111 | self.assertEqual( 112 | datetime.datetime(2021, 4, 12) 113 | .replace(tzinfo=datetime.timezone.utc) 114 | .date(), 115 | srcprice.time.date(), 116 | ) 117 | self.assertEqual("USD", srcprice.quote_currency) 118 | 119 | def test_get_historical_price(self): 120 | with response(content=response_assets_bitcoin_historical): 121 | srcprice = coincap.Source().get_historical_price( 122 | "bitcoin", datetime.datetime(2021, 1, 6).replace(tzinfo=timezone) 123 | ) 124 | self.assertEqual(Decimal("34869.7692419204775049"), srcprice.price) 125 | self.assertEqual( 126 | datetime.datetime(2021, 1, 6) 127 | .replace(tzinfo=datetime.timezone.utc) 128 | .date(), 129 | srcprice.time.date(), 130 | ) 131 | self.assertEqual("USD", srcprice.quote_currency) 132 | 133 | def test_get_prices_series(self): 134 | with response(content=response_bitcoin_history): 135 | srcprices = coincap.Source().get_prices_series( 136 | "bitcoin", 137 | datetime.datetime(2021, 1, 1).replace(tzinfo=timezone), 138 | datetime.datetime(2021, 3, 20).replace(tzinfo=timezone), 139 | ) 140 | self.assertEqual(len(srcprices), 8) 141 | self.assertEqual(Decimal("29232.6707650537687673"), srcprices[0].price) 142 | self.assertEqual( 143 | datetime.datetime(2021, 1, 1) 144 | .replace(tzinfo=datetime.timezone.utc) 145 | .date(), 146 | srcprices[0].time.date(), 147 | ) 148 | self.assertEqual("USD", srcprices[0].quote_currency) 149 | 150 | 151 | if __name__ == "__main__": 152 | unittest.main() 153 | -------------------------------------------------------------------------------- /beanprice/sources/coinmarketcap.py: -------------------------------------------------------------------------------- 1 | """A source fetching cryptocurrency prices from Coinmarketcap. 2 | 3 | Valid tickers are in the form "XXX-YYY", such as "BTC-CHF". 4 | 5 | It requires a free api key which needs to be set in the 6 | environment variable "COINMARKETCAP_API_KEY" 7 | 8 | Here is the API documentation: 9 | https://coinmarketcap.com/api/documentation/v1/ 10 | """ 11 | 12 | from decimal import Decimal 13 | import re 14 | from os import environ 15 | import requests 16 | from dateutil.parser import parse 17 | from beanprice import source 18 | 19 | 20 | class CoinmarketcapApiError(ValueError): 21 | "An error from the CoinMarketCap API." 22 | 23 | 24 | def _parse_ticker(ticker): 25 | """Parse the base and quote currencies from the ticker. 26 | 27 | Args: 28 | ticker: A string, the symbol in XXX-YYY format. 29 | Returns: 30 | A pair of (base, quote) currencies. 31 | """ 32 | match = re.match(r"^(?P\w+)-(?P\w+)$", ticker) 33 | if not match: 34 | raise ValueError('Invalid ticker. Use "BASE-SYMBOL" format.') 35 | return match.groups() 36 | 37 | 38 | class Source(source.Source): 39 | def get_latest_price(self, ticker): 40 | symbol, base = _parse_ticker(ticker) 41 | headers = { 42 | "X-CMC_PRO_API_KEY": environ["COINMARKETCAP_API_KEY"], 43 | } 44 | params = { 45 | "symbol": symbol, 46 | "convert": base, 47 | } 48 | 49 | resp = requests.get( 50 | url="https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest", 51 | params=params, 52 | headers=headers, 53 | ) 54 | if resp.status_code != requests.codes.ok: 55 | raise CoinmarketcapApiError( 56 | "Invalid response ({}): {}".format(resp.status_code, resp.text) 57 | ) 58 | data = resp.json() 59 | if data["status"]["error_code"] != 0: 60 | status = data["status"] 61 | raise CoinmarketcapApiError( 62 | "Invalid response ({}): {}".format( 63 | status["error_code"], status["error_message"] 64 | ) 65 | ) 66 | 67 | quote = data["data"][symbol]["quote"][base] 68 | price = Decimal(str(quote["price"])) 69 | date = parse(quote["last_updated"]) 70 | 71 | return source.SourcePrice(price, date, base) 72 | 73 | def get_historical_price(self, ticker, time): 74 | return None 75 | -------------------------------------------------------------------------------- /beanprice/sources/coinmarketcap_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from decimal import Decimal 3 | from os import environ 4 | 5 | from unittest import mock 6 | 7 | import requests 8 | 9 | from beanprice import source 10 | from beanprice.sources import coinmarketcap 11 | 12 | 13 | def response(contents, status_code=requests.codes.ok): 14 | """Return a context manager to patch a JSON response.""" 15 | response = mock.Mock() 16 | response.status_code = status_code 17 | response.text = "" 18 | response.json.return_value = contents 19 | return mock.patch("requests.get", return_value=response) 20 | 21 | 22 | class CoinmarketcapPriceFetcher(unittest.TestCase): 23 | def setUp(self): 24 | environ["COINMARKETCAP_API_KEY"] = "foo" 25 | 26 | def tearDown(self): 27 | del environ["COINMARKETCAP_API_KEY"] 28 | 29 | def test_error_invalid_ticker(self): 30 | with self.assertRaises(ValueError): 31 | coinmarketcap.Source().get_latest_price("INVALID") 32 | 33 | def test_error_network(self): 34 | with response("Foobar", 404): 35 | with self.assertRaises(ValueError): 36 | coinmarketcap.Source().get_latest_price("BTC-CHF") 37 | 38 | def test_error_request(self): 39 | contents = { 40 | "status": { 41 | "error_code": 2, 42 | "error_message": "foobar", 43 | } 44 | } 45 | with response(contents): 46 | with self.assertRaises(ValueError): 47 | coinmarketcap.Source().get_latest_price("BTC-CHF") 48 | 49 | def test_valid_response(self): 50 | contents = { 51 | "data": { 52 | "BTC": { 53 | "quote": { 54 | "CHF": { 55 | "price": 1234.56, 56 | "last_updated": "2018-08-09T21:56:28.000Z", 57 | } 58 | } 59 | } 60 | }, 61 | "status": { 62 | "error_code": 0, 63 | "error_message": "", 64 | }, 65 | } 66 | with response(contents): 67 | srcprice = coinmarketcap.Source().get_latest_price("BTC-CHF") 68 | self.assertIsInstance(srcprice, source.SourcePrice) 69 | self.assertEqual(Decimal("1234.56"), srcprice.price) 70 | self.assertEqual("CHF", srcprice.quote_currency) 71 | 72 | 73 | if __name__ == "__main__": 74 | unittest.main() 75 | -------------------------------------------------------------------------------- /beanprice/sources/eastmoneyfund.py: -------------------------------------------------------------------------------- 1 | """ 2 | A source fetching fund price(net value) from eastmoneyfund(天天基金) 3 | which is a chinese securities company. 4 | 5 | eastmoneyfund supports many kinds of fund, such as fixed income fund, ETF, etc. 6 | this script only supports specific fund which table's header is following: 7 | https://fundf10.eastmoney.com/F10DataApi.aspx?type=lsjz&code=377240. 8 | 9 | fixed income fund is not supported, likes: 10 | https://fundf10.eastmoney.com/F10DataApi.aspx?type=lsjz&code=040003 11 | 12 | the API, as far as I know, is undocumented. 13 | 14 | Prices are denoted in CNY. 15 | Timezone information: the http API requests GMT+8, 16 | the function transfers timezone to GMT+8 automatically 17 | """ 18 | 19 | import datetime 20 | import re 21 | from decimal import Decimal 22 | import requests 23 | from beanprice import source 24 | 25 | 26 | # All of the easymoney funds are in CNY. 27 | CURRENCY = "CNY" 28 | 29 | TIMEZONE = datetime.timezone(datetime.timedelta(hours=+8), "Asia/Shanghai") 30 | 31 | 32 | headers = { 33 | "content-type": "application/json", 34 | "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:22.0)" 35 | "Gecko/20100101 Firefox/22.0", 36 | } 37 | 38 | 39 | class EastMoneyFundError(ValueError): 40 | "An error from the EastMoneyFund API." 41 | 42 | 43 | UnsupportTickerError = EastMoneyFundError("header not match, dont support this ticker type") 44 | 45 | 46 | def parse_page(page): 47 | tr_re = re.compile(r"(.*?)") 48 | item_re = re.compile( 49 | r"(\d{4}-\d{2}-\d{2})(.*?)(.*?)" 50 | "(.*?)(.*?)(.*?)", 51 | re.X, 52 | ) 53 | header_match = re.compile( 54 | r"单位净值累计净值日增长率" 55 | "申购状态赎回状态.*?分红送配" 56 | ) 57 | table = tr_re.findall(page) 58 | if not header_match.match(table[0]): 59 | raise UnsupportTickerError 60 | try: 61 | table = [ 62 | ( 63 | datetime.datetime.fromisoformat(t[0]).replace(hour=15, tzinfo=TIMEZONE), 64 | Decimal(t[1]), 65 | ) 66 | for t in [item_re.match(x).groups() for x in table[1:]] 67 | ] 68 | except AttributeError: 69 | return None 70 | return table 71 | 72 | 73 | def get_price_series( 74 | ticker: str, time_begin: datetime.datetime, time_end: datetime.datetime 75 | ): 76 | base_url = "https://fundf10.eastmoney.com/F10DataApi.aspx" 77 | time_delta_day = (time_end - time_begin).days + 1 78 | pages = time_delta_day // 30 + 1 79 | res = [] 80 | for page in range(1, pages + 1): 81 | query = { 82 | "code": ticker, 83 | "page": str(page), 84 | "sdate": time_begin.astimezone(TIMEZONE).date().isoformat(), 85 | "edate": time_end.astimezone(TIMEZONE).date().isoformat(), 86 | "type": "lsjz", 87 | "per": str(30), 88 | } 89 | response = requests.get(base_url, params=query, headers=headers) 90 | if response.status_code != requests.codes.ok: 91 | raise EastMoneyFundError( 92 | f"Invalid response ({response.status_code}): {response.text}" 93 | ) 94 | 95 | price = parse_page(response.text) 96 | if price is None and page == 1: 97 | raise EastMoneyFundError( 98 | f"Invalid ticker {ticker} or " 99 | f"search day {time_begin.date().isoformat()}~{time_end.date().isoformat()}" 100 | ) 101 | if price is None: 102 | break 103 | res.extend(price) 104 | return res 105 | 106 | 107 | class Source(source.Source): 108 | def get_latest_price(self, ticker): 109 | end_time = datetime.datetime.now(TIMEZONE) 110 | begin_time = end_time - datetime.timedelta(days=10) 111 | prices = get_price_series(ticker, begin_time, end_time) 112 | last_price = prices[0] 113 | return source.SourcePrice(last_price[1], last_price[0], CURRENCY) 114 | 115 | def get_historical_price(self, ticker, time): 116 | prices = get_price_series(ticker, time - datetime.timedelta(days=10), time) 117 | last_price = prices[0] 118 | return source.SourcePrice(last_price[1], last_price[0], CURRENCY) 119 | 120 | def get_prices_series(self, ticker, time_begin, time_end): 121 | res = [ 122 | source.SourcePrice(x[1], x[0], CURRENCY) 123 | for x in get_price_series(ticker, time_begin, time_end) 124 | ] 125 | return sorted(res, key=lambda x: x.time) 126 | -------------------------------------------------------------------------------- /beanprice/sources/eastmoneyfund_test.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import unittest 3 | from decimal import Decimal 4 | 5 | from unittest import mock 6 | from dateutil import tz 7 | 8 | import requests 9 | 10 | from beanprice.sources import eastmoneyfund 11 | from beanprice import source 12 | 13 | 14 | # ruff: noqa: E501,RUF001 15 | 16 | CONTENTS = """ 17 | var apidata={ content:"
净值日期单位净值累计净值日增长率申购状态赎回状态分红送配
2020-10-095.18905.18904.11%开放申购开放赎回
2020-09-304.98404.98400.12%开放申购开放赎回
2020-09-294.97804.97801.14%开放申购开放赎回
2020-09-284.92204.92200.22%开放申购开放赎回
2020-09-254.91104.91100.88%开放申购开放赎回
2020-09-244.86804.8680-3.81%开放申购开放赎回
2020-09-235.06105.06102.41%开放申购开放赎回
2020-09-224.94204.9420-1.02%开放申购开放赎回
2020-09-214.99304.9930-1.29%开放申购开放赎回
2020-09-185.05805.05800.48%开放申购开放赎回
2020-09-175.03405.03400.60%开放申购开放赎回
2020-09-165.00405.0040-1.28%开放申购开放赎回
2020-09-155.06905.06901.06%开放申购开放赎回
2020-09-145.01605.01600.42%开放申购开放赎回
2020-09-114.99504.99503.39%开放申购开放赎回
2020-09-104.83104.8310-0.29%开放申购开放赎回
",records:16,pages:1,curpage:1};\ 18 | """ 19 | 20 | UNSUPPORT_CONTENT = """ 21 | var apidata={ content:"
净值日期每万份收益7日年化收益率（%）申购状态赎回状态分红送配
2020-09-100.42301.5730%开放申购开放赎回
",records:1,pages:1,curpage:1};""" 26 | 27 | 28 | def response(contents, status_code=requests.codes.ok): 29 | """Return a context manager to patch a JSON response.""" 30 | response = mock.Mock() 31 | response.status_code = status_code 32 | response.text = contents 33 | return mock.patch("requests.get", return_value=response) 34 | 35 | 36 | class EastMoneyFundFetcher(unittest.TestCase): 37 | def test_error_network(self): 38 | with response(None, 404): 39 | with self.assertRaises(ValueError): 40 | eastmoneyfund.get_price_series( 41 | "377240", datetime.datetime.now(), datetime.datetime.now() 42 | ) 43 | 44 | def test_unsupport_page(self): 45 | with response(UNSUPPORT_CONTENT): 46 | with self.assertRaises(ValueError) as exc: 47 | eastmoneyfund.get_price_series( 48 | "377240", datetime.datetime.now(), datetime.datetime.now() 49 | ) 50 | self.assertEqual(eastmoneyfund.UnsupportTickerError, exc.exception) 51 | 52 | def test_latest_price(self): 53 | with response(CONTENTS): 54 | srcprice = eastmoneyfund.Source().get_latest_price("377240") 55 | self.assertIsInstance(srcprice, source.SourcePrice) 56 | self.assertEqual(Decimal("5.1890"), srcprice.price) 57 | self.assertEqual("CNY", srcprice.quote_currency) 58 | 59 | def test_historical_price(self): 60 | with response(CONTENTS): 61 | time = datetime.datetime(2018, 3, 27, 0, 0, 0, tzinfo=tz.tzutc()) 62 | srcprice = eastmoneyfund.Source().get_historical_price("377240", time) 63 | self.assertIsInstance(srcprice, source.SourcePrice) 64 | self.assertEqual(Decimal("5.1890"), srcprice.price) 65 | self.assertEqual("CNY", srcprice.quote_currency) 66 | self.assertEqual( 67 | datetime.datetime(2020, 10, 9, 15, 0, 0, tzinfo=eastmoneyfund.TIMEZONE), 68 | srcprice.time, 69 | ) 70 | 71 | def test_get_prices_series(self): 72 | with response(CONTENTS): 73 | time = datetime.datetime(2018, 3, 27, 0, 0, 0, tzinfo=tz.tzutc()) 74 | srcprice = eastmoneyfund.Source().get_prices_series( 75 | "377240", time - datetime.timedelta(days=10), time 76 | ) 77 | self.assertIsInstance(srcprice, list) 78 | self.assertIsInstance(srcprice[-1], source.SourcePrice) 79 | self.assertEqual(Decimal("5.1890"), srcprice[-1].price) 80 | self.assertEqual("CNY", srcprice[-1].quote_currency) 81 | self.assertEqual( 82 | datetime.datetime(2020, 10, 9, 15, 0, 0, tzinfo=eastmoneyfund.TIMEZONE), 83 | srcprice[-1].time, 84 | ) 85 | self.assertIsInstance(srcprice[0], source.SourcePrice) 86 | self.assertEqual(Decimal("4.8310"), srcprice[0].price) 87 | self.assertEqual("CNY", srcprice[0].quote_currency) 88 | self.assertEqual( 89 | datetime.datetime(2020, 9, 10, 15, 0, 0, tzinfo=eastmoneyfund.TIMEZONE), 90 | srcprice[0].time, 91 | ) 92 | 93 | 94 | if __name__ == "__main__": 95 | unittest.main() 96 | -------------------------------------------------------------------------------- /beanprice/sources/ecbrates.py: -------------------------------------------------------------------------------- 1 | """A source fetching exchange rates using European Central Bank's datasets 2 | 3 | This source leverages daily avarage rates to/from EUR. For other currency pairs 4 | the final rate is derived by dividing rates to/from EUR. 5 | 6 | Valid tickers are in the form "XXX-YYY", such as "EUR-CHF", which denotes rate EUR->CHF 7 | 8 | Here is the API documentation: 9 | https://data.ecb.europa.eu/help/api/overview 10 | 11 | Timezone information: Input and output datetimes are specified via UTC 12 | timestamps. 13 | """ 14 | 15 | from decimal import Decimal, getcontext 16 | 17 | import re 18 | import csv 19 | from io import StringIO 20 | from dateutil.tz import tz 21 | from dateutil.parser import parse 22 | import requests 23 | 24 | from beanprice import source 25 | 26 | 27 | class ECBRatesError(ValueError): 28 | "An error from the ECB Rates." 29 | 30 | 31 | def _parse_ticker(ticker): 32 | """Parse the base and quote currencies from the ticker. 33 | 34 | Args: 35 | ticker: A string, the symbol in XXX-YYY format. 36 | Returns: 37 | A pair of (base, quote) currencies. 38 | """ 39 | match = re.match(r"^(?P\w+)-(?P\w+)$", ticker) 40 | if not match: 41 | raise ValueError('Invalid ticker. Use "BASE-SYMBOL" format.') 42 | return match.groups() 43 | 44 | 45 | def _get_rate_EUR_to_CCY(currency, date): 46 | # Call API 47 | symbol = f"D.{currency}.EUR.SP00.A" 48 | params = {"format": "csvdata", "detail": "full", "lastNObservations": 1} 49 | if date is not None: 50 | params["endPeriod"] = date 51 | url = f"https://data-api.ecb.europa.eu/service/data/EXR/{symbol}" 52 | response = requests.get(url, params=params) 53 | if response.status_code != requests.codes.ok: 54 | raise ECBRatesError( 55 | f"Invalid response ({response.status_code}): {response.text}" 56 | ) 57 | 58 | # Parse results to a DictReader iterator 59 | results = csv.DictReader(StringIO(response.text)) 60 | 61 | # Retrieve exchange rate 62 | try: 63 | observation = next(results) 64 | except StopIteration: 65 | # When there's no data for a given date, an empty string is returned 66 | return None, None, None 67 | else: 68 | # Checking only the first observation and raising errors if there's a date mismatch 69 | rate = observation.get("OBS_VALUE") 70 | obs_date = observation.get("TIME_PERIOD") 71 | decimals = observation.get("DECIMALS") 72 | precision = int(decimals) + len(rate.split(".")[0].lstrip("0")) 73 | return Decimal(rate), obs_date, precision 74 | 75 | 76 | def _get_quote(ticker, date): 77 | base, symbol = _parse_ticker(ticker) 78 | 79 | if base == symbol: 80 | raise ECBRatesError( 81 | f"Base currency {base} must be different than symbol currency {symbol}" 82 | ) 83 | 84 | # Get EUR rates by calling the API (or use defaults) 85 | if base == "EUR" and symbol != "EUR": 86 | eur_to_symbol, symbol_rate_date, symbol_rate_precision = _get_rate_EUR_to_CCY( 87 | symbol, date 88 | ) 89 | eur_to_base = Decimal(1) 90 | base_rate_date = symbol_rate_date 91 | base_rate_precision = 28 92 | elif base != "EUR" and symbol == "EUR": 93 | eur_to_base, base_rate_date, base_rate_precision = _get_rate_EUR_to_CCY( 94 | base, date 95 | ) 96 | eur_to_symbol = Decimal(1) 97 | symbol_rate_date = base_rate_date 98 | symbol_rate_precision = 28 99 | else: 100 | eur_to_base, base_rate_date, base_rate_precision = _get_rate_EUR_to_CCY( 101 | base, date 102 | ) 103 | eur_to_symbol, symbol_rate_date, symbol_rate_precision = _get_rate_EUR_to_CCY( 104 | symbol, date 105 | ) 106 | 107 | # Raise error if retrieved subrates for differnt dates 108 | if base_rate_date != symbol_rate_date: 109 | raise ECBRatesError( 110 | f"Subrates for different dates: ({base}, {base_rate_date}) \ 111 | vs. ({symbol}, {symbol_rate_date})" 112 | ) 113 | 114 | # Calculate base -> symbol 115 | if eur_to_symbol is None or eur_to_base is None: 116 | raise ECBRatesError( 117 | f"At least one of the subrates returned None: \ 118 | (EUR{symbol}: {eur_to_symbol}, EUR{base}: {eur_to_base})" 119 | ) 120 | 121 | # Derive precision from sunrates (must be at least 5) 122 | minimal_precision = 5 123 | getcontext().prec = max( 124 | minimal_precision, min(base_rate_precision, symbol_rate_precision) 125 | ) 126 | price = eur_to_symbol / eur_to_base 127 | time = parse(base_rate_date).replace(tzinfo=tz.tzutc()) 128 | return source.SourcePrice(price, time, symbol) 129 | 130 | 131 | class Source(source.Source): 132 | 133 | def get_latest_price(self, ticker): 134 | return _get_quote(ticker, None) 135 | 136 | def get_historical_price(self, ticker, time): 137 | return _get_quote(ticker, time.date().isoformat()) 138 | -------------------------------------------------------------------------------- /beanprice/sources/ecbrates_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from datetime import datetime 3 | from decimal import Decimal 4 | from unittest import mock 5 | import requests 6 | from dateutil import tz 7 | from beanprice import source 8 | from beanprice.sources import ecbrates 9 | 10 | 11 | ECB_CSV = """KEY,FREQ,CURRENCY,CURRENCY_DENOM,EXR_TYPE,EXR_SUFFIX,TIME_PERIOD,OBS_VALUE,OBS\ 12 | _STATUS,OBS_CONF,OBS_PRE_BREAK,OBS_COM,TIME_FORMAT,BREAKS,COLLECTION,COMPILING_ORG,DISS_ORG\ 13 | ,DOM_SER_IDS,PUBL_ECB,PUBL_MU,PUBL_PUBLIC,UNIT_INDEX_BASE,COMPILATION,COVERAGE,DECIMALS,NAT\ 14 | _TITLE,SOURCE_AGENCY,SOURCE_PUB,TITLE,TITLE_COMPL,UNIT,UNIT_MULT 15 | EXR.D.SEK.EUR.SP00.A,D,SEK,EUR,SP00,A,2024-12-24,11.5335,A,F,,,P1D,,A,,,,,,,,,,4,,4F0,,Euro\ 16 | /Swedish krona,"ECB reference exchange rate, Euro/Swedish krona, 2:15 pm (C.E.T.)",SEK,0 17 | """ 18 | 19 | ECB_CSV_HIST = """KEY,FREQ,CURRENCY,CURRENCY_DENOM,EXR_TYPE,EXR_SUFFIX,TIME_PERIOD,OBS_VALU\ 20 | E,OBS_STATUS,OBS_CONF,OBS_PRE_BREAK,OBS_COM,TIME_FORMAT,BREAKS,COLLECTION,COMPILING_ORG,DIS\ 21 | S_ORG,DOM_SER_IDS,PUBL_ECB,PUBL_MU,PUBL_PUBLIC,UNIT_INDEX_BASE,COMPILATION,COVERAGE,DECIMAL\ 22 | S,NAT_TITLE,SOURCE_AGENCY,SOURCE_PUB,TITLE,TITLE_COMPL,UNIT,UNIT_MULT 23 | EXR.D.SEK.EUR.SP00.A,D,SEK,EUR,SP00,A,2024-12-06,11.523,A,F,,,P1D,,A,,,,,,,,,,4,,4F0,,Euro/\ 24 | Swedish krona,"ECB reference exchange rate, Euro/Swedish krona, 2:15 pm (C.E.T.)",SEK,0 25 | """ 26 | 27 | 28 | def response(contents, status_code=requests.codes.ok): 29 | """Return a context manager to patch a CSV response.""" 30 | response = mock.Mock() 31 | response.status_code = status_code 32 | response.text = contents 33 | return mock.patch("requests.get", return_value=response) 34 | 35 | 36 | class ECBRatesErrorFetcher(unittest.TestCase): 37 | def test_error_invalid_ticker(self): 38 | with self.assertRaises(ValueError) as exc: 39 | ecbrates.Source().get_latest_price("INVALID") 40 | 41 | def test_error_network(self): 42 | with response("Foobar", 404): 43 | with self.assertRaises(ValueError) as exc: 44 | ecbrates.Source().get_latest_price("EUR-SEK") 45 | 46 | def test_empty_response(self): 47 | with response("", 200): 48 | with self.assertRaises(ecbrates.ECBRatesError) as exc: 49 | ecbrates.Source().get_latest_price("EUR-SEK") 50 | 51 | def test_valid_response(self): 52 | contents = ECB_CSV 53 | with response(contents): 54 | srcprice = ecbrates.Source().get_latest_price("EUR-SEK") 55 | self.assertIsInstance(srcprice, source.SourcePrice) 56 | self.assertEqual(Decimal("11.5335"), srcprice.price) 57 | self.assertEqual("SEK", srcprice.quote_currency) 58 | self.assertIsInstance(srcprice.time, datetime) 59 | self.assertEqual( 60 | datetime(2024, 12, 24, 0, 0, 0, tzinfo=tz.tzutc()), srcprice.time 61 | ) 62 | 63 | def test_historical_price(self): 64 | time = datetime(2024, 12, 6, 16, 0, 0, tzinfo=tz.tzlocal()).astimezone( 65 | tz.tzutc() 66 | ) 67 | contents = ECB_CSV_HIST 68 | with response(contents): 69 | srcprice = ecbrates.Source().get_historical_price("EUR-SEK", time) 70 | self.assertIsInstance(srcprice, source.SourcePrice) 71 | self.assertEqual(Decimal("11.523"), srcprice.price) 72 | self.assertEqual("SEK", srcprice.quote_currency) 73 | self.assertIsInstance(srcprice.time, datetime) 74 | self.assertEqual( 75 | datetime(2024, 12, 6, 0, 0, 0, tzinfo=tz.tzutc()), srcprice.time 76 | ) 77 | 78 | 79 | if __name__ == "__main__": 80 | unittest.main() 81 | -------------------------------------------------------------------------------- /beanprice/sources/iex.py: -------------------------------------------------------------------------------- 1 | """Fetch prices from the IEX 1.0 public API. 2 | 3 | This is a really fantastic exchange API with a lot of relevant information. 4 | 5 | Timezone information: There is currency no support for historical prices. The 6 | output datetime is provided as a UNIX timestamp. 7 | """ 8 | 9 | __copyright__ = "Copyright (C) 2018-2020 Martin Blais" 10 | __license__ = "GNU GPLv2" 11 | 12 | import datetime 13 | from decimal import Decimal 14 | 15 | from dateutil import tz 16 | import requests 17 | 18 | from beanprice import source 19 | 20 | 21 | class IEXError(ValueError): 22 | "An error from the IEX API." 23 | 24 | 25 | def fetch_quote(ticker): 26 | """Fetch the latest price for the given ticker.""" 27 | 28 | url = "https://api.iextrading.com/1.0/tops/last?symbols={}".format(ticker.upper()) 29 | response = requests.get(url) 30 | if response.status_code != requests.codes.ok: 31 | raise IEXError( 32 | "Invalid response ({}): {}".format(response.status_code, response.text) 33 | ) 34 | 35 | results = response.json() 36 | if len(results) != 1: 37 | raise IEXError("Invalid number of responses from IEX: {}".format(response.text)) 38 | result = results[0] 39 | 40 | price = Decimal(result["price"]).quantize(Decimal("0.01")) 41 | 42 | # IEX is American markets. 43 | us_timezone = tz.gettz("America/New_York") 44 | time = datetime.datetime.fromtimestamp(result["time"] / 1000) 45 | time = time.astimezone(us_timezone) 46 | 47 | # As far as can tell, all the instruments on IEX are priced in USD. 48 | return source.SourcePrice(price, time, "USD") 49 | 50 | 51 | class Source(source.Source): 52 | "IEX API price extractor." 53 | 54 | def get_latest_price(self, ticker): 55 | """See contract in beanprice.source.Source.""" 56 | return fetch_quote(ticker) 57 | 58 | def get_historical_price(self, ticker, time): 59 | """See contract in beanprice.source.Source.""" 60 | raise NotImplementedError( 61 | "This is now implemented at https://iextrading.com/developers/docs/#hist and " 62 | "needs to be added here." 63 | ) 64 | -------------------------------------------------------------------------------- /beanprice/sources/iex_test.py: -------------------------------------------------------------------------------- 1 | __copyright__ = "Copyright (C) 2018-2020 Martin Blais" 2 | __license__ = "GNU GPLv2" 3 | 4 | import datetime 5 | import unittest 6 | from unittest import mock 7 | from decimal import Decimal 8 | 9 | from dateutil import tz 10 | import requests 11 | 12 | from beanprice import date_utils 13 | from beanprice import source 14 | from beanprice.sources import iex 15 | 16 | 17 | def response(contents, status_code=requests.codes.ok): 18 | """Produce a context manager to patch a JSON response.""" 19 | response = mock.Mock() 20 | response.status_code = status_code 21 | response.text = "" 22 | response.json.return_value = contents 23 | return mock.patch("requests.get", return_value=response) 24 | 25 | 26 | class IEXPriceFetcher(unittest.TestCase): 27 | def test_error_network(self): 28 | with response(None, 404): 29 | with self.assertRaises(ValueError) as exc: 30 | iex.fetch_quote("AAPL") 31 | self.assertRegex(exc.message, "premium") 32 | 33 | def _test_valid_response(self): 34 | contents = [{"symbol": "HOOL", "price": 183.61, "size": 100, "time": 1590177596030}] 35 | with response(contents): 36 | srcprice = iex.fetch_quote("HOOL") 37 | self.assertIsInstance(srcprice, source.SourcePrice) 38 | self.assertEqual(Decimal("183.61"), srcprice.price) 39 | self.assertEqual( 40 | datetime.datetime(2020, 5, 22, 19, 59, 56, 30000, tzinfo=tz.tzutc()), 41 | srcprice.time.astimezone(tz.tzutc()), 42 | ) 43 | self.assertEqual("USD", srcprice.quote_currency) 44 | 45 | def test_valid_response(self): 46 | for tzname in "America/New_York", "Europe/Berlin", "Asia/Tokyo": 47 | with date_utils.intimezone(tzname): 48 | self._test_valid_response() 49 | 50 | 51 | if __name__ == "__main__": 52 | unittest.main() 53 | -------------------------------------------------------------------------------- /beanprice/sources/oanda.py: -------------------------------------------------------------------------------- 1 | """A source fetching currency prices from OANDA. 2 | 3 | Valid tickers are in the form "XXX_YYY", such as "EUR_USD". 4 | 5 | Here is the API documentation: 6 | https://developer.oanda.com/rest-live/rates/ 7 | 8 | For example: 9 | https://api-fxtrade.oanda.com/v1/candles?instrument=EUR_USD&granularity=D&start=2016-03-27T00%3A00%3A00Z&end=2016-04-04T00%3A00%3A00Z&candleFormat=midpoint 10 | 11 | Timezone information: Input and output datetimes are specified via UTC 12 | timestamps. 13 | """ 14 | 15 | __copyright__ = "Copyright (C) 2018-2020 Martin Blais" 16 | __license__ = "GNU GPLv2" 17 | 18 | import re 19 | import datetime 20 | import json 21 | import logging 22 | from urllib import parse 23 | from decimal import Decimal 24 | 25 | from dateutil import tz 26 | 27 | from beanprice import source 28 | from beanprice import net_utils 29 | 30 | 31 | URL = "https://api-fxtrade.oanda.com/v1/candles" 32 | 33 | 34 | def _get_currencies(ticker): 35 | """Parse the base and quote currencies from the ticker. 36 | 37 | Args: 38 | ticker: A string, the symbol in XXX_YYY format. 39 | Returns: 40 | A pair of (base, quote) currencies. 41 | """ 42 | match = re.match("([A-Z]+)_([A-Z]+)$", ticker) 43 | if not match: 44 | return None, None 45 | return match.groups() 46 | 47 | 48 | def _fetch_candles(params): 49 | """Fetch the given URL from OANDA and return a list of (utc-time, price). 50 | 51 | Args: 52 | params: A dict of URL params values. 53 | Returns: 54 | A sorted list of (time, price) points. 55 | """ 56 | 57 | url = "?".join((URL, parse.urlencode(sorted(params.items())))) 58 | logging.info("Fetching '%s'", url) 59 | 60 | # Fetch the data. 61 | response = net_utils.retrying_urlopen(url) 62 | if response is None: 63 | return None 64 | data_string = response.read().decode("utf-8") 65 | 66 | # Parse it. 67 | data = json.loads(data_string, parse_float=Decimal) 68 | try: 69 | # Find the candle with the latest time before the given time we're searching 70 | # for. 71 | time_prices = [] 72 | candles = sorted(data["candles"], key=lambda candle: candle["time"]) 73 | for candle in candles: 74 | candle_dt_utc = datetime.datetime.strptime( 75 | candle["time"], r"%Y-%m-%dT%H:%M:%S.%fZ" 76 | ).replace(tzinfo=tz.tzutc()) 77 | candle_price = Decimal(candle["openMid"]) 78 | time_prices.append((candle_dt_utc, candle_price)) 79 | except KeyError: 80 | logging.error("Unexpected response data: %s", data) 81 | return None 82 | return sorted(time_prices) 83 | 84 | 85 | def _fetch_price(params_dict, time): 86 | """Fetch a price from OANDA using the given parameters.""" 87 | ticker = params_dict["instrument"] 88 | _, quote_currency = _get_currencies(ticker) 89 | if quote_currency is None: 90 | logging.error("Invalid price source ticker '%s'; must be like 'EUR_USD'", ticker) 91 | return 92 | 93 | time_prices = _fetch_candles(params_dict) 94 | if not time_prices: 95 | logging.error("No prices returned.") 96 | return 97 | 98 | # Get all the prices before and on the same date and find the latest. 99 | sorted_prices = [item for item in time_prices if item[0] <= time] 100 | if not sorted_prices: 101 | logging.error("No prices matched.") 102 | return 103 | 104 | time, price = sorted_prices[-1] 105 | return source.SourcePrice(price, time, quote_currency) 106 | 107 | 108 | class Source(source.Source): 109 | "OANDA price source extractor." 110 | 111 | def get_latest_price(self, ticker): 112 | """See contract in beanprice.source.Source.""" 113 | time = datetime.datetime.now(tz.tzutc()) 114 | params_dict = { 115 | "instrument": ticker, 116 | "granularity": "S5", # Every two hours. 117 | "count": "10", 118 | "candleFormat": "midpoint", 119 | } 120 | return _fetch_price(params_dict, time) 121 | 122 | def get_historical_price(self, ticker, time): 123 | """See contract in beanprice.source.Source.""" 124 | time = time.astimezone(tz.tzutc()) 125 | query_interval_begin = time - datetime.timedelta(days=5) 126 | query_interval_end = time + datetime.timedelta(days=1) 127 | params_dict = { 128 | "instrument": ticker, 129 | "granularity": "H2", # Every two hours. 130 | "candleFormat": "midpoint", 131 | "start": query_interval_begin.isoformat("T"), 132 | "end": query_interval_end.isoformat("T"), 133 | } 134 | return _fetch_price(params_dict, time) 135 | -------------------------------------------------------------------------------- /beanprice/sources/oanda_test.py: -------------------------------------------------------------------------------- 1 | """Test for price extractor of OANDA.""" 2 | 3 | __copyright__ = "Copyright (C) 2018-2020 Martin Blais" 4 | __license__ = "GNU GPLv2" 5 | 6 | import os 7 | import time 8 | import datetime 9 | import unittest 10 | from unittest import mock 11 | from decimal import Decimal 12 | 13 | from dateutil import tz 14 | 15 | from beanprice import date_utils 16 | from beanprice import net_utils 17 | from beanprice import source 18 | from beanprice.sources import oanda 19 | 20 | 21 | UTC = tz.tzutc() 22 | 23 | 24 | def response(code, contents=None): 25 | urlopen = mock.MagicMock(return_value=None) 26 | if isinstance(contents, str): 27 | response = mock.MagicMock() 28 | response.read = mock.MagicMock(return_value=contents.encode("utf-8")) 29 | response.getcode = mock.MagicMock(return_value=200) 30 | urlopen.return_value = response 31 | return mock.patch.object(net_utils, "retrying_urlopen", urlopen) 32 | 33 | 34 | class TestOandaMisc(unittest.TestCase): 35 | def test_get_currencies(self): 36 | self.assertEqual(("USD", "CAD"), oanda._get_currencies("USD_CAD")) 37 | 38 | def test_get_currencies_invalid(self): 39 | self.assertEqual((None, None), oanda._get_currencies("USDCAD")) 40 | 41 | 42 | class TimezoneTestBase: 43 | def setUp(self): 44 | tz_value = "Europe/Berlin" 45 | self.tz_old = os.environ.get("TZ", None) 46 | os.environ["TZ"] = tz_value 47 | time.tzset() 48 | 49 | def tearDown(self): 50 | if self.tz_old is None: 51 | del os.environ["TZ"] 52 | else: 53 | os.environ["TZ"] = self.tz_old 54 | time.tzset() 55 | 56 | 57 | class TestOandaFetchCandles(TimezoneTestBase, unittest.TestCase): 58 | @response(404) 59 | def test_null_response(self): 60 | self.assertIs(None, oanda._fetch_candles({})) 61 | 62 | @response( 63 | 200, 64 | """ 65 | { 66 | "instrument" : "USD_CAD", 67 | "granularity" : "S5" 68 | } 69 | """, 70 | ) 71 | def test_key_error(self): 72 | self.assertIs(None, oanda._fetch_candles({})) 73 | 74 | @response( 75 | 200, 76 | """ 77 | { 78 | "instrument" : "USD_CAD", 79 | "granularity" : "S5", 80 | "candles" : [ 81 | { 82 | "time" : "2017-01-23T00:45:15.000000Z", 83 | "openMid" : 1.330115, 84 | "highMid" : 1.33012, 85 | "lowMid" : 1.33009, 86 | "closeMid" : 1.33009, 87 | "volume" : 9, 88 | "complete" : true 89 | }, 90 | { 91 | "time" : "2017-01-23T00:45:20.000000Z", 92 | "openMid" : 1.330065, 93 | "highMid" : 1.330065, 94 | "lowMid" : 1.330065, 95 | "closeMid" : 1.330065, 96 | "volume" : 1, 97 | "complete" : true 98 | } 99 | ] 100 | } 101 | """, 102 | ) 103 | def test_valid(self): 104 | self.assertEqual( 105 | [ 106 | ( 107 | datetime.datetime(2017, 1, 23, 0, 45, 15, tzinfo=UTC), 108 | Decimal("1.330115"), 109 | ), 110 | ( 111 | datetime.datetime(2017, 1, 23, 0, 45, 20, tzinfo=UTC), 112 | Decimal("1.330065"), 113 | ), 114 | ], 115 | oanda._fetch_candles({}), 116 | ) 117 | 118 | 119 | class TestOandaGetLatest(unittest.TestCase): 120 | def setUp(self): 121 | self.fetcher = oanda.Source() 122 | 123 | def test_invalid_ticker(self): 124 | srcprice = self.fetcher.get_latest_price("NOTATICKER") 125 | self.assertIsNone(srcprice) 126 | 127 | def test_no_candles(self): 128 | with mock.patch.object(oanda, "_fetch_candles", return_value=None): 129 | self.assertEqual(None, self.fetcher.get_latest_price("USD_CAD")) 130 | 131 | def _test_valid(self): 132 | candles = [ 133 | (datetime.datetime(2017, 1, 21, 0, 45, 15, tzinfo=UTC), Decimal("1.330115")), 134 | (datetime.datetime(2017, 1, 21, 0, 45, 20, tzinfo=UTC), Decimal("1.330065")), 135 | ] 136 | with mock.patch.object(oanda, "_fetch_candles", return_value=candles): 137 | srcprice = self.fetcher.get_latest_price("USD_CAD") 138 | # Latest price, with current time as time. 139 | self.assertEqual( 140 | source.SourcePrice( 141 | Decimal("1.330065"), 142 | datetime.datetime(2017, 1, 21, 0, 45, 20, tzinfo=UTC), 143 | "CAD", 144 | ), 145 | srcprice, 146 | ) 147 | 148 | def test_valid(self): 149 | for tzname in "America/New_York", "Europe/Berlin", "Asia/Tokyo": 150 | with date_utils.intimezone(tzname): 151 | self._test_valid() 152 | 153 | 154 | class TestOandaGetHistorical(TimezoneTestBase, unittest.TestCase): 155 | def setUp(self): 156 | self.fetcher = oanda.Source() 157 | super().setUp() 158 | 159 | def test_invalid_ticker(self): 160 | srcprice = self.fetcher.get_latest_price("NOTATICKER") 161 | self.assertIsNone(srcprice) 162 | 163 | def test_no_candles(self): 164 | with mock.patch.object(oanda, "_fetch_candles", return_value=None): 165 | self.assertEqual(None, self.fetcher.get_latest_price("USD_CAD")) 166 | 167 | def _check_valid(self, query_date, out_time, out_price): 168 | candles = [ 169 | (datetime.datetime(2017, 1, 21, 0, 0, 0, tzinfo=UTC), Decimal("1.3100")), 170 | (datetime.datetime(2017, 1, 21, 8, 0, 0, tzinfo=UTC), Decimal("1.3300")), 171 | (datetime.datetime(2017, 1, 21, 16, 0, 0, tzinfo=UTC), Decimal("1.3500")), 172 | (datetime.datetime(2017, 1, 22, 0, 0, 0, tzinfo=UTC), Decimal("1.3700")), 173 | (datetime.datetime(2017, 1, 22, 8, 0, 0, tzinfo=UTC), Decimal("1.3900")), 174 | (datetime.datetime(2017, 1, 22, 16, 0, 0, tzinfo=UTC), Decimal("1.4100")), 175 | (datetime.datetime(2017, 1, 23, 0, 0, 0, tzinfo=UTC), Decimal("1.4300")), 176 | (datetime.datetime(2017, 1, 23, 8, 0, 0, tzinfo=UTC), Decimal("1.4500")), 177 | (datetime.datetime(2017, 1, 23, 16, 0, 0, tzinfo=UTC), Decimal("1.4700")), 178 | ] 179 | with mock.patch.object(oanda, "_fetch_candles", return_value=candles): 180 | query_time = datetime.datetime.combine( 181 | query_date, time=datetime.time(16, 0, 0), tzinfo=UTC 182 | ) 183 | srcprice = self.fetcher.get_historical_price("USD_CAD", query_time) 184 | if out_time is not None: 185 | self.assertEqual(source.SourcePrice(out_price, out_time, "CAD"), srcprice) 186 | else: 187 | self.assertEqual(None, srcprice) 188 | 189 | def test_valid_same_date(self): 190 | for tzname in "America/New_York", "Europe/Berlin", "Asia/Tokyo": 191 | with date_utils.intimezone(tzname): 192 | self._check_valid( 193 | datetime.date(2017, 1, 22), 194 | datetime.datetime(2017, 1, 22, 16, 0, tzinfo=UTC), 195 | Decimal("1.4100"), 196 | ) 197 | 198 | def test_valid_before(self): 199 | for tzname in "America/New_York", "Europe/Berlin", "Asia/Tokyo": 200 | with date_utils.intimezone(tzname): 201 | self._check_valid( 202 | datetime.date(2017, 1, 23), 203 | datetime.datetime(2017, 1, 23, 16, 0, tzinfo=UTC), 204 | Decimal("1.4700"), 205 | ) 206 | 207 | def test_valid_after(self): 208 | for tzname in "America/New_York", "Europe/Berlin", "Asia/Tokyo": 209 | with date_utils.intimezone(tzname): 210 | self._check_valid(datetime.date(2017, 1, 20), None, None) 211 | 212 | 213 | if __name__ == "__main__": 214 | unittest.main() 215 | -------------------------------------------------------------------------------- /beanprice/sources/quandl.py: -------------------------------------------------------------------------------- 1 | """Fetch prices from Quandl's simple URL-based API. 2 | 3 | Quandl is a useful source of alternative data and it offers a simple REST API 4 | that serves CSV and JSON and XML formats. There's also a Python client library, 5 | but we specifically avoid using that here, in order to keep Beancount 6 | dependency-free. 7 | 8 | Many of the datasets are freely available, which is why this is included here. 9 | You can get information about the available databases and associated lists of 10 | symbols you can use here: https://www.quandl.com/search 11 | 12 | If you have a paid account and would like to be able to access the premium 13 | databases from the Quandl site, you can set QUANDL_API_KEY environment variable. 14 | 15 | Use the ":" format to refer to Quandl symbols. 16 | Note that their symbols are usually identified by "/". 17 | If Quandl's output for the symbol you're interested in doesn't contain 18 | the default "Adj. Close" or "Close" column, you may specify the column to use 19 | after an additional semicolon, e.g. "::". 20 | If the column name contains spaces, use underscores instead, in order to not 21 | collide with the general price source syntax, e.g. "LBMA:GOLD:USD_(PM)". 22 | 23 | (For now, this supports only the Time-Series API. There is also a Tables API, 24 | which could easily get integrated. We would just have to encode the 25 | 'datatable_code' and 'format' and perhaps other fields in the ticker name.) 26 | 27 | Timezone information: Input and output datetimes are limited to dates, and I 28 | believe the dates are presumed to live in the timezone of each particular data 29 | source. (It's unclear, not documented.) 30 | """ 31 | 32 | __copyright__ = "Copyright (C) 2018-2020 Martin Blais" 33 | __license__ = "GNU GPLv2" 34 | 35 | import collections 36 | import datetime 37 | import re 38 | import os 39 | from decimal import Decimal 40 | 41 | from dateutil import tz 42 | 43 | import requests 44 | 45 | from beanprice import source 46 | 47 | 48 | class QuandlError(ValueError): 49 | "An error from the Quandl API." 50 | 51 | 52 | TickerSpec = collections.namedtuple("TickerSpec", "database dataset column") 53 | 54 | 55 | def parse_ticker(ticker): 56 | """Convert ticker to Quandl codes.""" 57 | if not re.match(r"[A-Z0-9]+:[A-Z0-9]+(:[^:; ]+)?$", ticker): 58 | raise ValueError('Invalid code. Use ":[:]" format.') 59 | split = ticker.split(":") 60 | if len(split) == 2: 61 | return TickerSpec(split[0], split[1], None) 62 | return TickerSpec(split[0], split[1], split[2].replace("_", " ")) 63 | 64 | 65 | def fetch_time_series(ticker, time=None): 66 | """Fetch""" 67 | # Create request payload. 68 | ticker_spec = parse_ticker(ticker) 69 | url = "https://www.quandl.com/api/v3/datasets/{}/{}.json".format( 70 | ticker_spec.database, ticker_spec.dataset 71 | ) 72 | payload = {"limit": 1} 73 | if time is not None: 74 | date = time.date() 75 | payload["start_date"] = (date - datetime.timedelta(days=10)).isoformat() 76 | payload["end_date"] = date.isoformat() 77 | 78 | # Add API key, if it is set in the environment. 79 | if "QUANDL_API_KEY" in os.environ: 80 | payload["api_key"] = os.environ["QUANDL_API_KEY"] 81 | 82 | # Fetch and process errors. 83 | response = requests.get(url, params=payload) 84 | if response.status_code != requests.codes.ok: 85 | raise QuandlError( 86 | "Invalid response ({}): {}".format(response.status_code, response.text) 87 | ) 88 | result = response.json() 89 | if "quandl_error" in result: 90 | raise QuandlError(result["quandl_error"]["message"]) 91 | 92 | # Parse result container. 93 | dataset = result["dataset"] 94 | column_names = dataset["column_names"] 95 | date_index = column_names.index("Date") 96 | if ticker_spec.column is not None: 97 | data_index = column_names.index(ticker_spec.column) 98 | else: 99 | try: 100 | data_index = column_names.index("Adj. Close") 101 | except ValueError: 102 | data_index = column_names.index("Close") 103 | data = dataset["data"][0] 104 | 105 | # Gather time and assume it's in UTC timezone (Quandl does not provide the 106 | # market's timezone). 107 | time = datetime.datetime.strptime(data[date_index], "%Y-%m-%d") 108 | time = time.replace(tzinfo=tz.tzutc()) 109 | 110 | # Gather price. 111 | # Quantize with the same precision default rendering of floats occur. 112 | price_float = data[data_index] 113 | price = Decimal(price_float) 114 | match = re.search(r"(\..*)", str(price_float)) 115 | if match: 116 | price = price.quantize(Decimal(match.group(1))) 117 | 118 | # Note: There is no currency information in the response (surprising). 119 | return source.SourcePrice(price, time, None) 120 | 121 | 122 | class Source(source.Source): 123 | "Quandl API price extractor." 124 | 125 | def get_latest_price(self, ticker): 126 | """See contract in beanprice.source.Source.""" 127 | return fetch_time_series(ticker) 128 | 129 | def get_historical_price(self, ticker, time): 130 | """See contract in beanprice.source.Source.""" 131 | return fetch_time_series(ticker, time) 132 | -------------------------------------------------------------------------------- /beanprice/sources/quandl_test.py: -------------------------------------------------------------------------------- 1 | __copyright__ = "Copyright (C) 2018-2020 Martin Blais" 2 | __license__ = "GNU GPLv2" 3 | 4 | import datetime 5 | import unittest 6 | from unittest import mock 7 | from decimal import Decimal 8 | 9 | from dateutil import tz 10 | import requests 11 | 12 | from beanprice import date_utils 13 | from beanprice import source 14 | from beanprice.sources import quandl 15 | 16 | 17 | def response(contents, status_code=requests.codes.ok): 18 | """Produce a context manager to patch a JSON response.""" 19 | response = mock.Mock() 20 | response.status_code = status_code 21 | response.text = "" 22 | response.json.return_value = contents 23 | return mock.patch("requests.get", return_value=response) 24 | 25 | 26 | class QuandlPriceFetcher(unittest.TestCase): 27 | def test_parse_ticker(self): 28 | # NOTE(pmarciniak): "LBMA:GOLD:USD (PM)" is a valid ticker in Quandl 29 | # requests, but since space is not allowed in price source syntax, we're 30 | # representing space with an underscore. 31 | self.assertEqual( 32 | quandl.TickerSpec("WIKI", "FB", None), quandl.parse_ticker("WIKI:FB") 33 | ) 34 | self.assertEqual( 35 | quandl.TickerSpec("LBMA", "GOLD", "USD (PM)"), 36 | quandl.parse_ticker("LBMA:GOLD:USD_(PM)"), 37 | ) 38 | for test in [ 39 | "WIKI/FB", 40 | "FB", 41 | "WIKI.FB", 42 | "WIKI,FB", 43 | "LBMA:GOLD:USD (PM)", 44 | "LBMA:GOLD:col:umn", 45 | ]: 46 | with self.assertRaises(ValueError): 47 | quandl.parse_ticker(test) 48 | 49 | def test_error_premium(self): 50 | contents = { 51 | "quandl_error": { 52 | "code": "QEPx05", 53 | "message": ( 54 | "You have attempted to view a premium database in " 55 | "anonymous mode, i.e., without providing a Quandl " 56 | "key. Please register for a free Quandl account, " 57 | "and then include your API key with your " 58 | "requests." 59 | ), 60 | } 61 | } 62 | with response(contents): 63 | with self.assertRaises(ValueError) as exc: 64 | quandl.fetch_time_series("WIKI:FB", None) 65 | self.assertRegex(exc.message, "premium") 66 | 67 | def test_error_subscription(self): 68 | contents = { 69 | "quandl_error": { 70 | "code": "QEPx04", 71 | "message": ( 72 | "You do not have permission to view this dataset. " 73 | "Please subscribe to this database to get " 74 | "access." 75 | ), 76 | } 77 | } 78 | with response(contents): 79 | with self.assertRaises(ValueError) as exc: 80 | quandl.fetch_time_series("WIKI:FB", None) 81 | self.assertRegex(exc.message, "premium") 82 | 83 | def test_error_network(self): 84 | with response(None, 404): 85 | with self.assertRaises(ValueError) as exc: 86 | quandl.fetch_time_series("WIKI:FB", None) 87 | self.assertRegex(exc.message, "premium") 88 | 89 | def _test_valid_response(self): 90 | contents = { 91 | "dataset": { 92 | "collapse": None, 93 | "column_index": None, 94 | "column_names": [ 95 | "Date", 96 | "Open", 97 | "High", 98 | "Low", 99 | "Close", 100 | "Volume", 101 | "Ex-Dividend", 102 | "Split Ratio", 103 | "Adj. Open", 104 | "Adj. High", 105 | "Adj. Low", 106 | "Adj. Close", 107 | "Adj. Volume", 108 | ], 109 | "data": [ 110 | [ 111 | "2018-03-27", 112 | 1063.9, 113 | 1064.54, 114 | 997.62, 115 | 1006.94, 116 | 2940957.0, 117 | 0.0, 118 | 1.0, 119 | 1063.9, 120 | 1064.54, 121 | 997.62, 122 | 1006.94, 123 | 2940957.0, 124 | ] 125 | ], 126 | "database_code": "WIKI", 127 | "database_id": 4922, 128 | "dataset_code": "GOOGL", 129 | "description": "This dataset has no description.", 130 | "end_date": "2018-03-27", 131 | "frequency": "daily", 132 | "id": 11304017, 133 | "limit": 1, 134 | "name": ( 135 | "Alphabet Inc (GOOGL) Prices, Dividends, Splits and " "Trading Volume" 136 | ), 137 | "newest_available_date": "2018-03-27", 138 | "oldest_available_date": "2004-08-19", 139 | "order": None, 140 | "premium": False, 141 | "refreshed_at": "2018-03-27T21:46:11.201Z", 142 | "start_date": "2004-08-19", 143 | "transform": None, 144 | "type": "Time Series", 145 | } 146 | } 147 | with response(contents): 148 | srcprice = quandl.fetch_time_series("WIKI:FB", None) 149 | self.assertIsInstance(srcprice, source.SourcePrice) 150 | 151 | self.assertEqual(Decimal("1006.94"), srcprice.price) 152 | self.assertEqual( 153 | datetime.datetime(2018, 3, 27, 0, 0, 0, tzinfo=tz.tzutc()), 154 | srcprice.time.astimezone(tz.tzutc()), 155 | ) 156 | self.assertEqual(None, srcprice.quote_currency) 157 | 158 | def test_valid_response(self): 159 | for tzname in "America/New_York", "Europe/Berlin", "Asia/Tokyo": 160 | with date_utils.intimezone(tzname): 161 | self._test_valid_response() 162 | 163 | def test_non_standard_columns(self): 164 | contents = { 165 | "dataset": { 166 | "collapse": None, 167 | "column_index": None, 168 | "column_names": [ 169 | "Date", 170 | "USD (AM)", 171 | "USD (PM)", 172 | "GBP (AM)", 173 | "GBP (PM)", 174 | "EURO (AM)", 175 | "EURO (PM)", 176 | ], 177 | "data": [ 178 | ["2019-06-18", 1344.55, 1341.35, 1073.22, 1070.67, 1201.89, 1198.09] 179 | ], 180 | "end_date": "2019-06-18", 181 | "frequency": "daily", 182 | "order": None, 183 | "limit": 1, 184 | "start_date": "2019-06-08", 185 | "transform": None, 186 | } 187 | } 188 | with response(contents): 189 | srcprice = quandl.fetch_time_series("LBMA:GOLD:USD_(PM)", None) 190 | self.assertIsInstance(srcprice, source.SourcePrice) 191 | 192 | self.assertEqual(Decimal("1341.35"), srcprice.price) 193 | self.assertEqual( 194 | datetime.datetime(2019, 6, 18, 0, 0, 0, tzinfo=tz.tzutc()), 195 | srcprice.time.astimezone(tz.tzutc()), 196 | ) 197 | self.assertEqual(None, srcprice.quote_currency) 198 | 199 | 200 | if __name__ == "__main__": 201 | unittest.main() 202 | -------------------------------------------------------------------------------- /beanprice/sources/ratesapi.py: -------------------------------------------------------------------------------- 1 | """A source fetching exchangerates from https://exchangerate.host. 2 | 3 | Valid tickers are in the form "XXX-YYY", such as "EUR-CHF". 4 | 5 | Here is the API documentation: 6 | https://api.frankfurter.app/ 7 | 8 | For example: 9 | 10 | https://api.frankfurter.app/latest?base=EUR&symbols=CHF 11 | 12 | 13 | Timezone information: Input and output datetimes are specified via UTC 14 | timestamps. 15 | """ 16 | 17 | from decimal import Decimal 18 | 19 | import re 20 | import requests 21 | from dateutil.tz import tz 22 | from dateutil.parser import parse 23 | 24 | from beanprice import source 25 | 26 | 27 | class RatesApiError(ValueError): 28 | "An error from the Rates API." 29 | 30 | 31 | def _parse_ticker(ticker): 32 | """Parse the base and quote currencies from the ticker. 33 | 34 | Args: 35 | ticker: A string, the symbol in XXX-YYY format. 36 | Returns: 37 | A pair of (base, quote) currencies. 38 | """ 39 | match = re.match(r"^(?P\w+)-(?P\w+)$", ticker) 40 | if not match: 41 | raise ValueError('Invalid ticker. Use "BASE-SYMBOL" format.') 42 | return match.groups() 43 | 44 | 45 | def _get_quote(ticker, date): 46 | """Fetch a exchangerate from ratesapi.""" 47 | base, symbol = _parse_ticker(ticker) 48 | params = { 49 | "base": base, 50 | "symbol": symbol, 51 | } 52 | response = requests.get(url="https://api.frankfurter.app/" + date, params=params) 53 | 54 | if response.status_code != requests.codes.ok: 55 | raise RatesApiError( 56 | "Invalid response ({}): {}".format(response.status_code, response.text) 57 | ) 58 | 59 | result = response.json() 60 | 61 | price = Decimal(str(result["rates"][symbol])) 62 | time = parse(result["date"]).replace(tzinfo=tz.tzutc()) 63 | 64 | return source.SourcePrice(price, time, symbol) 65 | 66 | 67 | class Source(source.Source): 68 | def get_latest_price(self, ticker): 69 | return _get_quote(ticker, "latest") 70 | 71 | def get_historical_price(self, ticker, time): 72 | return _get_quote(ticker, time.date().isoformat()) 73 | -------------------------------------------------------------------------------- /beanprice/sources/ratesapi_test.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import unittest 3 | from decimal import Decimal 4 | 5 | from unittest import mock 6 | from dateutil import tz 7 | 8 | import requests 9 | 10 | from beanprice import source 11 | from beanprice.sources import ratesapi 12 | 13 | 14 | def response(contents, status_code=requests.codes.ok): 15 | """Return a context manager to patch a JSON response.""" 16 | response = mock.Mock() 17 | response.status_code = status_code 18 | response.text = "" 19 | response.json.return_value = contents 20 | return mock.patch("requests.get", return_value=response) 21 | 22 | 23 | class RatesapiPriceFetcher(unittest.TestCase): 24 | def test_error_invalid_ticker(self): 25 | with self.assertRaises(ValueError): 26 | ratesapi.Source().get_latest_price("INVALID") 27 | 28 | def test_error_network(self): 29 | with response("Foobar", 404): 30 | with self.assertRaises(ValueError): 31 | ratesapi.Source().get_latest_price("EUR-CHF") 32 | 33 | def test_valid_response(self): 34 | contents = { 35 | "base": "EUR", 36 | "rates": {"CHF": "1.2001"}, 37 | "date": "2019-04-20", 38 | } 39 | with response(contents): 40 | srcprice = ratesapi.Source().get_latest_price("EUR-CHF") 41 | self.assertIsInstance(srcprice, source.SourcePrice) 42 | self.assertEqual(Decimal("1.2001"), srcprice.price) 43 | self.assertEqual("CHF", srcprice.quote_currency) 44 | 45 | def test_historical_price(self): 46 | time = datetime.datetime(2018, 3, 27, 0, 0, 0, tzinfo=tz.tzutc()) 47 | contents = { 48 | "base": "EUR", 49 | "rates": {"CHF": "1.2001"}, 50 | "date": "2018-03-27", 51 | } 52 | with response(contents): 53 | srcprice = ratesapi.Source().get_historical_price("EUR-CHF", time) 54 | self.assertIsInstance(srcprice, source.SourcePrice) 55 | self.assertEqual(Decimal("1.2001"), srcprice.price) 56 | self.assertEqual("CHF", srcprice.quote_currency) 57 | self.assertEqual( 58 | datetime.datetime(2018, 3, 27, 0, 0, 0, tzinfo=tz.tzutc()), srcprice.time 59 | ) 60 | 61 | 62 | if __name__ == "__main__": 63 | unittest.main() 64 | -------------------------------------------------------------------------------- /beanprice/sources/tsp.py: -------------------------------------------------------------------------------- 1 | """Fetch prices from US Government Thrift Savings Plan 2 | 3 | As of 7 July 2020, the Thrift Savings Plan (TSP) rolled out a new 4 | web site that has an API (instead of scraping a CSV). Unable to 5 | find docs on the API. A web directory listing with various tools 6 | is available at: 7 | 8 | https://secure.tsp.gov/components/CORS/ 9 | """ 10 | 11 | __copyright__ = "Copyright (C) 2020 Martin Blais" 12 | __license__ = "GNU GPLv2" 13 | 14 | import csv 15 | from collections import OrderedDict 16 | import datetime 17 | from decimal import Decimal 18 | 19 | import requests 20 | 21 | from beanprice import source 22 | 23 | # All of the TSP funds are in USD. 24 | CURRENCY = "USD" 25 | 26 | TIMEZONE = datetime.timezone(datetime.timedelta(hours=-4), "America/New_York") 27 | 28 | TSP_FUND_NAMES = [ 29 | "LInco", # 0 30 | "L2025", # 1 31 | "L2030", # 2 32 | "L2035", # 3 33 | "L2040", # 4 34 | "L2045", # 5 35 | "L2050", # 6 36 | "L2055", # 7 37 | "L2060", # 8 38 | "L2065", # 9 39 | "GFund", # 10 40 | "FFund", # 11 41 | "CFund", # 12 42 | "SFund", # 13 43 | "IFund", # 14 44 | ] 45 | 46 | csv.register_dialect( 47 | "tsp", 48 | delimiter=",", 49 | quoting=csv.QUOTE_NONE, 50 | # NOTE(blais): This fails to import in 3.12 (and perhaps before). 51 | # quotechar='', 52 | lineterminator="\n", 53 | ) 54 | 55 | 56 | class TSPError(ValueError): 57 | "An error from the Thrift Savings Plan (TSP) API." 58 | 59 | 60 | def parse_tsp_csv(response: requests.models.Response) -> OrderedDict: 61 | """Parses a Thrift Savings Plan output CSV file. 62 | 63 | Function takes in a requests response and returns an 64 | OrderedDict with newest closing cost at front of OrderedDict. 65 | """ 66 | 67 | data = OrderedDict() 68 | 69 | text = response.iter_lines(decode_unicode=True) 70 | 71 | reader = csv.DictReader(text, dialect="tsp") 72 | 73 | for row in reader: 74 | # Date from TSP looks like "July 30. 2020" 75 | # There is indeed a period after the day of month. 76 | date = datetime.datetime.strptime(row["Date"], "%b %d. %Y") 77 | date = date.replace(hour=16, tzinfo=TIMEZONE) 78 | names = [ 79 | "L Income", 80 | "L 2025", 81 | "L 2030", 82 | "L 2035", 83 | "L 2040", 84 | "L 2045", 85 | "L 2050", 86 | "L 2055", 87 | "L 2060", 88 | "L 2065", 89 | "G Fund", 90 | "F Fund", 91 | "C Fund", 92 | "S Fund", 93 | "I Fund", 94 | ] 95 | data[date] = [ 96 | Decimal(row[name]) if row[name] else Decimal() for name in map(str.strip, names) 97 | ] 98 | 99 | return OrderedDict(sorted(data.items(), key=lambda t: t[0], reverse=True)) 100 | 101 | 102 | def parse_response(response: requests.models.Response) -> OrderedDict: 103 | """Process as response from TSP. 104 | 105 | Raises: 106 | TSPError: If there is an error in the response. 107 | """ 108 | if response.status_code != requests.codes.ok: 109 | raise TSPError("Error from TSP Parsing Status {}".format(response.status_code)) 110 | 111 | return parse_tsp_csv(response) 112 | 113 | 114 | class Source(source.Source): 115 | "US Thrift Savings Plan API Price Extractor" 116 | 117 | def get_latest_price(self, fund): 118 | """See contract in beanprice.source.Source.""" 119 | return self.get_historical_price(fund, datetime.datetime.now()) 120 | 121 | def get_historical_price(self, fund, time): 122 | """See contract in beanprice.source.Source.""" 123 | if requests is None: 124 | raise TSPError("You must install the 'requests' library.") 125 | 126 | if fund not in TSP_FUND_NAMES: 127 | raise TSPError( 128 | "Invalid TSP Fund Name '{}'. Valid Funds are:\n\t{}".format( 129 | fund, "\n\t".join(TSP_FUND_NAMES) 130 | ) 131 | ) 132 | 133 | url = "https://secure.tsp.gov/components/CORS/getSharePricesRaw.html" 134 | payload = { 135 | # Grabbing the last fourteen days of data in event the markets were closed. 136 | "startdate": (time - datetime.timedelta(days=14)).strftime("%Y%m%d"), 137 | "enddate": time.strftime("%Y%m%d"), 138 | "download": "0", 139 | "Lfunds": "1", 140 | "InvFunds": "1", 141 | } 142 | 143 | response = requests.get(url, params=payload) 144 | result = parse_response(response) 145 | trade_day = next(iter(result.items())) 146 | prices = trade_day[1] 147 | 148 | try: 149 | price = prices[TSP_FUND_NAMES.index(fund)] 150 | 151 | trade_time = trade_day[0] 152 | except KeyError as exc: 153 | raise TSPError("Invalid response from TSP: {}".format(repr(result))) from exc 154 | 155 | return source.SourcePrice(price, trade_time, CURRENCY) 156 | -------------------------------------------------------------------------------- /beanprice/sources/tsp_test.py: -------------------------------------------------------------------------------- 1 | __copyright__ = "Copyright (C) 2020 Martin Blais" 2 | __license__ = "GNU GPLv2" 3 | 4 | import datetime 5 | import textwrap 6 | import unittest 7 | from decimal import Decimal 8 | from unittest import mock 9 | 10 | import requests 11 | 12 | from beanprice.sources import tsp 13 | 14 | 15 | CURRENT_DATA = ( 16 | "Date,L Income,L 2025,L 2030,L 2035,L 2040,L 2045," 17 | "L 2050,L 2055,L 2060,L 2065,G Fund,F Fund,C Fund,S Fund,I Fund\n" 18 | "Jul 2. 2020, 21.1910, 10.0404, 34.1084, 10.0531, 37.3328, 10.0617," 19 | " 21.6880, 10.0780, 10.0780, 10.0781, 16.4477, 20.9449, 46.2229, 53.2754, 29.5401\n" 20 | "Jul 6. 2020, 21.2689, 10.1224, 34.4434, 10.1610, 37.7690, 10.1871," 21 | " 21.9759, 10.2392, 10.2392, 10.2393, 16.4490, 20.9555, 46.9567, 54.0427, 30.0523\n" 22 | "Jul 7. 2020, 21.2148, 10.0628, 34.1982, 10.0819, 37.4478, 10.0946," 23 | " 21.7627, 10.1173, 10.1174, 10.1175, 16.4493, 20.9946, 46.4489, 53.3353, 29.6545\n" 24 | "Jul 8. 2020, 21.2424, 10.0924, 34.3196, 10.1213, 37.6071, 10.1407," 25 | " 21.8687, 10.1771, 10.1772, 10.1773, 16.4496, 20.9960, 46.8131, 53.9192, 29.6863\n" 26 | "Jul 9. 2020, 21.2175, 10.0632, 34.1992, 10.0822, 37.4481, 10.0946," 27 | " 21.7622, 10.1144, 10.1145, 10.1146, 16.4499, 21.0549, 46.5615, 53.3815, 29.5161\n" 28 | "Jul 10. 2020, 21.2562, 10.1058, 34.3736, 10.1387, 37.6769, 10.1607," 29 | " 21.9144, 10.2014, 10.2014, 10.2015, 16.4502, 21.0288, 47.0497, 54.1006, 29.6343\n" 30 | "Jul 13. 2020, 21.2263, 10.0723, 34.2353, 10.0937, 37.4937, 10.1074," 31 | " 21.7911, 10.1317, 10.1317, 10.1318, 16.4512, 21.0316, 46.6089, 53.0366, 29.7075\n" 32 | "Jul 14. 2020, 21.2898, 10.1398, 34.5110, 10.1829, 37.8542, 10.2115," 33 | " 22.0301, 10.2651, 10.2651, 10.2652, 16.4515, 21.0608, 47.2391, 53.8560, 30.0643\n" 34 | "Jul 15. 2020, 21.3513, 10.2067, 34.7862, 10.2723, 38.2174, 10.3170," 35 | " 22.2736, 10.4025, 10.4026, 10.4027, 16.4519, 21.0574, 47.6702, 55.2910, 30.4751" 36 | ) 37 | 38 | 39 | HISTORIC_DATA = ( 40 | "Date,L Income,L 2025,L 2030,L 2035,L 2040,L 2045," 41 | "L 2050,L 2055,L 2060,L 2065,G Fund,F Fund,C Fund,S Fund,I Fund\n" 42 | "Jun 5. 2020, 21.2498,, 34.4615,, 37.8091,, 22.0123,,,," 43 | "16.4390, 20.6864, 47.1062, 54.6393, 30.1182\n" 44 | "Jun 8. 2020, 21.3126,, 34.7342,, 38.1665,, 22.2501,,,," 45 | "16.4400, 20.7235, 47.6757, 55.8492, 30.4200\n" 46 | "Jun 9. 2020, 21.2740,, 34.5504,, 37.9228,, 22.0861,,,," 47 | "16.4403, 20.7657, 47.3065, 54.7837, 30.2166\n" 48 | "Jun 10. 2020, 21.2531,, 34.4425,, 37.7784,, 21.9877,,,," 49 | "16.4407, 20.8290, 47.0540, 53.8901, 30.1676\n" 50 | "Jun 11. 2020, 20.9895,, 33.2506,, 36.2150,, 20.9486,,,," 51 | "16.4410, 20.8675, 44.2870, 50.2699, 28.5702\n" 52 | "Jun 12. 2020, 21.0436,, 33.4985,, 36.5408,, 21.1659,,,," 53 | "16.4413, 20.8332, 44.8769, 51.2514, 28.8452\n" 54 | "Jun 15. 2020, 21.0834,, 33.6719,, 36.7683,, 21.3177,,,," 55 | "16.4423, 20.8369, 45.2527, 52.3220, 28.9501\n" 56 | "Jun 16. 2020, 21.1666,, 34.0373,, 37.2440,, 21.6317,,,," 57 | "16.4426, 20.8358, 46.1115, 53.3428, 29.4106\n" 58 | "Jun 17. 2020, 21.1605,, 34.0068,, 37.2029,, 21.6035,,,," 59 | "16.4429, 20.8416, 45.9447, 52.8271, 29.5535\n" 60 | "Jun 18. 2020, 21.1562,, 33.9827,, 37.1713,, 21.5824,,,," 61 | "16.4432, 20.8718, 45.9718, 52.9328, 29.3908\n" 62 | "Jun 19. 2020, 21.1354,, 33.8890,, 37.0491,, 21.5018,,,," 63 | "16.4435, 20.8742, 45.7171, 52.7196, 29.2879" 64 | ) 65 | 66 | 67 | class MockResponse: 68 | """A mock requests.Response object for testing.""" 69 | 70 | def __init__(self, contents, status_code=requests.codes.ok): 71 | self.status_code = status_code 72 | self._content = contents 73 | 74 | def iter_lines(self, decode_unicode=False): 75 | return iter(self._content.splitlines()) 76 | 77 | 78 | class TSPFinancePriceFetcher(unittest.TestCase): 79 | def test_get_latest_price_L2050(self): 80 | response = MockResponse(textwrap.dedent(CURRENT_DATA)) 81 | with mock.patch("requests.get", return_value=response): 82 | srcprice = tsp.Source().get_latest_price("L2050") 83 | self.assertTrue(isinstance(srcprice.price, Decimal)) 84 | self.assertEqual(Decimal("22.2736"), srcprice.price) 85 | timezone = datetime.timezone(datetime.timedelta(hours=-4), "America/New_York") 86 | self.assertEqual( 87 | datetime.datetime(2020, 7, 15, 16, 0, 0, tzinfo=timezone), srcprice.time 88 | ) 89 | self.assertEqual("USD", srcprice.quote_currency) 90 | 91 | def test_get_latest_price_SFund(self): 92 | response = MockResponse(textwrap.dedent(CURRENT_DATA)) 93 | with mock.patch("requests.get", return_value=response): 94 | srcprice = tsp.Source().get_latest_price("SFund") 95 | self.assertTrue(isinstance(srcprice.price, Decimal)) 96 | self.assertEqual(Decimal("55.2910"), srcprice.price) 97 | timezone = datetime.timezone(datetime.timedelta(hours=-4), "America/New_York") 98 | self.assertEqual( 99 | datetime.datetime(2020, 7, 15, 16, 0, 0, tzinfo=timezone), srcprice.time 100 | ) 101 | self.assertEqual("USD", srcprice.quote_currency) 102 | 103 | def test_get_historical_price(self): 104 | response = MockResponse(textwrap.dedent(HISTORIC_DATA)) 105 | with mock.patch("requests.get", return_value=response): 106 | srcprice = tsp.Source().get_historical_price( 107 | "CFund", time=datetime.datetime(2020, 6, 19) 108 | ) 109 | self.assertTrue(isinstance(srcprice.price, Decimal)) 110 | self.assertEqual(Decimal("45.7171"), srcprice.price) 111 | timezone = datetime.timezone(datetime.timedelta(hours=-4), "America/New_York") 112 | self.assertEqual( 113 | datetime.datetime(2020, 6, 19, 16, 0, 0, tzinfo=timezone), srcprice.time 114 | ) 115 | self.assertEqual("USD", srcprice.quote_currency) 116 | 117 | def test_get_historical_price_L2060(self): 118 | # This fund did not exist until 01 Jul 2020. Ensuring we get a Decimal(0.0) back. 119 | response = MockResponse(textwrap.dedent(HISTORIC_DATA)) 120 | with mock.patch("requests.get", return_value=response): 121 | srcprice = tsp.Source().get_historical_price( 122 | "L2060", time=datetime.datetime(2020, 6, 19) 123 | ) 124 | self.assertTrue(isinstance(srcprice.price, Decimal)) 125 | self.assertEqual(Decimal("0.0"), srcprice.price) 126 | timezone = datetime.timezone(datetime.timedelta(hours=-4), "America/New_York") 127 | self.assertEqual( 128 | datetime.datetime(2020, 6, 19, 16, 0, 0, tzinfo=timezone), srcprice.time 129 | ) 130 | self.assertEqual("USD", srcprice.quote_currency) 131 | 132 | def test_invalid_fund_latest(self): 133 | with self.assertRaises(tsp.TSPError): 134 | tsp.Source().get_latest_price("InvalidFund") 135 | 136 | def test_invalid_fund_historical(self): 137 | with self.assertRaises(tsp.TSPError): 138 | tsp.Source().get_historical_price("InvalidFund", time=datetime.datetime.now()) 139 | 140 | 141 | if __name__ == "__main__": 142 | unittest.main() 143 | -------------------------------------------------------------------------------- /beanprice/sources/yahoo.py: -------------------------------------------------------------------------------- 1 | """Fetch prices from Yahoo Finance's CSV API. 2 | 3 | As of late 2017, the older Yahoo finance API deprecated. In particular, the 4 | ichart endpoint is gone, and the download endpoint requires a cookie (which 5 | could be gotten - here's some documentation for that 6 | http://blog.bradlucas.com/posts/2017-06-02-new-yahoo-finance-quote-download-url/). 7 | 8 | We're using both the v7 and v8 APIs here, both of which are, as far as I can 9 | tell, undocumented: 10 | 11 | https://query1.finance.yahoo.com/v7/finance/quote 12 | https://query1.finance.yahoo.com/v8/finance/chart/SYMBOL 13 | 14 | Timezone information: Input and output datetimes are specified via UNIX 15 | timestamps, but the timezone of the particular market is included in the output. 16 | """ 17 | 18 | __copyright__ = "Copyright (C) 2015-2020 Martin Blais" 19 | __license__ = "GNU GPLv2" 20 | 21 | from datetime import datetime, timedelta, timezone 22 | from decimal import Decimal 23 | from typing import Any, Dict, List, Optional, Tuple, Union 24 | 25 | from curl_cffi import requests 26 | 27 | from beanprice import source 28 | 29 | 30 | class YahooError(ValueError): 31 | "An error from the Yahoo API." 32 | 33 | 34 | def parse_response(response: requests.models.Response) -> Dict: 35 | """Process as response from Yahoo. 36 | 37 | Raises: 38 | YahooError: If there is an error in the response. 39 | """ 40 | json = response.json(parse_float=Decimal) 41 | content = next(iter(json.values())) 42 | if response.status_code != 200: 43 | raise YahooError("Status {}: {}".format(response.status_code, content["error"])) 44 | if len(json) != 1: 45 | raise YahooError( 46 | "Invalid format in response from Yahoo; many keys: {}".format( 47 | ",".join(json.keys()) 48 | ) 49 | ) 50 | if content["error"] is not None: 51 | raise YahooError("Error fetching Yahoo data: {}".format(content["error"])) 52 | if not content["result"]: 53 | raise YahooError("No data returned from Yahoo, ensure that the symbol is correct") 54 | return content["result"][0] 55 | 56 | 57 | # Note: Feel free to suggest more here via a PR. 58 | _MARKETS = { 59 | "us_market": "USD", 60 | "ca_market": "CAD", 61 | "ch_market": "CHF", 62 | } 63 | 64 | 65 | def parse_currency(result: Dict[str, Any]) -> Optional[str]: 66 | """Infer the currency from the result.""" 67 | if "market" not in result: 68 | return None 69 | return _MARKETS.get(result["market"], None) 70 | 71 | 72 | _DEFAULT_PARAMS = { 73 | "lang": "en-US", 74 | "corsDomain": "finance.yahoo.com", 75 | ".tsrc": "finance", 76 | } 77 | 78 | 79 | def get_price_series( 80 | ticker: str, 81 | time_begin: datetime, 82 | time_end: datetime, 83 | session: requests.Session, 84 | ) -> Tuple[List[Tuple[datetime, Decimal]], str]: 85 | """Return a series of timestamped prices.""" 86 | 87 | if requests is None: 88 | raise YahooError("You must install the 'requests' library.") 89 | url = "https://query1.finance.yahoo.com/v8/finance/chart/{}".format(ticker) 90 | payload: Dict[str, Union[int, str]] = { 91 | "period1": int(time_begin.timestamp()), 92 | "period2": int(time_end.timestamp()), 93 | "interval": "1d", 94 | } 95 | payload.update(_DEFAULT_PARAMS) 96 | response = session.get(url, params=payload) # Use shared session 97 | result = parse_response(response) 98 | 99 | meta = result["meta"] 100 | tzone = timezone( 101 | timedelta(hours=meta["gmtoffset"] / 3600), meta["exchangeTimezoneName"] 102 | ) 103 | 104 | if "timestamp" not in result: 105 | raise YahooError( 106 | "Yahoo returned no data for ticker {} for time range {} - {}".format( 107 | ticker, time_begin, time_end 108 | ) 109 | ) 110 | 111 | timestamp_array = result["timestamp"] 112 | close_array = result["indicators"]["quote"][0]["close"] 113 | series = [ 114 | (datetime.fromtimestamp(timestamp, tz=tzone), Decimal(price)) 115 | for timestamp, price in zip(timestamp_array, close_array) 116 | if price is not None 117 | ] 118 | 119 | currency = result["meta"]["currency"] 120 | return series, currency 121 | 122 | 123 | class Source(source.Source): 124 | "Yahoo Finance CSV API price extractor." 125 | 126 | def __init__(self): 127 | """Initialize a shared session with the required headers and cookies.""" 128 | # Using curl_cffi's requests to impersonate a Chrome browser 129 | # to avoid being blocked by Yahoo's bot detection. 130 | # See issue https://github.com/beancount/beanprice/issues/106 for more details. 131 | self.session = requests.Session(impersonate="chrome") 132 | self.session.headers.update( 133 | { 134 | "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) " 135 | "AppleWebKit/537.36 (KHTML, like Gecko) " 136 | "Chrome/121.0.0.0 Safari/537.36", 137 | "Sec-Ch-Ua": '"Google Chrome";v="121", ' 138 | '"Chromium";v="121", ";Not A Brand";v="99"', 139 | "Sec-Ch-Ua-Platform": '"Linux"', 140 | } 141 | ) 142 | # This populates the correct cookies in the session 143 | self.session.get("https://fc.yahoo.com") 144 | self.crumb = self.session.get( 145 | "https://query1.finance.yahoo.com/v1/test/getcrumb" 146 | ).text 147 | 148 | def get_latest_price(self, ticker: str) -> Optional[source.SourcePrice]: 149 | """See contract in beanprice.source.Source.""" 150 | 151 | url = "https://query1.finance.yahoo.com/v7/finance/quote" 152 | fields = ["symbol", "regularMarketPrice", "regularMarketTime"] 153 | payload = { 154 | "symbols": ticker, 155 | "fields": ",".join(fields), 156 | "exchange": "NYSE", 157 | "crumb": self.crumb, # Use the session’s crumb 158 | } 159 | payload.update(_DEFAULT_PARAMS) 160 | response = self.session.get(url, params=payload) # Use shared session 161 | 162 | try: 163 | result = parse_response(response) 164 | except YahooError as error: 165 | # The parse_response method cannot know which ticker failed, 166 | # but the user definitely needs to know which ticker failed! 167 | raise YahooError("%s (ticker: %s)" % (error, ticker)) from error 168 | try: 169 | price = Decimal(result["regularMarketPrice"]) 170 | 171 | tzone = timezone( 172 | timedelta(hours=result["gmtOffSetMilliseconds"] / 3600000), 173 | result["exchangeTimezoneName"], 174 | ) 175 | trade_time = datetime.fromtimestamp(result["regularMarketTime"], tz=tzone) 176 | except KeyError as exc: 177 | raise YahooError( 178 | "Invalid response from Yahoo: {}".format(repr(result)) 179 | ) from exc 180 | 181 | currency = parse_currency(result) 182 | 183 | return source.SourcePrice(price, trade_time, currency) 184 | 185 | def get_historical_price( 186 | self, ticker: str, time: datetime 187 | ) -> Optional[source.SourcePrice]: 188 | """See contract in beanprice.source.Source.""" 189 | 190 | # Get the latest data returned over the last 5 days. 191 | series, currency = get_price_series( 192 | ticker, time - timedelta(days=5), time, self.session 193 | ) 194 | latest = None 195 | for data_dt, price in sorted(series): 196 | if data_dt >= time: 197 | break 198 | latest = data_dt, price 199 | if latest is None: 200 | raise YahooError("Could not find price before {} in {}".format(time, series)) 201 | 202 | return source.SourcePrice(price, data_dt, currency) 203 | 204 | def get_daily_prices( 205 | self, ticker: str, time_begin: datetime, time_end: datetime 206 | ) -> Optional[List[source.SourcePrice]]: 207 | """See contract in beanprice.source.Source.""" 208 | series, currency = get_price_series(ticker, time_begin, time_end, self.session) 209 | return [source.SourcePrice(price, time, currency) for time, price in series] 210 | -------------------------------------------------------------------------------- /beanprice/sources/yahoo_test.py: -------------------------------------------------------------------------------- 1 | __copyright__ = "Copyright (C) 2015-2020 Martin Blais" 2 | __license__ = "GNU GPLv2" 3 | 4 | import datetime 5 | import json 6 | import textwrap 7 | import unittest 8 | from decimal import Decimal 9 | from unittest import mock 10 | 11 | from dateutil import tz 12 | import requests 13 | 14 | from beanprice import date_utils 15 | from beanprice.sources import yahoo 16 | 17 | 18 | class MockResponse: 19 | """A mock requests.Models.Response object for testing.""" 20 | 21 | def __init__(self, contents, status_code=requests.codes.ok): 22 | self.status_code = status_code 23 | self.contents = contents 24 | 25 | def json(self, **kwargs): 26 | return json.loads(self.contents, **kwargs) 27 | 28 | 29 | class YahooFinancePriceFetcher(unittest.TestCase): 30 | def _test_get_latest_price(self): 31 | response = MockResponse( 32 | textwrap.dedent(""" 33 | {"quoteResponse": 34 | {"error": null, 35 | "result": [{"esgPopulated": false, 36 | "exchange": "TOR", 37 | "exchangeDataDelayedBy": 15, 38 | "exchangeTimezoneName": "America/Toronto", 39 | "exchangeTimezoneShortName": "EDT", 40 | "fullExchangeName": "Toronto", 41 | "gmtOffSetMilliseconds": -14400000, 42 | "language": "en-US", 43 | "market": "ca_market", 44 | "marketState": "CLOSED", 45 | "quoteType": "ETF", 46 | "regularMarketPrice": 29.99, 47 | "regularMarketTime": 1522353589, 48 | "sourceInterval": 15, 49 | "symbol": "XSP.TO", 50 | "tradeable": false}]}} 51 | """) 52 | ) 53 | yahoo_source = yahoo.Source() 54 | with mock.patch.object(yahoo_source.session, "get", return_value=response): 55 | srcprice = yahoo_source.get_latest_price("XSP.TO") 56 | self.assertTrue(isinstance(srcprice.price, Decimal)) 57 | self.assertEqual(Decimal("29.99"), srcprice.price) 58 | timezone = datetime.timezone(datetime.timedelta(hours=-4), "America/Toronto") 59 | self.assertEqual( 60 | datetime.datetime(2018, 3, 29, 15, 59, 49, tzinfo=timezone), srcprice.time 61 | ) 62 | self.assertEqual("CAD", srcprice.quote_currency) 63 | 64 | def test_get_latest_price(self): 65 | for tzname in "America/New_York", "Europe/Berlin", "Asia/Tokyo": 66 | with date_utils.intimezone(tzname): 67 | self._test_get_latest_price() 68 | 69 | def _test_get_historical_price(self): 70 | response = MockResponse( 71 | textwrap.dedent(""" 72 | {"chart": 73 | {"error": null, 74 | "result": [{"indicators": {"adjclose": [{"adjclose": [29.236251831054688, 75 | 29.16683006286621, 76 | 29.196582794189453, 77 | 29.226333618164062]}], 78 | "quote": [{"close": [29.479999542236328, 79 | 29.40999984741211, 80 | 29.440000534057617, 81 | 29.469999313354492], 82 | "high": [29.510000228881836, 83 | 29.489999771118164, 84 | 29.469999313354492, 85 | 29.579999923706055], 86 | "low": [29.34000015258789, 87 | 29.350000381469727, 88 | 29.399999618530273, 89 | 29.43000030517578], 90 | "open": [29.360000610351562, 91 | 29.43000030517578, 92 | 29.43000030517578, 93 | 29.530000686645508], 94 | "volume": [160800, 95 | 118700, 96 | 98500, 97 | 227800]}]}, 98 | "meta": {"chartPreviousClose": 29.25, 99 | "currency": "CAD", 100 | "currentTradingPeriod": {"post": {"end": 1522702800, 101 | "gmtoffset": -14400, 102 | "start": 1522699200, 103 | "timezone": "EDT"}, 104 | "pre": {"end": 1522675800, 105 | "gmtoffset": -14400, 106 | "start": 1522670400, 107 | "timezone": "EDT"}, 108 | "regular": {"end": 1522699200, 109 | "gmtoffset": -14400, 110 | "start": 1522675800, 111 | "timezone": "EDT"}}, 112 | "dataGranularity": "1d", 113 | "exchangeName": "TOR", 114 | "exchangeTimezoneName": "America/Toronto", 115 | "firstTradeDate": 1018872000, 116 | "gmtoffset": -14400, 117 | "instrumentType": "ETF", 118 | "symbol": "XSP.TO", 119 | "timezone": "EDT", 120 | "validRanges": ["1d", "5d", "1mo", "3mo", "6mo", "1y", 121 | "2y", "5y", "10y", "ytd", "max"]}, 122 | "timestamp": [1509111000, 123 | 1509370200, 124 | 1509456600, 125 | 1509543000]}]}}""") 126 | ) 127 | yahoo_source = yahoo.Source() 128 | with mock.patch.object(yahoo_source.session, "get", return_value=response): 129 | srcprice = yahoo_source.get_historical_price( 130 | "XSP.TO", datetime.datetime(2017, 11, 1, 16, 0, 0, tzinfo=tz.tzutc()) 131 | ) 132 | self.assertTrue(isinstance(srcprice.price, Decimal)) 133 | self.assertEqual(Decimal("29.469999313354492"), srcprice.price) 134 | timezone = datetime.timezone(datetime.timedelta(hours=-4), "America/Toronto") 135 | self.assertEqual( 136 | datetime.datetime(2017, 11, 1, 9, 30, tzinfo=timezone), srcprice.time 137 | ) 138 | self.assertEqual("CAD", srcprice.quote_currency) 139 | 140 | def test_get_historical_price(self): 141 | for tzname in "America/New_York", "Europe/Berlin", "Asia/Tokyo": 142 | with date_utils.intimezone(tzname): 143 | self._test_get_historical_price() 144 | 145 | def test_parse_response_error_status_code(self): 146 | response = MockResponse( 147 | '{"quoteResponse": {"error": "Not supported", "result": [{}]}}', status_code=400 148 | ) 149 | with self.assertRaises(yahoo.YahooError): 150 | yahoo.parse_response(response) 151 | 152 | def test_parse_response_error_invalid_format(self): 153 | response = MockResponse( 154 | """{"quoteResponse": {"error": null, "result": [{}]}, 155 | "chart": {"error": null, "result": [{}]}}""" 156 | ) 157 | with self.assertRaises(yahoo.YahooError): 158 | yahoo.parse_response(response) 159 | 160 | def test_parse_response_error_not_none(self): 161 | response = MockResponse( 162 | '{"quoteResponse": {"error": "Non-zero error", "result": [{}]}}' 163 | ) 164 | with self.assertRaises(yahoo.YahooError): 165 | yahoo.parse_response(response) 166 | 167 | def test_parse_response_empty_result(self): 168 | response = MockResponse('{"quoteResponse": {"error": null, "result": []}}') 169 | with self.assertRaises(yahoo.YahooError): 170 | yahoo.parse_response(response) 171 | 172 | def test_parse_response_no_timestamp(self): 173 | response = MockResponse( 174 | textwrap.dedent(""" 175 | {"chart": 176 | {"error": null, 177 | "result": [{"indicators": {"adjclose": [{}], 178 | "quote": [{}]}, 179 | "meta": {"chartPreviousClose": 29.25, 180 | "currency": "CAD", 181 | "currentTradingPeriod": {"post": {"end": 1522702800, 182 | "gmtoffset": -14400, 183 | "start": 1522699200, 184 | "timezone": "EDT"}, 185 | "pre": {"end": 1522675800, 186 | "gmtoffset": -14400, 187 | "start": 1522670400, 188 | "timezone": "EDT"}, 189 | "regular": {"end": 1522699200, 190 | "gmtoffset": -14400, 191 | "start": 1522675800, 192 | "timezone": "EDT"}}, 193 | "dataGranularity": "1d", 194 | "exchangeName": "TOR", 195 | "exchangeTimezoneName": "America/Toronto", 196 | "firstTradeDate": 1018872000, 197 | "gmtoffset": -14400, 198 | "instrumentType": "ETF", 199 | "symbol": "XSP.TO", 200 | "timezone": "EDT", 201 | "validRanges": ["1d", "5d", "1mo", "3mo", "6mo", "1y", 202 | "2y", "5y", "10y", "ytd", "max"]}}]}} 203 | """) 204 | ) 205 | with self.assertRaises(yahoo.YahooError): 206 | yahoo_source = yahoo.Source() 207 | with mock.patch.object(yahoo_source.session, "get", return_value=response): 208 | _ = yahoo_source.get_historical_price( 209 | "XSP.TO", datetime.datetime(2017, 11, 1, 16, 0, 0, tzinfo=tz.tzutc()) 210 | ) 211 | 212 | def test_parse_null_prices_in_series(self): 213 | response = MockResponse( 214 | textwrap.dedent(""" 215 | {"chart": {"result":[ {"meta":{ 216 | "currency":"USD","symbol":"FBIIX", 217 | "exchangeName":"NAS","instrumentType":"MUTUALFUND", 218 | "firstTradeDate":1570714200,"regularMarketTime":1646053572, 219 | "gmtoffset":-18000,"timezone":"EST", 220 | "exchangeTimezoneName":"America/New_York", 221 | "regularMarketPrice":9.69,"chartPreviousClose":9.69, 222 | "priceHint":2, 223 | "currentTradingPeriod":{ 224 | "pre":{"timezone":"EST","start":1646038800,"end":1646058600,"gmtoffset":-18000}, 225 | "regular":{"timezone":"EST","start":1646058600,"end":1646082000,"gmtoffset":-18000}, 226 | "post":{"timezone":"EST","start":1646082000,"end":1646096400,"gmtoffset":-18000} 227 | }, 228 | "dataGranularity":"1d","range":"", 229 | "validRanges":["1mo","3mo","6mo","ytd","1y","2y","5y","10y","max"]}, 230 | "timestamp":[1645626600,1645713000,1645799400,1646058600], 231 | "indicators":{ 232 | "quote":[ 233 | {"open":[9.6899995803833,9.710000038146973,9.6899995803833,null], 234 | "low":[9.6899995803833,9.710000038146973,9.6899995803833,null], 235 | "high":[9.6899995803833,9.710000038146973,9.6899995803833,null], 236 | "volume":[0,0,0,null], 237 | "close":[9.6899995803833,9.710000038146973,9.6899995803833,null]} 238 | ],"adjclose":[ 239 | {"adjclose":[9.6899995803833,9.710000038146973,9.6899995803833,null]} 240 | ] 241 | }}],"error":null}} 242 | """) 243 | ) 244 | 245 | yahoo_source = yahoo.Source() 246 | with mock.patch.object(yahoo_source.session, "get", return_value=response): 247 | srcprice = yahoo_source.get_historical_price( 248 | "XSP.TO", datetime.datetime(2022, 2, 28, 16, 0, 0, tzinfo=tz.tzutc()) 249 | ) 250 | self.assertTrue(isinstance(srcprice.price, Decimal)) 251 | self.assertEqual(Decimal("9.6899995803833"), srcprice.price) 252 | timezone = datetime.timezone(datetime.timedelta(hours=-5), "America/New_York") 253 | self.assertEqual( 254 | datetime.datetime(2022, 2, 25, 9, 30, tzinfo=timezone), srcprice.time 255 | ) 256 | self.assertEqual("USD", srcprice.quote_currency) 257 | 258 | 259 | if __name__ == "__main__": 260 | unittest.main() 261 | -------------------------------------------------------------------------------- /bin/BUILD: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | # Note: The genrule targets rename the scripts to .py generated files. 4 | # Note: subpar does not yet support C extensions. 5 | 6 | genrule( 7 | name = "bean_price_py", 8 | srcs = ["bean-price"], 9 | outs = ["bean_price.py"], 10 | cmd = "cat $(locations :bean-price) > $@", 11 | ) 12 | 13 | py_binary( 14 | name = "bean_price", 15 | srcs = ["bean_price.py"], 16 | deps = [ 17 | "//beancount/prices:price", 18 | "//beancount:loader_with_plugins", 19 | ], 20 | ) 21 | -------------------------------------------------------------------------------- /bin/bean-price: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | __copyright__ = "Copyright (C) 2013-2020 Martin Blais" 3 | __license__ = "GNU GPLv2" 4 | from beanprice.price import main; main() 5 | -------------------------------------------------------------------------------- /etc/env: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # My environment initialization for this project. 4 | 5 | USERPATH=$USERPATH:$PROJDIR/bin 6 | PYTHONPATH=$PYTHONPATH:$PROJDIR 7 | -------------------------------------------------------------------------------- /experiments/dividends/download_dividends.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Download all dividends in a particular date interval.""" 3 | 4 | __copyright__ = "Copyright (C) 2020 Martin Blais" 5 | __license__ = "GNU GPLv2" 6 | 7 | from datetime import date as Date 8 | from decimal import Decimal 9 | from typing import List, Tuple 10 | import argparse 11 | import csv 12 | import datetime 13 | import io 14 | import pprint 15 | 16 | import dateutil.parser 17 | import requests 18 | 19 | 20 | def download_dividends( 21 | instrument: str, start_date: Date, end_date: Date 22 | ) -> List[Tuple[Date, Decimal]]: 23 | """Download a list of dividends issued over a time interval.""" 24 | tim = datetime.time() 25 | payload = { 26 | "period1": str(int(datetime.datetime.combine(start_date, tim).timestamp())), 27 | "period2": str(int(datetime.datetime.combine(end_date, tim).timestamp())), 28 | "interval": "1d", 29 | "events": "div", 30 | "includeAdjustedClose": "true", 31 | } 32 | template = " https://query1.finance.yahoo.com/v7/finance/download/{ticker}" 33 | url = template.format(ticker=instrument) 34 | resp = requests.get(url, params=payload) 35 | if not resp.ok: 36 | raise ValueError("Error fetching dividends: {}".format(resp.text)) 37 | 38 | rows = iter(csv.reader(io.StringIO(resp.text))) 39 | header = next(rows) 40 | if header != ["Date", "Dividends"]: 41 | raise ValueError( 42 | "Error fetching dividends: " "invalid response format: {}".format(header) 43 | ) 44 | 45 | dividends = [] 46 | for row in rows: 47 | date = datetime.datetime.strptime(row[0], "%Y-%m-%d").date() 48 | dividend = Decimal(row[1]) 49 | dividends.append((date, dividend)) 50 | return dividends 51 | 52 | 53 | def main(): 54 | """Top-level function.""" 55 | today = datetime.date.today() 56 | parser = argparse.ArgumentParser(description=__doc__.strip()) 57 | parser.add_argument("instrument", help="Yahoo!Finance code for financial instrument.") 58 | parser.add_argument( 59 | "start", 60 | action="store", 61 | type=lambda x: dateutil.parser.parse(x).date(), 62 | default=today.replace(year=today.year - 1), 63 | help="Start date of interval. Default is one year ago.", 64 | ) 65 | parser.add_argument( 66 | "end", 67 | action="store", 68 | type=lambda x: dateutil.parser.parse(x).date(), 69 | default=today, 70 | help="End date of interval. Default is today ago.", 71 | ) 72 | 73 | args = parser.parse_args() 74 | 75 | dividends = download_dividends(args.instrument, args.start, args.end) 76 | pprint.pprint(dividends) 77 | 78 | 79 | if __name__ == "__main__": 80 | main() 81 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ['setuptools'] 3 | build-backend = 'setuptools.build_meta' 4 | 5 | [project] 6 | name = 'beanprice' 7 | version = '2.0.0' 8 | description = 'Price quotes fetcher for Beancount' 9 | license = { file = 'COPYING' } 10 | readme = 'README.md' 11 | authors = [ 12 | { name = 'Martin Blais', email = 'blais@furius.ca' }, 13 | ] 14 | maintainers = [ 15 | { name = 'Martin Blais', email = 'blais@furius.ca' }, 16 | ] 17 | keywords = [ 18 | 'accounting', 'ledger', 'beancount', 'price' 19 | ] 20 | classifiers = [ 21 | 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)', 22 | 'Programming Language :: Python :: 3 :: Only', 23 | 'Programming Language :: Python :: 3.9', 24 | 'Programming Language :: Python :: 3.10', 25 | 'Programming Language :: Python :: 3.11', 26 | 'Programming Language :: Python :: 3.12', 27 | 'Programming Language :: SQL', 28 | 'Topic :: Office/Business :: Financial :: Accounting', 29 | ] 30 | requires-python = '>= 3.9' 31 | dependencies = [ 32 | 'beancount >= 3.0.0', 33 | 'python-dateutil >= 2.6.0', 34 | 'requests >= 2.0', 35 | 'curl_cffi>=0.6.5', 36 | ] 37 | 38 | [project.scripts] 39 | bean-price = 'beanprice.price:main' 40 | 41 | [project.urls] 42 | homepage = 'https://github.com/beancount/beanprice' 43 | issues = 'https://github.com/beancount/beanprice/issues' 44 | 45 | [tool.setuptools.packages] 46 | find = {} 47 | 48 | [tool.coverage.run] 49 | branch = true 50 | 51 | [tool.coverage.report] 52 | exclude_also = [ 53 | 'if typing.TYPE_CHECKING:', 54 | ] 55 | 56 | [tool.ruff] 57 | line-length = 92 58 | target-version = 'py39' 59 | 60 | [tool.ruff.lint] 61 | select = ['E', 'F', 'W', 'UP', 'B', 'C4', 'PL', 'RUF'] 62 | 63 | # TODO(blais): Review these ignores. 64 | ignore = [ 65 | 'RUF013', 66 | 'RUF005', 67 | 'PLW0603', 68 | 'UP014', 69 | 'UP031', 70 | 'B007', 71 | 'B905', 72 | 'C408', 73 | 'E731', 74 | 'PLR0911', 75 | 'PLR0912', 76 | 'PLR0913', 77 | 'PLR0915', 78 | 'PLR1714', 79 | 'PLR2004', 80 | 'PLW2901', 81 | 'RUF012', 82 | 'UP007', 83 | 'UP032', 84 | ] 85 | 86 | [tool.mypy] 87 | disable_error_code = ["import-untyped"] 88 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | beancount>=3.0.0 2 | python-dateutil>=2.6.0 3 | requests>=2.0 4 | curl_cffi>=0.6.5 -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | pylint==3.3.3 3 | pytest==5.4.2 4 | mypy==1.14.1 5 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Install script for beanprice.""" 3 | 4 | __copyright__ = "Copyright (C) 2008-2020 Martin Blais" 5 | __license__ = "GNU GPLv2" 6 | 7 | from setuptools import setup, find_packages 8 | 9 | 10 | setup( 11 | name="beanprice", 12 | version="2.0.0", 13 | description="Price quotes fetcher for Beancount", 14 | long_description=""" 15 | A script to fetch market data prices from various sources on the internet 16 | and render them for plain text accounting price syntax (and Beancount). 17 | """, 18 | license="GNU GPLv2 only", 19 | author="Martin Blais", 20 | author_email="blais@furius.ca", 21 | url="http://github.com/beancount/beanprice", 22 | download_url="https://github.com/beancount/beanprice", 23 | packages=find_packages(), 24 | install_requires=[ 25 | # Beancount library itself. 26 | "beancount>=3.0.0", 27 | # Testing support now uses the pytest module. 28 | "pytest", 29 | # This is required to parse dates from command-line options in a 30 | # loose, accepting format. Note that we use dateutil for timezone 31 | # database definitions as well, although it is inferior to pytz, but 32 | # because it can use the OS timezone database in the Windows 33 | # registry. See this article for context: 34 | # https://www.assert.cc/2014/05/25/which-python-time-zone-library.html 35 | # However, for creating offset timezones, we use the datetime.timezone 36 | # helper class because it is built-in. 37 | # Where this matters is for price source fetchers. 38 | # (Note: If pytz supported the Windows registry timezone information, 39 | # I would switch to that.) 40 | "python-dateutil", 41 | # This library is needed to make requests for price sources. 42 | "requests", 43 | ], 44 | entry_points={ 45 | "console_scripts": [ 46 | "bean-price = beanprice.price:main", 47 | ] 48 | }, 49 | python_requires=">=3.9", 50 | ) 51 | --------------------------------------------------------------------------------