├── .github └── workflows │ ├── ExtensionTemplate.yml │ ├── MainDistributionPipeline.yml │ └── _extension_deploy.yml ├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── LICENSE ├── Makefile ├── README.md ├── crypto_sample.zip ├── extension_config.cmake ├── images └── scrooge-plot.png ├── scripts ├── extension-upload.sh ├── generate_token_map.py ├── generate_uniswap_map.py └── scrap_pools.py ├── src ├── functions │ ├── aliases.cpp │ ├── first.cpp │ ├── last.cpp │ ├── portfolio_frontier.cpp │ └── timebucket.cpp ├── include │ ├── functions │ │ ├── functions.hpp │ │ └── scanner.hpp │ ├── scrooge_extension.hpp │ └── util │ │ ├── eth_maps.hpp │ │ ├── eth_tokens_map.hpp │ │ ├── eth_uniswap_map.hpp │ │ ├── hex_converter.hpp │ │ └── http_util.hpp ├── scanner │ ├── ethereum_blockchain.cpp │ └── yahoo_finance.cpp └── scrooge_extension.cpp ├── test └── sql │ └── scrooge │ ├── test_eth_local.test │ ├── test_eth_projection.test │ ├── test_eth_roi.test │ ├── test_eth_scan.test │ ├── test_eth_uniswap.test │ ├── test_scrooge_first.test │ ├── test_scrooge_last.test │ ├── test_scrooge_timebucket.test │ └── test_yahoo.test ├── third_party └── json.hpp └── vcpkg.json /.github/workflows/ExtensionTemplate.yml: -------------------------------------------------------------------------------- 1 | # 2 | # NOTE: this workflow is for testing the extension template itself, 3 | # this workflow will be removed when scripts/bootstrap-template.py is run 4 | # 5 | name: Extension Template 6 | on: [push, pull_request,repository_dispatch] 7 | concurrency: 8 | group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} 9 | cancel-in-progress: true 10 | 11 | jobs: 12 | linux: 13 | name: Linux 14 | if: ${{ vars.RUN_RENAME_TEST == 'true' || github.repository == 'duckdb/extension-template' }} 15 | runs-on: ubuntu-latest 16 | container: ubuntu:18.04 17 | strategy: 18 | matrix: 19 | # Add commits/tags to build against other DuckDB versions 20 | duckdb_version: [ '' ] 21 | env: 22 | VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake 23 | VCPKG_TARGET_TRIPLET: 'x64-linux' 24 | GEN: ninja 25 | defaults: 26 | run: 27 | shell: bash 28 | 29 | steps: 30 | - name: Install required ubuntu packages 31 | run: | 32 | apt-get update -y -qq 33 | apt-get install -y -qq software-properties-common 34 | add-apt-repository ppa:git-core/ppa 35 | apt-get update -y -qq 36 | apt-get install -y -qq ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client 37 | 38 | - name: Install Git 2.18.5 39 | run: | 40 | wget https://github.com/git/git/archive/refs/tags/v2.18.5.tar.gz 41 | tar xvf v2.18.5.tar.gz 42 | cd git-2.18.5 43 | make 44 | make prefix=/usr install 45 | git --version 46 | 47 | - uses: actions/checkout@v4 48 | with: 49 | fetch-depth: 0 50 | submodules: 'true' 51 | 52 | - name: Checkout DuckDB to version 53 | if: ${{ matrix.duckdb_version != ''}} 54 | run: | 55 | cd duckdb 56 | git checkout ${{ matrix.duckdb_version }} 57 | 58 | - uses: ./duckdb/.github/actions/ubuntu_18_setup 59 | 60 | - name: Setup vcpkg 61 | uses: lukka/run-vcpkg@v11.1 62 | with: 63 | vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6 64 | 65 | - name: Rename extension 66 | run: | 67 | python3 scripts/bootstrap-template.py ext_1_a_123b_b11 68 | 69 | - name: Build 70 | run: | 71 | make 72 | 73 | - name: Test 74 | run: | 75 | make test 76 | 77 | macos: 78 | name: MacOS 79 | if: ${{ vars.RUN_RENAME_TEST == 'true' || github.repository == 'duckdb/extension-template' }} 80 | runs-on: macos-latest 81 | strategy: 82 | matrix: 83 | # Add commits/tags to build against other DuckDB versions 84 | duckdb_version: [ ''] 85 | env: 86 | VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake 87 | VCPKG_TARGET_TRIPLET: 'x64-osx' 88 | OSX_BUILD_ARCH: 'x86_64' 89 | GEN: ninja 90 | defaults: 91 | run: 92 | shell: bash 93 | 94 | steps: 95 | - uses: actions/checkout@v4 96 | with: 97 | fetch-depth: 0 98 | submodules: 'true' 99 | 100 | - name: Install Ninja 101 | run: brew install ninja 102 | 103 | - uses: actions/setup-python@v2 104 | with: 105 | python-version: '3.11' 106 | 107 | - name: Checkout DuckDB to version 108 | if: ${{ matrix.duckdb_version != ''}} 109 | run: | 110 | cd duckdb 111 | git checkout ${{ matrix.duckdb_version }} 112 | 113 | - name: Setup vcpkg 114 | uses: lukka/run-vcpkg@v11.1 115 | with: 116 | vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6 117 | 118 | - name: Rename extension 119 | run: | 120 | python scripts/bootstrap-template.py ext_1_a_123b_b11 121 | 122 | - name: Build 123 | run: | 124 | make 125 | 126 | - name: Test 127 | run: | 128 | make test 129 | 130 | # windows: 131 | # name: Windows 132 | # if: ${{ vars.RUN_RENAME_TEST == 'true' || github.repository == 'duckdb/extension-template' }} 133 | # runs-on: windows-latest 134 | # strategy: 135 | # matrix: 136 | # # Add commits/tags to build against other DuckDB versions 137 | # duckdb_version: [ '' ] 138 | # env: 139 | # VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake 140 | # VCPKG_TARGET_TRIPLET: 'x64-windows-static-md' 141 | # defaults: 142 | # run: 143 | # shell: bash 144 | # 145 | # steps: 146 | # - uses: actions/checkout@v3 147 | # with: 148 | # fetch-depth: 0 149 | # submodules: 'true' 150 | # 151 | # - uses: actions/setup-python@v2 152 | # with: 153 | # python-version: '3.11' 154 | # 155 | # - name: Checkout DuckDB to version 156 | # # Add commits/tags to build against other DuckDB versions 157 | # if: ${{ matrix.duckdb_version != ''}} 158 | # run: | 159 | # cd duckdb 160 | # git checkout ${{ matrix.duckdb_version }} 161 | # 162 | # - name: Setup vcpkg 163 | # uses: lukka/run-vcpkg@v11.5 164 | # with: 165 | # vcpkgGitCommitId: 5e0cab206a5ea620130caf672fce3e4a6b5666a1 166 | # 167 | # - name: Rename extension 168 | # run: | 169 | # python scripts/bootstrap-template.py ext_1_a_123b_b11 170 | # 171 | # - name: Build 172 | # run: | 173 | # make 174 | # 175 | # - name: Test extension 176 | # run: | 177 | # build/release/test/Release/unittest.exe -------------------------------------------------------------------------------- /.github/workflows/MainDistributionPipeline.yml: -------------------------------------------------------------------------------- 1 | # 2 | # This workflow calls the main distribution pipeline from DuckDB to build, test and (optionally) release the extension 3 | # 4 | name: Main Extension Distribution Pipeline 5 | on: 6 | push: 7 | pull_request: 8 | workflow_dispatch: 9 | 10 | concurrency: 11 | group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} 12 | cancel-in-progress: true 13 | 14 | jobs: 15 | duckdb-stable-build: 16 | name: Build extension binaries 17 | uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main 18 | with: 19 | duckdb_version: v1.2.2 20 | extension_name: scrooge 21 | exclude_archs: "windows_amd64_rtools" 22 | ci_tools_version: main 23 | 24 | duckdb-stable-deploy: 25 | name: Deploy extension binaries 26 | needs: duckdb-stable-build 27 | uses: ./.github/workflows/_extension_deploy.yml 28 | secrets: inherit 29 | with: 30 | duckdb_version: v1.2.2 31 | extension_name: scrooge 32 | deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} 33 | exclude_archs: "windows_amd64_rtools" -------------------------------------------------------------------------------- /.github/workflows/_extension_deploy.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Reusable workflow that deploys the artifacts produced by github.com/duckdb/duckdb/.github/workflows/_extension_distribution.yml 3 | # 4 | # note: this workflow needs to be located in the extension repository, as it requires secrets to be passed to the 5 | # deploy script. However, it should generally not be necessary to modify this workflow in your extension repository, as 6 | # this workflow can be configured to use a custom deploy script. 7 | 8 | 9 | name: Extension Deployment 10 | on: 11 | workflow_call: 12 | inputs: 13 | # The name of the extension 14 | extension_name: 15 | required: true 16 | type: string 17 | # DuckDB version to build against 18 | duckdb_version: 19 | required: true 20 | type: string 21 | # ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64' 22 | exclude_archs: 23 | required: false 24 | type: string 25 | default: "" 26 | # Whether to upload this deployment as the latest. This may overwrite a previous deployment. 27 | deploy_latest: 28 | required: false 29 | type: boolean 30 | default: false 31 | # Whether to upload this deployment under a versioned path. These will not be deleted automatically 32 | deploy_versioned: 33 | required: false 34 | type: boolean 35 | default: false 36 | # Postfix added to artifact names. Can be used to guarantee unique names when this workflow is called multiple times 37 | artifact_postfix: 38 | required: false 39 | type: string 40 | default: "" 41 | # Override the default deploy script with a custom script 42 | deploy_script: 43 | required: false 44 | type: string 45 | default: "./scripts/extension-upload.sh" 46 | # Override the default matrix parse script with a custom script 47 | matrix_parse_script: 48 | required: false 49 | type: string 50 | default: "./duckdb/scripts/modify_distribution_matrix.py" 51 | 52 | jobs: 53 | generate_matrix: 54 | name: Generate matrix 55 | runs-on: ubuntu-latest 56 | outputs: 57 | deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }} 58 | steps: 59 | - uses: actions/checkout@v4 60 | with: 61 | fetch-depth: 0 62 | submodules: 'true' 63 | 64 | - name: Checkout DuckDB to version 65 | run: | 66 | cd duckdb 67 | git checkout ${{ inputs.duckdb_version }} 68 | 69 | - id: parse-matrices 70 | run: | 71 | python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty 72 | deploy_matrix="`cat deploy_matrix.json`" 73 | echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT 74 | echo `cat $GITHUB_OUTPUT` 75 | 76 | deploy: 77 | name: Deploy 78 | runs-on: ubuntu-latest 79 | needs: generate_matrix 80 | if: ${{ needs.generate_matrix.outputs.deploy_matrix != '{}' && needs.generate_matrix.outputs.deploy_matrix != '' }} 81 | strategy: 82 | matrix: ${{fromJson(needs.generate_matrix.outputs.deploy_matrix)}} 83 | 84 | steps: 85 | - uses: actions/checkout@v4 86 | with: 87 | fetch-depth: 0 88 | submodules: 'true' 89 | 90 | - name: Checkout DuckDB to version 91 | run: | 92 | cd duckdb 93 | git checkout ${{ inputs.duckdb_version }} 94 | 95 | - uses: actions/download-artifact@v4 96 | with: 97 | name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}${{startsWith(matrix.duckdb, 'wasm') && '.wasm' || ''}} 98 | path: | 99 | /tmp/extension 100 | 101 | - name: Deploy 102 | shell: bash 103 | env: 104 | AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} 105 | AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} 106 | AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} 107 | BUCKET_NAME: ${{ secrets.S3_BUCKET }} 108 | DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.S3_DUCKDB_ORG_EXTENSION_SIGNING_PK }} 109 | run: | 110 | pwd 111 | python3 -m pip install pip awscli 112 | git config --global --add safe.directory '*' 113 | cd duckdb 114 | git fetch --tags 115 | export DUCKDB_VERSION=`git tag --points-at HEAD` 116 | export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} 117 | cd .. 118 | git fetch --tags 119 | export EXT_VERSION=`git tag --points-at HEAD` 120 | export EXT_VERSION=${EXT_VERSION:=`git log -1 --format=%h`} 121 | ${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}} 122 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | .idea 3 | cmake-build-debug 4 | duckdb_unittest_tempdir/ 5 | .DS_Store -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "duckdb"] 2 | path = duckdb 3 | url = https://github.com/duckdb/duckdb.git 4 | branch = main 5 | 6 | [submodule "extension-ci-tools"] 7 | path = extension-ci-tools 8 | url = https://github.com/duckdb/extension-ci-tools/ 9 | branch = main 10 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.5) 2 | 3 | set(TARGET_NAME scrooge) 4 | 5 | # DuckDB's extension distribution supports vcpkg. As such, dependencies can be 6 | # added in ./vcpkg.json and then used in cmake with find_package. Feel free to 7 | # remove or replace with other dependencies. Note that it should also be removed 8 | # from vcpkg.json to prevent needlessly installing it.. 9 | find_package(OpenSSL REQUIRED) 10 | 11 | set(EXTENSION_NAME ${TARGET_NAME}_extension) 12 | set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension) 13 | 14 | project(${TARGET_NAME}) 15 | include_directories(src/include) 16 | include_directories(third_party) 17 | include_directories(duckdb/third_party/httplib) 18 | include_directories(${OPENSSL_INCLUDE_DIR}) 19 | 20 | set(EXTENSION_SOURCES 21 | src/scrooge_extension.cpp 22 | src/functions/first.cpp 23 | src/functions/last.cpp 24 | src/functions/timebucket.cpp 25 | src/scanner/yahoo_finance.cpp 26 | src/scanner/ethereum_blockchain.cpp 27 | src/functions/aliases.cpp 28 | src/functions/portfolio_frontier.cpp) 29 | 30 | build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) 31 | build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) 32 | 33 | # Link OpenSSL in both the static library as the loadable extension 34 | target_link_libraries(${EXTENSION_NAME} OpenSSL::SSL OpenSSL::Crypto) 35 | target_link_libraries(${LOADABLE_EXTENSION_NAME} OpenSSL::SSL OpenSSL::Crypto) 36 | 37 | install( 38 | TARGETS ${EXTENSION_NAME} 39 | EXPORT "${DUCKDB_EXPORT_SET}" 40 | LIBRARY DESTINATION "${INSTALL_LIB_DIR}" 41 | ARCHIVE DESTINATION "${INSTALL_LIB_DIR}") 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Pedro Holanda 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) 2 | 3 | 4 | # Configuration of extension 5 | EXT_NAME=scrooge 6 | EXT_CONFIG=${PROJ_DIR}extension_config.cmake 7 | 8 | # We need this for testing 9 | CORE_EXTENSIONS='httpfs' 10 | 11 | # Include the Makefile from extension-ci-tools 12 | include extension-ci-tools/makefiles/duckdb_extension.Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Scrooge McDuck Extension 2 | 3 | Scrooge McDuck is a third-party financial extension for [DuckDB](https://www.duckdb.org). The main goal of this extension is to support a set of aggregation functions and data scanners for financial data. It currently supports access to the logs of Ethereum nodes and stock information from Yahoo Finance. 4 | 5 | This extension is still under development, with no official version released yet. 6 | 7 | You can find more details on the [supported scanners](https://github.com/pdet/Scrooge-McDuck/wiki/Data-Scanners), [custom functions](https://github.com/pdet/Scrooge-McDuck/wiki/Custom-Functions), and [usage](https://github.com/pdet/Scrooge-McDuck/wiki/Usage) in the Scrooge wiki. 8 | 9 | > **Disclaimer:** This extension is in no way affiliated with the [DuckDB Foundation](https://duckdb.org/foundation/) or [DuckDB Labs](https://duckdblabs.com/). Therefore, any binaries produced and distributed of this extension are unsigned. 10 | 11 | ## Roadmap 12 | A roadmap for the next version of Scrooge is currently maintained as a discussion. You can find it [here](https://github.com/pdet/Scrooge-McDuck/discussions/22). 13 | 14 | ## Why Scrooge? 15 | 1. DuckDB is an easy-to-use, fast system for analytics. Scrooge takes advantage of all the design decisions of DuckDB that make it a highly performant database system for analytics (e.g., columnar storage, vectorized execution). 16 | 2. Privacy/Security. Since DuckDB runs locally, all your queries are completely private and fully executed on your machine. 17 | 3. Cost-Efficiency. Both DuckDB and Scrooge are completely free and available under an MIT License. There are no costs associated with using them, unlike with a cloud-based engine; all you need is your own machine. 18 | 4. [Subquery Flattening](https://duckdb.org/2023/05/26/correlated-subqueries-in-sql.html). This is a DuckDB optimization that few systems implement. Financial queries (e.g., ROIs) can get extremely complex and will be efficiently executed by DuckDB. 19 | 20 | ## Build 21 | To build, type 22 | ``` sh 23 | make 24 | ``` 25 | 26 | To run, run the `duckdb` shell with the unsined flag: 27 | ``` sh 28 | cd build/release/ 29 | ./duckdb -unsigned 30 | ``` 31 | 32 | Then, load the Scrooge McDuck extension like so: 33 | ```SQL 34 | LOAD 'extension/scrooge/scrooge.duckdb_extension'; 35 | ``` 36 | ## Blogposts 37 | - [Candle-Stick Aggregation](https://pdet-blog.github.io/2022/08/16/scrooge.html) 38 | - [Yahoo Scanner](https://pdet-blog.github.io/2023/02/25/yahoofinance.html) 39 | 40 | -------------------------------------------------------------------------------- /crypto_sample.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pdet/Scrooge-McDuck/222e094570f307208258b8faf90dd401fa152acd/crypto_sample.zip -------------------------------------------------------------------------------- /extension_config.cmake: -------------------------------------------------------------------------------- 1 | # This file is included by DuckDB's build system. It specifies which extension to load 2 | 3 | # Extension from this repo 4 | duckdb_extension_load(scrooge 5 | SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} 6 | LOAD_TESTS 7 | ) 8 | 9 | # Build the httpfs extension to test with s3/http 10 | duckdb_extension_load(json) 11 | -------------------------------------------------------------------------------- /images/scrooge-plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pdet/Scrooge-McDuck/222e094570f307208258b8faf90dd401fa152acd/images/scrooge-plot.png -------------------------------------------------------------------------------- /scripts/extension-upload.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Extension upload script 4 | 5 | # Usage: ./extension-upload.sh 6 | # : Name of the extension 7 | # : Version (commit / version tag) of the extension 8 | # : Version (commit / version tag) of DuckDB 9 | # : Architecture target of the extension binary 10 | # : S3 bucket to upload to 11 | # : Set this as the latest version ("true" / "false", default: "false") 12 | # : Set this as a versioned version that will prevent its deletion 13 | 14 | set -e 15 | 16 | if [[ $4 == wasm* ]]; then 17 | ext="/tmp/extension/$1.duckdb_extension.wasm" 18 | else 19 | ext="/tmp/extension/$1.duckdb_extension" 20 | fi 21 | 22 | echo $ext 23 | 24 | script_dir="$(dirname "$(readlink -f "$0")")" 25 | 26 | # calculate SHA256 hash of extension binary 27 | cat $ext > $ext.append 28 | 29 | if [[ $4 == wasm* ]]; then 30 | # 0 for custom section 31 | # 113 in hex = 275 in decimal, total lenght of what follows (1 + 16 + 2 + 256) 32 | # [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x02] 33 | echo -n -e '\x00' >> $ext.append 34 | echo -n -e '\x93\x02' >> $ext.append 35 | # 10 in hex = 16 in decimal, lenght of name, 1 byte 36 | echo -n -e '\x10' >> $ext.append 37 | echo -n -e 'duckdb_signature' >> $ext.append 38 | # the name of the WebAssembly custom section, 16 bytes 39 | # 100 in hex, 256 in decimal 40 | # [1(continuation) + 0000000(payload) = ff, 0(continuation) + 10(payload)], 41 | # for a grand total of 2 bytes 42 | echo -n -e '\x80\x02' >> $ext.append 43 | fi 44 | 45 | # (Optionally) Sign binary 46 | if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then 47 | echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem 48 | $script_dir/../duckdb/scripts/compute-extension-hash.sh $ext.append > $ext.hash 49 | openssl pkeyutl -sign -in $ext.hash -inkey private.pem -pkeyopt digest:sha256 -out $ext.sign 50 | rm -f private.pem 51 | fi 52 | 53 | # Signature is always there, potentially defaulting to 256 zeros 54 | truncate -s 256 $ext.sign 55 | 56 | # append signature to extension binary 57 | cat $ext.sign >> $ext.append 58 | 59 | # compress extension binary 60 | if [[ $4 == wasm_* ]]; then 61 | brotli < $ext.append > "$ext.compressed" 62 | else 63 | gzip < $ext.append > "$ext.compressed" 64 | fi 65 | 66 | set -e 67 | 68 | # Abort if AWS key is not set 69 | if [ -z "$AWS_ACCESS_KEY_ID" ]; then 70 | echo "No AWS key found, skipping.." 71 | exit 0 72 | fi 73 | 74 | # upload versioned version 75 | if [[ $7 = 'true' ]]; then 76 | if [[ $4 == wasm* ]]; then 77 | aws s3 cp $ext.compressed s3://$5/$1/$2/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" 78 | else 79 | aws s3 cp $ext.compressed s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read 80 | fi 81 | fi 82 | 83 | # upload to latest version 84 | if [[ $6 = 'true' ]]; then 85 | if [[ $4 == wasm* ]]; then 86 | aws s3 cp $ext.compressed s3://$5/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" 87 | else 88 | aws s3 cp $ext.compressed s3://$5/$3/$4/$1.duckdb_extension.gz --acl public-read 89 | fi 90 | fi -------------------------------------------------------------------------------- /scripts/generate_token_map.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | import duckdb 4 | 5 | def get_ethereum_tokens(base_url): 6 | headers = { 7 | 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' 8 | } 9 | tokens = [] 10 | page_number = 1 11 | 12 | while True: 13 | url = f"{base_url}?p={page_number}" 14 | response = requests.get(url, headers=headers) 15 | soup = BeautifulSoup(response.content, 'html.parser') 16 | 17 | table = soup.find('table', {'class': 'table-hover'}) 18 | 19 | if not table: 20 | break 21 | 22 | rows = table.find_all('tr') 23 | 24 | for row in rows[1:]: 25 | columns = row.find_all('td') 26 | if len(columns) > 1: 27 | name_tag = columns[1].find('a') 28 | address_tag = columns[1].find('a', href=True) 29 | 30 | if name_tag and address_tag: 31 | full_name = name_tag.text.strip() 32 | address = address_tag['href'].split('/')[-1] 33 | 34 | if '\n' in full_name: 35 | name, symbol = full_name.split('\n') 36 | symbol = symbol.strip('()') 37 | else: 38 | name = full_name 39 | symbol = '' 40 | 41 | tokens.append((name.strip(), symbol.strip(), address.strip())) 42 | print(f"Found token: {name.strip()}, {symbol.strip()}, {address.strip()}") 43 | 44 | page_number += 1 45 | next_page = soup.find('a', {'aria-label': 'Next'}) 46 | if not next_page or 'disabled' in next_page.get('class', []): 47 | break 48 | 49 | return tokens 50 | 51 | def generate_header(tokens, header_file): 52 | conn = duckdb.connect() 53 | conn.execute("CREATE TABLE tokens (Name VARCHAR, Symbol VARCHAR, Address VARCHAR)") 54 | conn.executemany("INSERT INTO tokens VALUES (?, ?, ?)", tokens) 55 | sorted_tokens = conn.execute(""" 56 | FROM tokens 57 | ORDER BY Symbol 58 | """).fetchall() 59 | 60 | # Open the header file for writing 61 | with open(header_file, 'w') as f: 62 | f.write('//===----------------------------------------------------------------------===//\n') 63 | f.write('// Scrooge\n') 64 | f.write('//\n') 65 | f.write('// util/eth_tokens_map.hpp\n') 66 | f.write('//\n') 67 | f.write('//===----------------------------------------------------------------------===//\n\n') 68 | f.write('//===----This file is auto-generated by scripts/generate_token_map.py----===//\n\n') 69 | f.write('#pragma once\n\n') 70 | f.write('#include \n') 71 | f.write('#include \n\n') 72 | f.write('namespace duckdb {\n') 73 | f.write('namespace scrooge {\n') 74 | f.write('const std::unordered_map token_addresses = {\n') 75 | 76 | # Iterate over the sorted tokens and write each token and address 77 | for row in sorted_tokens: 78 | name, symbol, address, = row[0], row[1], row[2] 79 | f.write(f' {{"{symbol}", "{address}"}}, // {name}\n') 80 | 81 | f.write('};\n') 82 | 83 | f.write('const vector token_symbols = {\n') 84 | 85 | for row in sorted_tokens: 86 | symbol = row[1] 87 | f.write(f'"{symbol}",\n') 88 | 89 | f.write('};\n') 90 | 91 | f.write('} // namespace scrooge\n') 92 | f.write('} // namespace duckdb\n') 93 | 94 | if __name__ == "__main__": 95 | base_url = 'https://etherscan.io/tokens' # Base URL for the pages with Ethereum tokens 96 | tokens = get_ethereum_tokens(base_url) 97 | header_file = 'src/include/util/eth_tokens_map.hpp' # Output header file 98 | generate_header(tokens, header_file) 99 | print(f"Header file {header_file} generated successfully.") 100 | -------------------------------------------------------------------------------- /scripts/generate_uniswap_map.py: -------------------------------------------------------------------------------- 1 | import duckdb 2 | 3 | def generate_cpp_header(csv_file_path, header_file_path): 4 | con = duckdb.connect() 5 | df = con.execute(f"SELECT * FROM read_csv_auto('{csv_file_path}')").fetchdf() 6 | 7 | cpp_header = """//===----------------------------------------------------------------------===// 8 | // Scrooge 9 | // 10 | // util/eth_uniswap_map.hpp 11 | // 12 | //===----------------------------------------------------------------------===// 13 | 14 | //===----This file is auto-generated by scripts/generate_uniswap_map.py----===// 15 | 16 | #pragma once 17 | 18 | #include 19 | #include 20 | 21 | namespace duckdb { 22 | namespace scrooge { 23 | const unordered_map uniswap_addresses = { 24 | """ 25 | 26 | for _, row in df.iterrows(): 27 | pair = f'{{"{row["token0_symbol"]}_{row["token1_symbol"]}", "{row["pair_address"]}"}}' 28 | cpp_header += f" {pair},\n" 29 | 30 | cpp_header = cpp_header.rstrip(",\n") + "\n};\n\n" 31 | cpp_header+= "const vector uniswap_symbols = {" 32 | for _, row in df.iterrows(): 33 | cpp_header += f'"{row["token0_symbol"]}_{row["token1_symbol"]}",\n' 34 | cpp_header = cpp_header.rstrip(",\n") + "\n};" 35 | cpp_header+="\n}}" 36 | 37 | with open(header_file_path, 'w') as file: 38 | file.write(cpp_header) 39 | 40 | csv_file_path = '/Users/holanda/Desktop/uniswap_pairs.csv' 41 | header_file_path = 'src/include/util/eth_uniswap_map.hpp' 42 | generate_cpp_header(csv_file_path, header_file_path) 43 | -------------------------------------------------------------------------------- /scripts/scrap_pools.py: -------------------------------------------------------------------------------- 1 | from web3 import Web3 2 | import requests 3 | import csv 4 | 5 | infura_url = 'https://mainnet.infura.io/v3/token' 6 | 7 | web3 = Web3(Web3.HTTPProvider(infura_url)) 8 | 9 | if not web3.is_connected(): 10 | print("Failed to connect to Ethereum node") 11 | exit() 12 | 13 | uniswap_factory_address = '0x5C69bEe701ef814a2B6a3EDD4B1652CB9cc5aA6f' 14 | uniswap_factory_abi = ''' 15 | [ 16 | { 17 | "constant": true, 18 | "inputs": [], 19 | "name": "allPairsLength", 20 | "outputs": [ 21 | { 22 | "internalType": "uint256", 23 | "name": "", 24 | "type": "uint256" 25 | } 26 | ], 27 | "payable": false, 28 | "stateMutability": "view", 29 | "type": "function" 30 | }, 31 | { 32 | "constant": true, 33 | "inputs": [ 34 | { 35 | "internalType": "uint256", 36 | "name": "", 37 | "type": "uint256" 38 | } 39 | ], 40 | "name": "allPairs", 41 | "outputs": [ 42 | { 43 | "internalType": "address", 44 | "name": "", 45 | "type": "address" 46 | } 47 | ], 48 | "payable": false, 49 | "stateMutability": "view", 50 | "type": "function" 51 | } 52 | ] 53 | ''' 54 | 55 | uniswap_pair_abi = ''' 56 | [ 57 | { 58 | "constant": true, 59 | "inputs": [], 60 | "name": "token0", 61 | "outputs": [ 62 | { 63 | "internalType": "address", 64 | "name": "", 65 | "type": "address" 66 | } 67 | ], 68 | "payable": false, 69 | "stateMutability": "view", 70 | "type": "function" 71 | }, 72 | { 73 | "constant": true, 74 | "inputs": [], 75 | "name": "token1", 76 | "outputs": [ 77 | { 78 | "internalType": "address", 79 | "name": "", 80 | "type": "address" 81 | } 82 | ], 83 | "payable": false, 84 | "stateMutability": "view", 85 | "type": "function" 86 | } 87 | ] 88 | ''' 89 | erc20_abi = ''' 90 | [ 91 | { 92 | "constant": true, 93 | "inputs": [], 94 | "name": "symbol", 95 | "outputs": [ 96 | { 97 | "internalType": "string", 98 | "name": "", 99 | "type": "string" 100 | } 101 | ], 102 | "payable": false, 103 | "stateMutability": "view", 104 | "type": "function" 105 | } 106 | ] 107 | ''' 108 | 109 | uniswap_factory = web3.eth.contract(address=uniswap_factory_address, abi=uniswap_factory_abi) 110 | 111 | num_pairs = uniswap_factory.functions.allPairsLength().call() 112 | 113 | with open('uniswap_pairs.csv', mode='w', newline='') as file: 114 | writer = csv.writer(file) 115 | writer.writerow(['token0_symbol', 'token0_address', 'token1_symbol', 'token1_address', 'pair_address']) 116 | 117 | for i in range(num_pairs): 118 | pair_address = uniswap_factory.functions.allPairs(i).call() 119 | pair_contract = web3.eth.contract(address=pair_address, abi=uniswap_pair_abi) 120 | 121 | token0_address = pair_contract.functions.token0().call() 122 | token1_address = pair_contract.functions.token1().call() 123 | 124 | token0_contract = web3.eth.contract(address=token0_address, abi=erc20_abi) 125 | token1_contract = web3.eth.contract(address=token1_address, abi=erc20_abi) 126 | 127 | try: 128 | token0_symbol = token0_contract.functions.symbol().call() 129 | if isinstance(token0_symbol, bytes): 130 | token0_symbol = token0_symbol.decode('utf-8').rstrip('\x00') 131 | except Exception as e: 132 | token0_symbol = 'Unknown' 133 | print(f"Could not retrieve symbol for token0 at address {token0_address}: {e}") 134 | 135 | try: 136 | token1_symbol = token1_contract.functions.symbol().call() 137 | if isinstance(token1_symbol, bytes): 138 | token1_symbol = token1_symbol.decode('utf-8').rstrip('\x00') 139 | except Exception as e: 140 | token1_symbol = 'Unknown' 141 | print(f"Could not retrieve symbol for token1 at address {token1_address}: {e}") 142 | 143 | writer.writerow([token0_symbol, token0_address, token1_symbol, token1_address, pair_address]) 144 | 145 | -------------------------------------------------------------------------------- /src/functions/aliases.cpp: -------------------------------------------------------------------------------- 1 | #include "functions/functions.hpp" 2 | #include "duckdb/catalog/catalog_entry/aggregate_function_catalog_entry.hpp" 3 | #include "duckdb/common/helper.hpp" 4 | 5 | namespace duckdb { 6 | 7 | namespace scrooge { 8 | void Aliases::Register(Connection &conn, Catalog &catalog) { 9 | // Register Volatility 10 | auto &stddev = 11 | catalog 12 | .GetEntry(*conn.context, CatalogType::AGGREGATE_FUNCTION_ENTRY, 13 | DEFAULT_SCHEMA, "stddev_pop") 14 | .Cast(); 15 | auto volatility = stddev.functions; 16 | volatility.name = "volatility"; 17 | CreateAggregateFunctionInfo volatility_info(volatility); 18 | catalog.CreateFunction(*conn.context, volatility_info); 19 | 20 | // Register SMA 21 | auto &avg = 22 | catalog 23 | .GetEntry(*conn.context, CatalogType::AGGREGATE_FUNCTION_ENTRY, 24 | DEFAULT_SCHEMA, "avg") 25 | .Cast(); 26 | auto sma = avg.functions; 27 | sma.name = "sma"; 28 | CreateAggregateFunctionInfo sma_info(sma); 29 | catalog.CreateFunction(*conn.context, sma_info); 30 | } 31 | } // namespace scrooge 32 | } // namespace duckdb -------------------------------------------------------------------------------- /src/functions/first.cpp: -------------------------------------------------------------------------------- 1 | #include "functions/functions.hpp" 2 | #include "duckdb/function/function_set.hpp" 3 | #include "duckdb/parser/parsed_data/create_aggregate_function_info.hpp" 4 | #include "duckdb/common/helper.hpp" 5 | 6 | namespace duckdb { 7 | namespace scrooge { 8 | 9 | template struct FirstScroogeState { 10 | T first; 11 | int64_t earliest_time; 12 | bool executed; 13 | }; 14 | 15 | struct FirstScroogeOperation { 16 | template static void Initialize(STATE &state) { 17 | state.earliest_time = NumericLimits::Maximum(); 18 | state.executed = false; 19 | } 20 | 21 | template 22 | static void Operation(STATE &state, const A_TYPE &x_data, 23 | const B_TYPE &y_data, AggregateBinaryInput &idata) { 24 | if (!state.executed || y_data < state.earliest_time) { 25 | state.earliest_time = y_data; 26 | state.first = x_data; 27 | state.executed = true; 28 | } 29 | } 30 | 31 | template 32 | static void Combine(const STATE &source, STATE &target, 33 | AggregateInputData &aggr_input_data) { 34 | if (!target.executed) { 35 | target = source; 36 | } else if (source.executed) { 37 | if (target.earliest_time > source.earliest_time) { 38 | target.earliest_time = source.earliest_time; 39 | target.first = source.first; 40 | } 41 | } 42 | } 43 | 44 | template 45 | static void Finalize(STATE &state, T &target, 46 | AggregateFinalizeData &finalize_data) { 47 | if (!state.executed) { 48 | finalize_data.ReturnNull(); 49 | } else { 50 | target = state.first; 51 | } 52 | } 53 | 54 | static bool IgnoreNull() { return true; } 55 | }; 56 | 57 | unique_ptr 58 | BindDoubleFirst(ClientContext &context, AggregateFunction &function, 59 | vector> &arguments) { 60 | auto &decimal_type = arguments[0]->return_type; 61 | switch (decimal_type.InternalType()) { 62 | case PhysicalType::INT16: { 63 | function = AggregateFunction::BinaryAggregate, 64 | int16_t, int64_t, int16_t, 65 | FirstScroogeOperation>( 66 | decimal_type, LogicalType::TIMESTAMP_TZ, decimal_type); 67 | break; 68 | } 69 | case PhysicalType::INT32: { 70 | function = AggregateFunction::BinaryAggregate, 71 | int32_t, int64_t, int32_t, 72 | FirstScroogeOperation>( 73 | decimal_type, LogicalType::TIMESTAMP_TZ, decimal_type); 74 | break; 75 | } 76 | case PhysicalType::INT64: { 77 | function = AggregateFunction::BinaryAggregate, 78 | int64_t, int64_t, int64_t, 79 | FirstScroogeOperation>( 80 | decimal_type, LogicalType::TIMESTAMP_TZ, decimal_type); 81 | break; 82 | } 83 | default: 84 | function = AggregateFunction::BinaryAggregate, 85 | Hugeint, int64_t, Hugeint, 86 | FirstScroogeOperation>( 87 | decimal_type, LogicalType::TIMESTAMP_TZ, decimal_type); 88 | } 89 | function.name = "first_s"; 90 | return nullptr; 91 | } 92 | 93 | AggregateFunction GetFirstScroogeFunction(const LogicalType ×tamp_type, 94 | const LogicalType &type) { 95 | switch (type.id()) { 96 | case LogicalTypeId::SMALLINT: 97 | return AggregateFunction::BinaryAggregate, 98 | int16_t, int64_t, int16_t, 99 | FirstScroogeOperation>( 100 | type, timestamp_type, type); 101 | case LogicalTypeId::TINYINT: 102 | return AggregateFunction::BinaryAggregate, int8_t, 103 | int64_t, int8_t, 104 | FirstScroogeOperation>( 105 | type, timestamp_type, type); 106 | case LogicalTypeId::INTEGER: 107 | return AggregateFunction::BinaryAggregate, 108 | int32_t, int64_t, int32_t, 109 | FirstScroogeOperation>( 110 | type, timestamp_type, type); 111 | case LogicalTypeId::BIGINT: 112 | return AggregateFunction::BinaryAggregate, 113 | int64_t, int64_t, int64_t, 114 | FirstScroogeOperation>( 115 | type, timestamp_type, type); 116 | case LogicalTypeId::DECIMAL: { 117 | auto decimal_aggregate = 118 | AggregateFunction::BinaryAggregate, 119 | hugeint_t, int64_t, hugeint_t, 120 | FirstScroogeOperation>( 121 | type, timestamp_type, type); 122 | decimal_aggregate.bind = BindDoubleFirst; 123 | return decimal_aggregate; 124 | } 125 | case LogicalTypeId::FLOAT: 126 | return AggregateFunction::BinaryAggregate< 127 | FirstScroogeState, float, int64_t, float, FirstScroogeOperation>( 128 | type, timestamp_type, type); 129 | case LogicalTypeId::DOUBLE: 130 | return AggregateFunction::BinaryAggregate, double, 131 | int64_t, double, 132 | FirstScroogeOperation>( 133 | type, timestamp_type, type); 134 | case LogicalTypeId::UTINYINT: 135 | return AggregateFunction::BinaryAggregate, 136 | uint8_t, int64_t, uint8_t, 137 | FirstScroogeOperation>( 138 | type, timestamp_type, type); 139 | case LogicalTypeId::USMALLINT: 140 | return AggregateFunction::BinaryAggregate, 141 | uint16_t, int64_t, uint16_t, 142 | FirstScroogeOperation>( 143 | type, timestamp_type, type); 144 | case LogicalTypeId::UINTEGER: 145 | return AggregateFunction::BinaryAggregate, 146 | uint32_t, int64_t, uint32_t, 147 | FirstScroogeOperation>( 148 | type, timestamp_type, type); 149 | case LogicalTypeId::UBIGINT: 150 | return AggregateFunction::BinaryAggregate, 151 | uint64_t, int64_t, uint64_t, 152 | FirstScroogeOperation>( 153 | type, timestamp_type, type); 154 | case LogicalTypeId::HUGEINT: 155 | return AggregateFunction::BinaryAggregate, 156 | hugeint_t, int64_t, hugeint_t, 157 | FirstScroogeOperation>( 158 | type, timestamp_type, type); 159 | case LogicalTypeId::UHUGEINT: 160 | return AggregateFunction::BinaryAggregate, 161 | uhugeint_t, int64_t, uhugeint_t, 162 | FirstScroogeOperation>( 163 | type, timestamp_type, type); 164 | default: 165 | throw InternalException( 166 | "Scrooge First Function only accept Numeric Inputs"); 167 | } 168 | } 169 | 170 | void FirstScrooge::RegisterFunction(Connection &conn, Catalog &catalog) { 171 | // The first aggregate allows you to get the first value of one column as 172 | // ordered by another e.g., first(temperature, time) returns the earliest 173 | // temperature value based on time within an aggregate group. 174 | AggregateFunctionSet first("first_s"); 175 | for (auto &type : LogicalType::Numeric()) { 176 | first.AddFunction(GetFirstScroogeFunction(LogicalType::TIMESTAMP_TZ, type)); 177 | first.AddFunction(GetFirstScroogeFunction(LogicalType::TIMESTAMP, type)); 178 | } 179 | CreateAggregateFunctionInfo first_info(first); 180 | catalog.CreateFunction(*conn.context, first_info); 181 | } 182 | 183 | } // namespace scrooge 184 | } // namespace duckdb -------------------------------------------------------------------------------- /src/functions/last.cpp: -------------------------------------------------------------------------------- 1 | #include "functions/functions.hpp" 2 | #include "duckdb/function/function_set.hpp" 3 | #include "duckdb/parser/parsed_data/create_aggregate_function_info.hpp" 4 | #include "duckdb/common/helper.hpp" 5 | 6 | namespace duckdb { 7 | namespace scrooge { 8 | 9 | template struct LastScroogeState { 10 | T last; 11 | int64_t last_time; 12 | bool executed; 13 | }; 14 | 15 | struct LastScroogeOperation { 16 | template static void Initialize(STATE &state) { 17 | state.last_time = NumericLimits::Minimum(); 18 | state.executed = false; 19 | } 20 | 21 | template 22 | static void Operation(STATE &state, const A_TYPE &x_data, 23 | const B_TYPE &y_data, AggregateBinaryInput &idata) { 24 | 25 | const auto time = y_data; 26 | if (!state.executed || time > state.last_time) { 27 | state.last_time = time; 28 | state.last = x_data; 29 | state.executed = true; 30 | } 31 | } 32 | 33 | template 34 | static void Combine(const STATE &source, STATE &target, 35 | AggregateInputData &aggr_input_data) { 36 | if (!target.executed) { 37 | target = source; 38 | } else if (source.executed) { 39 | if (target.last_time > source.last_time) { 40 | target.last_time = source.last_time; 41 | target.last = source.last; 42 | } 43 | } 44 | } 45 | 46 | template 47 | static void Finalize(STATE &state, T &target, 48 | AggregateFinalizeData &finalize_data) { 49 | if (!state.executed) { 50 | finalize_data.ReturnNull(); 51 | } else { 52 | target = state.last; 53 | } 54 | } 55 | 56 | static bool IgnoreNull() { return true; } 57 | }; 58 | 59 | unique_ptr 60 | BindDoupleLastFunctionDecimal(ClientContext &context, 61 | AggregateFunction &function, 62 | vector> &arguments) { 63 | auto &decimal_type = arguments[0]->return_type; 64 | switch (decimal_type.InternalType()) { 65 | case PhysicalType::INT16: { 66 | function = AggregateFunction::BinaryAggregate, 67 | int16_t, int64_t, int16_t, 68 | LastScroogeOperation>( 69 | decimal_type, LogicalType::TIMESTAMP_TZ, decimal_type); 70 | break; 71 | } 72 | case PhysicalType::INT32: { 73 | function = AggregateFunction::BinaryAggregate, 74 | int32_t, int64_t, int32_t, 75 | LastScroogeOperation>( 76 | decimal_type, LogicalType::TIMESTAMP_TZ, decimal_type); 77 | break; 78 | } 79 | case PhysicalType::INT64: { 80 | function = AggregateFunction::BinaryAggregate, 81 | int64_t, int64_t, int64_t, 82 | LastScroogeOperation>( 83 | decimal_type, LogicalType::TIMESTAMP_TZ, decimal_type); 84 | break; 85 | } 86 | default: 87 | function = AggregateFunction::BinaryAggregate, 88 | Hugeint, int64_t, Hugeint, 89 | LastScroogeOperation>( 90 | decimal_type, LogicalType::TIMESTAMP_TZ, decimal_type); 91 | } 92 | function.name = "last_s"; 93 | return nullptr; 94 | } 95 | 96 | AggregateFunction GetLastScroogeFunction(const LogicalType ×tamp_type, 97 | const LogicalType &type) { 98 | switch (type.id()) { 99 | case LogicalTypeId::SMALLINT: 100 | return AggregateFunction::BinaryAggregate, 101 | int16_t, int64_t, int16_t, 102 | LastScroogeOperation>( 103 | type, timestamp_type, type); 104 | case LogicalTypeId::TINYINT: 105 | return AggregateFunction::BinaryAggregate, int8_t, 106 | int64_t, int8_t, 107 | LastScroogeOperation>( 108 | type, timestamp_type, type); 109 | case LogicalTypeId::INTEGER: 110 | return AggregateFunction::BinaryAggregate, 111 | int32_t, int64_t, int32_t, 112 | LastScroogeOperation>( 113 | type, timestamp_type, type); 114 | case LogicalTypeId::BIGINT: 115 | return AggregateFunction::BinaryAggregate, 116 | int64_t, int64_t, int64_t, 117 | LastScroogeOperation>( 118 | type, timestamp_type, type); 119 | case LogicalTypeId::DECIMAL: { 120 | auto decimal_aggregate = 121 | AggregateFunction::BinaryAggregate, 122 | hugeint_t, int64_t, hugeint_t, 123 | LastScroogeOperation>( 124 | type, timestamp_type, type); 125 | decimal_aggregate.bind = BindDoupleLastFunctionDecimal; 126 | return decimal_aggregate; 127 | } 128 | // corr.AddFunction(AggregateFunction::BinaryAggregate( 130 | // LogicalType::DOUBLE, LogicalType::DOUBLE, LogicalType::DOUBLE)); 131 | case LogicalTypeId::FLOAT: 132 | return AggregateFunction::BinaryAggregate< 133 | LastScroogeState, float, int64_t, float, LastScroogeOperation>( 134 | type, timestamp_type, type); 135 | case LogicalTypeId::DOUBLE: 136 | return AggregateFunction::BinaryAggregate, double, 137 | int64_t, double, 138 | LastScroogeOperation>( 139 | type, timestamp_type, type); 140 | case LogicalTypeId::UTINYINT: 141 | return AggregateFunction::BinaryAggregate, 142 | uint8_t, int64_t, uint8_t, 143 | LastScroogeOperation>( 144 | type, timestamp_type, type); 145 | case LogicalTypeId::USMALLINT: 146 | return AggregateFunction::BinaryAggregate, 147 | uint16_t, int64_t, uint16_t, 148 | LastScroogeOperation>( 149 | type, timestamp_type, type); 150 | case LogicalTypeId::UINTEGER: 151 | return AggregateFunction::BinaryAggregate, 152 | uint32_t, int64_t, uint32_t, 153 | LastScroogeOperation>( 154 | type, timestamp_type, type); 155 | case LogicalTypeId::UBIGINT: 156 | return AggregateFunction::BinaryAggregate, 157 | uint64_t, int64_t, uint64_t, 158 | LastScroogeOperation>( 159 | type, timestamp_type, type); 160 | case LogicalTypeId::HUGEINT: 161 | return AggregateFunction::BinaryAggregate, 162 | hugeint_t, int64_t, hugeint_t, 163 | LastScroogeOperation>( 164 | type, timestamp_type, type); 165 | case LogicalTypeId::UHUGEINT: 166 | return AggregateFunction::BinaryAggregate, 167 | hugeint_t, int64_t, hugeint_t, 168 | LastScroogeOperation>( 169 | type, timestamp_type, type); 170 | default: 171 | throw InternalException( 172 | "Scrooge First Function only accept Numeric Inputs"); 173 | } 174 | } 175 | 176 | void LastScrooge::RegisterFunction(Connection &conn, Catalog &catalog) { 177 | // The last aggregate allows you to get the value of one column as ordered by 178 | // another. For example, last(temperature, time) returns the latest 179 | // temperature value based on time within an aggregate group. 180 | 181 | AggregateFunctionSet last("last_s"); 182 | for (auto &type : LogicalType::Numeric()) { 183 | last.AddFunction(GetLastScroogeFunction(LogicalType::TIMESTAMP_TZ, type)); 184 | last.AddFunction(GetLastScroogeFunction(LogicalType::TIMESTAMP, type)); 185 | } 186 | CreateAggregateFunctionInfo last_info(last); 187 | catalog.CreateFunction(*conn.context, last_info); 188 | } 189 | 190 | } // namespace scrooge 191 | } // namespace duckdb -------------------------------------------------------------------------------- /src/functions/portfolio_frontier.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "functions/scanner.hpp" 4 | #include "duckdb/execution/operator/csv_scanner/csv_reader_options.hpp" 5 | #include "duckdb/main/relation/read_csv_relation.hpp" 6 | #include "duckdb/main/relation/projection_relation.hpp" 7 | #include "duckdb/parser/expression/star_expression.hpp" 8 | #include "duckdb/parser/expression/constant_expression.hpp" 9 | #include "duckdb/main/relation/aggregate_relation.hpp" 10 | #include "duckdb/parser/expression/function_expression.hpp" 11 | #include "duckdb/common/helper.hpp" 12 | 13 | namespace duckdb { 14 | namespace scrooge { 15 | 16 | struct Asset { 17 | string symbol; 18 | double volatility; 19 | double expected_return; 20 | }; 21 | 22 | struct Portfolio { 23 | vector assets; 24 | vector> weights; 25 | }; 26 | 27 | double calculate_portfolio_return(Portfolio &portfolio) { 28 | double portfolio_return = 0.0; 29 | for (int i = 0; i < portfolio.assets.size(); i++) { 30 | portfolio_return += 31 | portfolio.weights.back()[i] * portfolio.assets[i].expected_return; 32 | } 33 | return portfolio_return; 34 | } 35 | 36 | double calculate_portfolio_volatility(Portfolio &portfolio) { 37 | double portfolio_volatility = 0.0; 38 | for (int i = 0; i < portfolio.assets.size(); i++) { 39 | portfolio_volatility += 40 | pow(portfolio.weights.back()[i] * portfolio.assets[i].volatility, 2.0); 41 | } 42 | return sqrt(portfolio_volatility); 43 | } 44 | 45 | vector generate_random_weights(int n) { 46 | vector weights(n); 47 | double sum = 0.0; 48 | for (int i = 0; i < n; i++) { 49 | weights[i] = (double)rand() / RAND_MAX; 50 | sum += weights[i]; 51 | } 52 | for (int i = 0; i < n; i++) { 53 | weights[i] /= sum; 54 | } 55 | return weights; 56 | } 57 | 58 | vector> calculate_efficient_frontier(Portfolio &portfolio, 59 | int n) { 60 | vector> efficient_frontier(n); 61 | for (int i = 0; i < n; i++) { 62 | portfolio.weights.emplace_back( 63 | generate_random_weights(portfolio.assets.size())); 64 | double portfolio_return = calculate_portfolio_return(portfolio); 65 | double portfolio_volatility = calculate_portfolio_volatility(portfolio); 66 | efficient_frontier[i] = make_pair(portfolio_volatility, portfolio_return); 67 | } 68 | // sort(efficient_frontier.begin(), efficient_frontier.end()); 69 | return efficient_frontier; 70 | } 71 | 72 | struct PortfolioFrontierData : public TableFunctionData { 73 | Portfolio portfolio; 74 | vector> portfolio_stats; 75 | int n; 76 | int cur = 0; 77 | }; 78 | 79 | unique_ptr 80 | PortfolioFrontier::Bind(ClientContext &context, TableFunctionBindInput &input, 81 | vector &return_types, 82 | vector &names) { 83 | auto result = make_uniq(); 84 | auto conn = make_uniq(*context.db); 85 | if (input.inputs[1].type() != LogicalType::VARCHAR && 86 | input.inputs[1].type() != LogicalType::DATE) { 87 | throw InvalidInputException( 88 | "Start Period must be a Date or a Date-VARCHAR "); 89 | } 90 | if (input.inputs[2].type() != LogicalType::VARCHAR && 91 | input.inputs[2].type() != LogicalType::DATE) { 92 | throw InvalidInputException( 93 | "Start Period must be a Date or a Date-VARCHAR "); 94 | } 95 | result->n = input.inputs[3].GetValue(); 96 | vector parameters{input.inputs[0], input.inputs[1], input.inputs[2], 97 | "1d"}; 98 | 99 | auto tbl_rel = make_shared_ptr( 100 | conn->context, "yahoo_finance", std::move(parameters)); 101 | vector> expressions; 102 | vector> groups; 103 | auto group_column = make_uniq("symbol"); 104 | 105 | auto value_column = make_uniq("Adj Close"); 106 | vector> children; 107 | children.emplace_back(std::move(value_column)); 108 | auto volatility = 109 | make_uniq("stddev_pop", std::move(children)); 110 | 111 | auto date_column = make_uniq("Date"); 112 | value_column = make_uniq("Adj Close"); 113 | vector> children_min; 114 | children_min.emplace_back(std::move(value_column)); 115 | children_min.emplace_back(std::move(date_column)); 116 | auto arg_min = 117 | make_uniq("arg_min", std::move(children_min)); 118 | 119 | date_column = make_uniq("Date"); 120 | value_column = make_uniq("Adj Close"); 121 | vector> children_min_2; 122 | children_min_2.emplace_back(std::move(value_column)); 123 | children_min_2.emplace_back(std::move(date_column)); 124 | auto arg_min_2 = 125 | make_uniq("arg_min", std::move(children_min_2)); 126 | 127 | date_column = make_uniq("Date"); 128 | value_column = make_uniq("Adj Close"); 129 | vector> children_max; 130 | children_max.emplace_back(std::move(value_column)); 131 | children_max.emplace_back(std::move(date_column)); 132 | auto arg_max = 133 | make_uniq("arg_max", std::move(children_max)); 134 | 135 | vector> substract_children; 136 | substract_children.emplace_back(std::move(arg_max)); 137 | substract_children.emplace_back(std::move(arg_min)); 138 | auto subtract = 139 | make_uniq("-", std::move(substract_children)); 140 | 141 | vector> expected_return_children; 142 | expected_return_children.emplace_back(std::move(subtract)); 143 | expected_return_children.emplace_back(std::move(arg_min_2)); 144 | auto expected_return = 145 | make_uniq("/", std::move(expected_return_children)); 146 | auto symbol_column = make_uniq("symbol"); 147 | vector> aggr_expression; 148 | aggr_expression.emplace_back(std::move(symbol_column)); 149 | aggr_expression.emplace_back(std::move(volatility)); 150 | aggr_expression.emplace_back(std::move(expected_return)); 151 | auto aggr_rel = make_shared_ptr( 152 | tbl_rel, std::move(aggr_expression), std::move(groups)); 153 | auto plan = std::move(aggr_rel); 154 | 155 | child_list_t children_struct; 156 | children_struct.emplace_back(make_pair("symbol", LogicalType::VARCHAR)); 157 | children_struct.emplace_back(make_pair("weight", LogicalType::DOUBLE)); 158 | return_types.emplace_back( 159 | LogicalType::LIST(LogicalType::STRUCT(children_struct))); 160 | return_types.emplace_back(LogicalType::DOUBLE); 161 | return_types.emplace_back(LogicalType::DOUBLE); 162 | 163 | names.emplace_back("Portfolio"); 164 | names.emplace_back("ExpectedReturn"); 165 | names.emplace_back("Volatility"); 166 | 167 | auto res = plan->Execute(); 168 | auto result_chunk = res->Fetch(); 169 | while (result_chunk) { 170 | for (idx_t i = 0; i < result_chunk->size(); i++) { 171 | Asset asset; 172 | asset.symbol = result_chunk->data[0].GetValue(i).GetValueUnsafe(); 173 | asset.volatility = 174 | result_chunk->data[1].GetValue(i).GetValueUnsafe(); 175 | asset.expected_return = 176 | result_chunk->data[2].GetValue(i).GetValueUnsafe(); 177 | result->portfolio.assets.emplace_back(asset); 178 | } 179 | result_chunk = res->Fetch(); 180 | } 181 | result->portfolio_stats = 182 | calculate_efficient_frontier(result->portfolio, result->n); 183 | return std::move(result); 184 | } 185 | void PortfolioFrontier::Scan(ClientContext &context, TableFunctionInput &data_p, 186 | DataChunk &output) { 187 | 188 | auto &data = (PortfolioFrontierData &)*data_p.bind_data; 189 | idx_t cur_out = 0; 190 | for (; data.cur < data.portfolio_stats.size(); data.cur++) { 191 | if (cur_out == STANDARD_VECTOR_SIZE) { 192 | break; 193 | } 194 | vector list; 195 | for (idx_t j = 0; j < data.portfolio.assets.size(); j++) { 196 | child_list_t children_struct; 197 | children_struct.emplace_back( 198 | make_pair("symbol", data.portfolio.assets[j].symbol)); 199 | children_struct.emplace_back( 200 | make_pair("weight", data.portfolio.weights[data.cur][j])); 201 | list.emplace_back(Value::STRUCT(children_struct)); 202 | } 203 | output.SetValue(0, data.cur, Value::LIST(list)); 204 | output.SetValue(1, data.cur, data.portfolio_stats[data.cur].second); 205 | output.SetValue(2, data.cur, data.portfolio_stats[data.cur].first); 206 | cur_out++; 207 | } 208 | output.SetCardinality(cur_out); 209 | } 210 | } // namespace scrooge 211 | } // namespace duckdb -------------------------------------------------------------------------------- /src/functions/timebucket.cpp: -------------------------------------------------------------------------------- 1 | #include "duckdb/function/function_set.hpp" 2 | #include "duckdb/parser/parsed_data/create_scalar_function_info.hpp" 3 | #include "functions/functions.hpp" 4 | #include "duckdb/common/helper.hpp" 5 | 6 | namespace duckdb { 7 | namespace scrooge { 8 | 9 | void TimeBucketFunction(DataChunk &args, ExpressionState &state, 10 | Vector &result) { 11 | D_ASSERT(args.ColumnCount() == 2); 12 | auto ×tamp_vector = args.data[0]; 13 | auto interval_value = args.data[1].GetValue(0); 14 | if (interval_value.IsNull()) { 15 | throw std::runtime_error("Timebucket interval can't be null"); 16 | } 17 | auto interval = Interval::GetMicro(interval_value.GetValue()); 18 | if (timestamp_vector.GetVectorType() == VectorType::CONSTANT_VECTOR) { 19 | result.SetVectorType(VectorType::CONSTANT_VECTOR); 20 | if (ConstantVector::IsNull(timestamp_vector)) { 21 | ConstantVector::SetNull(result, true); 22 | } else { 23 | auto timestamp_ptr = 24 | ConstantVector::GetData(timestamp_vector); 25 | result.SetValue(0, timestamp_ptr[0].value - 26 | (timestamp_ptr[0].value % interval)); 27 | } 28 | } 29 | if (timestamp_vector.GetVectorType() == VectorType::FLAT_VECTOR) { 30 | auto timestamp_ptr = FlatVector::GetData(timestamp_vector); 31 | auto timestamp_validity = FlatVector::Validity(timestamp_vector); 32 | 33 | auto result_ptr = FlatVector::GetData(result); 34 | if (timestamp_validity.AllValid()) { 35 | for (idx_t i = 0; i < args.size(); i++) { 36 | result_ptr[i] = 37 | timestamp_ptr[i].value - (timestamp_ptr[i].value % interval); 38 | } 39 | } else { 40 | auto &result_validity = FlatVector::Validity(result); 41 | for (idx_t i = 0; i < args.size(); i++) { 42 | if (timestamp_validity.RowIsValid(i)) { 43 | result_ptr[i] = 44 | timestamp_ptr[i].value - (timestamp_ptr[i].value % interval); 45 | } else { 46 | result_validity.SetInvalid(i); 47 | } 48 | } 49 | } 50 | } else { 51 | UnifiedVectorFormat timestamp_data; 52 | timestamp_vector.ToUnifiedFormat(args.size(), timestamp_data); 53 | auto timestamp_ptr = (const timestamp_t *)timestamp_data.data; 54 | 55 | auto result_ptr = FlatVector::GetData(result); 56 | 57 | if (timestamp_data.validity.AllValid()) { 58 | for (idx_t i = 0; i < args.size(); ++i) { 59 | const auto idx = timestamp_data.sel->get_index(i); 60 | result_ptr[i] = 61 | timestamp_ptr[idx].value - (timestamp_ptr[idx].value % interval); 62 | } 63 | } else { 64 | auto &result_validity = FlatVector::Validity(result); 65 | for (idx_t i = 0; i < args.size(); i++) { 66 | if (timestamp_data.validity.RowIsValid(i)) { 67 | result_ptr[i] = 68 | timestamp_ptr[i].value - (timestamp_ptr[i].value % interval); 69 | } else { 70 | result_validity.SetInvalid(i); 71 | } 72 | } 73 | } 74 | } 75 | } 76 | 77 | void TimeBucketScrooge::RegisterFunction(Connection &conn, Catalog &catalog) { 78 | // The time_bucket function is similar to the standard PostgreSQL date_trunc 79 | // function. Unlike date_trunc, it allows for arbitrary time intervals 80 | // instead of second, minute, and hour intervals. The return value is the 81 | // bucket's start time. 82 | 83 | ScalarFunctionSet timebucket("timebucket"); 84 | timebucket.AddFunction( 85 | ScalarFunction({LogicalType::TIMESTAMP_TZ, LogicalType::INTERVAL}, 86 | LogicalType::TIMESTAMP_TZ, TimeBucketFunction)); 87 | timebucket.AddFunction( 88 | ScalarFunction({LogicalType::TIMESTAMP, LogicalType::INTERVAL}, 89 | LogicalType::TIMESTAMP, TimeBucketFunction)); 90 | CreateScalarFunctionInfo timebucket_info(timebucket); 91 | catalog.CreateFunction(*conn.context, timebucket_info); 92 | } 93 | } // namespace scrooge 94 | } // namespace duckdb -------------------------------------------------------------------------------- /src/include/functions/functions.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // Scrooge 3 | // 4 | // functions/functions.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | #include "duckdb.hpp" 11 | #include "duckdb/function/aggregate_function.hpp" 12 | 13 | namespace duckdb { 14 | namespace scrooge { 15 | 16 | struct FirstScrooge { 17 | static void RegisterFunction(Connection &conn, Catalog &catalog); 18 | }; 19 | 20 | struct LastScrooge { 21 | static void RegisterFunction(Connection &conn, Catalog &catalog); 22 | }; 23 | 24 | struct TimeBucketScrooge { 25 | static void RegisterFunction(Connection &conn, Catalog &catalog); 26 | }; 27 | 28 | struct Aliases { 29 | static void Register(Connection &conn, Catalog &catalog); 30 | }; 31 | 32 | } // namespace scrooge 33 | } // namespace duckdb 34 | -------------------------------------------------------------------------------- /src/include/functions/scanner.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // Scrooge 3 | // 4 | // functions/scanner.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb.hpp" 12 | 13 | #define CPPHTTPLIB_OPENSSL_SUPPORT 14 | #include "httplib.hpp" 15 | 16 | namespace duckdb { 17 | namespace scrooge { 18 | struct YahooScanner { 19 | static unique_ptr Bind(ClientContext &context, 20 | TableFunctionBindInput &input, 21 | vector &return_types, 22 | vector &names); 23 | static void Scan(ClientContext &context, TableFunctionInput &data_p, 24 | DataChunk &output); 25 | }; 26 | 27 | struct PortfolioFrontier { 28 | static unique_ptr Bind(ClientContext &context, 29 | TableFunctionBindInput &input, 30 | vector &return_types, 31 | vector &names); 32 | static void Scan(ClientContext &context, TableFunctionInput &data_p, 33 | DataChunk &output); 34 | }; 35 | 36 | struct EthRPC { 37 | static unique_ptr Bind(ClientContext &context, 38 | TableFunctionBindInput &input, 39 | vector &return_types, 40 | vector &names); 41 | static void Scan(ClientContext &context, TableFunctionInput &data_p, 42 | DataChunk &output); 43 | static unique_ptr 44 | 45 | InitLocal(ExecutionContext &context, TableFunctionInitInput &input, 46 | GlobalTableFunctionState *global_state_p); 47 | 48 | static unique_ptr 49 | InitGlobal(ClientContext &context, TableFunctionInitInput &input); 50 | 51 | static double ProgressBar(ClientContext &context, 52 | const FunctionData *bind_data_p, 53 | const GlobalTableFunctionState *global_state); 54 | }; 55 | } // namespace scrooge 56 | 57 | } // namespace duckdb 58 | -------------------------------------------------------------------------------- /src/include/scrooge_extension.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "duckdb.hpp" 4 | 5 | namespace duckdb { 6 | 7 | class ScroogeExtension : public Extension { 8 | public: 9 | void Load(DuckDB &db) override; 10 | std::string Name() override; 11 | }; 12 | 13 | } // namespace duckdb 14 | -------------------------------------------------------------------------------- /src/include/util/eth_maps.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // Scrooge 3 | // 4 | // util/eth_maps.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "util/eth_tokens_map.hpp" 12 | 13 | namespace duckdb { 14 | namespace scrooge { 15 | // Event data structure 16 | struct Event { 17 | string name; 18 | vector parameterTypes; 19 | uint8_t id; 20 | }; 21 | 22 | // Known event signatures and their descriptions 23 | const unordered_map event_signatures = { 24 | {"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", 25 | {"Transfer", {"address", "address", "uint256"}, 0}}, 26 | {"0x8c5be1e5ebec7d5bd14f714f0f3a56d4af4fef1d7f7bb78a0c69d4cdb365d97e", 27 | {"Approval", {"address", "address", "uint256"}, 1}}, 28 | {"0x1c411e9a96e071241c2f21f7726b17ae89e3cab4c78be50e062b03a9fffbbad1", 29 | {"Sync", {"uint112", "uint112"}, 2}}, 30 | {"0x4a39dc06d4c0dbc64b70b7bfa42387d156a1b455ad1583a19957f547256c22e5", 31 | {"TransferSingle", 32 | {"address", "address", "address", "uint256", "uint256"}, 33 | 3}}, 34 | {"0xc3d58168cbe0a160b82265397fa6b10ff1c847f6b8df03e1d36e5e4bba925ed5", 35 | {"TransferBatch", 36 | {"address", "address", "address", "uint256[]", "uint256[]"}, 37 | 4}}, 38 | {"0x17307eab39c17eae00a0c514c4b17d95e15ee86d924b1cf3b9c9dc7f59e3e5a1", 39 | {"ApprovalForAll", {"address", "address", "bool"}, 5}}}; 40 | 41 | const unordered_map event_to_hex_signatures = { 42 | {"TRANSFER", 43 | "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"}, 44 | {"APPROVAL", 45 | "0x8c5be1e5ebec7d5bd14f714f13107d2b7b6e6d2027d8710d3b0b4f43363d9ab8"}, 46 | {"DEPOSIT", 47 | "0xe1fffcc4923d54a4bdb6e0a28d0b6b7b2fb29a260555b24c75f6b1e7b3bfb123"}, 48 | {"WITHDRAWAL", 49 | "0x7fcf26fc5cc6d7e6e05e1144b4b68871c514e020f1fc0d7d839bd09f61a8bc86"}, 50 | {"SYNC", 51 | "0x1c411e9a96e071241c2f21f7726b17ae89e3cab4c78be50e062b03a9fffbbad1"}}; 52 | 53 | const unordered_map event_to_enum = { 54 | {"Transfer", 0}, 55 | {"Approval", 1}, 56 | {"Sync", 2}, 57 | {"Transfer", 0}, 58 | }; 59 | 60 | const vector event_strings = {"TRANSFER", "SYNC"}; 61 | } // namespace scrooge 62 | } // namespace duckdb 63 | -------------------------------------------------------------------------------- /src/include/util/hex_converter.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // Scrooge 3 | // 4 | // util/hex_converter.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | namespace duckdb { 12 | namespace scrooge { 13 | 14 | class HexConverter { 15 | public: 16 | static uint64_t HexToUBigInt(const string &hex_str) { 17 | return std::stoull(hex_str, nullptr, 16); 18 | } 19 | static bool HexToUhugeiInt(const std::string &hex, bool strict, 20 | uhugeint_t &result) { 21 | std::size_t firstNonZero = hex.find_first_not_of('0'); 22 | if (hex.size() - firstNonZero > 32) { 23 | if (strict) { 24 | throw NotImplementedException( 25 | "This number requires a uint256, that is not yet supported"); 26 | } 27 | return false; 28 | } 29 | std::size_t trim_size = 32 - (hex.size() - firstNonZero); 30 | std::string paddedHex(trim_size, '0'); 31 | paddedHex += hex.substr(firstNonZero); 32 | // Split the padded hex string into upper and lower parts 33 | std::string upperHex = paddedHex.substr(0, 16); 34 | std::string lowerHex = paddedHex.substr(16, 16); 35 | 36 | // Convert the hex parts to uint64_t 37 | result.upper = HexToUBigInt(upperHex); 38 | result.lower = HexToUBigInt(lowerHex); 39 | return true; 40 | } 41 | 42 | template static string NumericToHex(T value) { 43 | std::stringstream stream_to; 44 | stream_to << "0x" << std::hex << value; 45 | return stream_to.str(); 46 | } 47 | 48 | static bool IsHex(const string &hex) { 49 | return hex.size() >= 2 && hex.substr(0, 2) == "0x"; 50 | } 51 | }; 52 | } // namespace scrooge 53 | } // namespace duckdb 54 | -------------------------------------------------------------------------------- /src/include/util/http_util.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // Scrooge 3 | // 4 | // util/http_util.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include 12 | 13 | namespace duckdb { 14 | namespace scrooge { 15 | 16 | class HTTPUtil { 17 | public: 18 | static duckdb_httplib_openssl::Result Request(const std::string &url, 19 | const std::string &request) { 20 | std::string host, path; 21 | bool use_ssl = false; 22 | string clean_url = url; 23 | // Find the protocol 24 | size_t protocol_end = url.find("://"); 25 | if (protocol_end != std::string::npos) { 26 | std::string protocol = url.substr(0, protocol_end); 27 | if (protocol == "https") { 28 | use_ssl = true; 29 | } else if (protocol != "http") { 30 | throw std::runtime_error("Unsupported protocol: " + protocol); 31 | } 32 | clean_url = url.substr(protocol_end + 3); 33 | } 34 | 35 | // Split the host and path 36 | size_t path_start = clean_url.find('/'); 37 | if (path_start != std::string::npos) { 38 | host = clean_url.substr(0, path_start); 39 | path = clean_url.substr(path_start); 40 | } else { 41 | host = clean_url; 42 | path = "/"; 43 | } 44 | 45 | // Create an HTTP or HTTPS client based on the protocol 46 | // std::unique_ptr client; 47 | if (use_ssl) { 48 | auto client = make_uniq(host); 49 | // Perform the HTTP POST request 50 | auto res = client->Post(path.c_str(), request, "application/json"); 51 | 52 | // Check if the request was successful 53 | if (!res || res->status != 200) { 54 | throw std::runtime_error("HTTP Request failed with status: " + 55 | std::to_string(res ? res->status : -1)); 56 | } 57 | return res; 58 | // dynamic_cast(client.get())->enable_server_certificate_verification(false); 59 | } else { 60 | auto client = make_uniq(host); 61 | // Perform the HTTP POST request 62 | auto res = client->Post(path.c_str(), request, "application/json"); 63 | 64 | // Check if the request was successful 65 | if (!res || res->status != 200) { 66 | throw std::runtime_error("HTTP Request failed with status: " + 67 | std::to_string(res ? res->status : -1)); 68 | } 69 | return res; 70 | } 71 | } 72 | }; 73 | 74 | } // namespace scrooge 75 | } // namespace duckdb 76 | -------------------------------------------------------------------------------- /src/scanner/ethereum_blockchain.cpp: -------------------------------------------------------------------------------- 1 | #include "functions/scanner.hpp" 2 | #include "duckdb/common/helper.hpp" 3 | #include 4 | #include 5 | #include 6 | #include "json.hpp" 7 | #include "util/hex_converter.hpp" 8 | #include "util/eth_maps.hpp" 9 | #include "util/http_util.hpp" 10 | #include "util/eth_uniswap_map.hpp" 11 | 12 | namespace duckdb { 13 | namespace scrooge { 14 | class EthGetLogsRequest : public TableFunctionData { 15 | public: 16 | // Constructor to initialize the JSON-RPC request with given parameters 17 | EthGetLogsRequest(uint32_t id, string address, string topic, 18 | int64_t from_block_p, int64_t to_block_p, 19 | int64_t blocks_per_thread_p, string rpc_url_p, 20 | const bool strict_p) 21 | : id(id), address(std::move(address)), topic(std::move(topic)), 22 | from_block(from_block_p), to_block(to_block_p), 23 | blocks_per_thread(blocks_per_thread_p), rpc_url(std::move(rpc_url_p)), 24 | strict((strict_p)) {} 25 | 26 | const uint32_t id; 27 | const string address; 28 | const string topic; 29 | const idx_t from_block; 30 | const idx_t to_block; 31 | const int64_t blocks_per_thread; 32 | const string rpc_url; 33 | const bool strict; 34 | }; 35 | 36 | unique_ptr EthRPC::Bind(ClientContext &context, 37 | TableFunctionBindInput &input, 38 | vector &return_types, 39 | vector &names) { 40 | // Get Arguments 41 | auto address = input.inputs[0].GetValue(); 42 | auto topic = input.inputs[1].GetValue(); 43 | auto from_block = input.inputs[2].GetValue(); 44 | auto to_block = input.inputs[3].GetValue(); 45 | int64_t blocks_per_thread = -1; 46 | bool strict = false; 47 | 48 | for (auto &kv : input.named_parameters) { 49 | auto loption = StringUtil::Lower(kv.first); 50 | if (loption == "blocks_per_thread") { 51 | blocks_per_thread = kv.second.GetValue(); 52 | if (blocks_per_thread < -1 || blocks_per_thread == 0) { 53 | throw InvalidInputException( 54 | "blocks_per_thread must be higher than 0 or equal to -1. -1 means " 55 | "one thread will read all the blocks"); 56 | } 57 | } else if (loption == "strict") { 58 | strict = kv.second.GetValue(); 59 | } else { 60 | throw BinderException( 61 | "Unrecognized function name \"%s\" for read_eth definition", 62 | kv.first); 63 | } 64 | } 65 | 66 | if (from_block < 0) { 67 | throw InvalidInputException("FromBlock must be higher or equal to 0"); 68 | } 69 | 70 | if (!(address.size() >= 2 && address.substr(0, 2) == "0x")) { 71 | transform(address.begin(), address.end(), address.begin(), ::toupper); 72 | if (token_addresses.find(address) == token_addresses.end()) { 73 | if (uniswap_addresses.find(address) == uniswap_addresses.end()) { 74 | vector candidates; 75 | if (address.size() < 8) { 76 | candidates = StringUtil::TopNLevenshtein(token_symbols, address); 77 | } else { 78 | candidates = StringUtil::TopNLevenshtein(uniswap_symbols, address); 79 | } 80 | std::ostringstream error; 81 | error << "Failed to infer the address \"" << address << "\""; 82 | error << ". Address must be either a hex (e.g., 0x...) or a valid " 83 | "token symbol.\n"; 84 | error << "Suggested token symbols: \n"; 85 | for (idx_t candidate_idx = 0; candidate_idx < candidates.size(); 86 | candidate_idx++) { 87 | error << candidates[candidate_idx]; 88 | if (candidate_idx < candidates.size() - 1) { 89 | error << ", "; 90 | } 91 | } 92 | throw InvalidInputException(error.str()); 93 | } else { 94 | address = uniswap_addresses.at(address); 95 | } 96 | } else { 97 | address = token_addresses.at(address); 98 | } 99 | } 100 | 101 | if (!(topic.size() >= 2 && topic.substr(0, 2) == "0x")) { 102 | transform(topic.begin(), topic.end(), topic.begin(), ::toupper); 103 | if (event_to_hex_signatures.find(topic) == event_to_hex_signatures.end()) { 104 | std::ostringstream error; 105 | error << "Failed to infer the topic address \"" << topic << "\""; 106 | error << ". Address must be either a hex (e.g., 0x...) or a valid topic " 107 | "string.\n"; 108 | error << "Suggested topic: " 109 | << StringUtil::TopNLevenshtein(event_strings, topic, 1).back(); 110 | 111 | throw InvalidInputException(error.str()); 112 | } 113 | topic = event_to_hex_signatures.at(topic); 114 | } 115 | 116 | // address: Contract address emitting the log. 117 | return_types.emplace_back(LogicalType::VARCHAR); 118 | names.emplace_back("address"); 119 | // event_type: Contract address emitting the log. 120 | string enum_name = "ETH_EVENT"; 121 | Vector order_errors(LogicalType::VARCHAR, 7); 122 | order_errors.SetValue(0, "Transfer"); 123 | order_errors.SetValue(1, "Approval"); 124 | order_errors.SetValue(2, "Sync"); 125 | order_errors.SetValue(3, "TransferSingle"); 126 | order_errors.SetValue(4, "TransferBatch"); 127 | order_errors.SetValue(5, "ApprovalForAll"); 128 | order_errors.SetValue(6, "Unknown"); 129 | LogicalType enum_type = LogicalType::ENUM(enum_name, order_errors, 7); 130 | return_types.emplace_back(enum_type); 131 | names.emplace_back("event_type"); 132 | // blockHash: Hash of the block containing the log. 133 | return_types.emplace_back(LogicalType::VARCHAR); 134 | names.emplace_back("block_hash"); 135 | // blockNumber: Number of the block containing the log. 136 | return_types.emplace_back(LogicalType::INTEGER); 137 | names.emplace_back("block_number"); 138 | // data: Event-specific data (e.g., amount transferred). 139 | return_types.emplace_back(LogicalType::LIST(LogicalType::UHUGEINT)); 140 | names.emplace_back("data"); 141 | // logIndex: Log's position within the block. 142 | return_types.emplace_back(LogicalType::UINTEGER); 143 | names.emplace_back("log_index"); 144 | // removed: Indicates if the log was removed in a chain reorganization. 145 | return_types.emplace_back(LogicalType::BOOLEAN); 146 | names.emplace_back("removed"); 147 | // topics: Indexed event parameters (e.g., event signature, sender, and 148 | // receiver addresses). 149 | return_types.emplace_back(LogicalType::LIST(LogicalType::VARCHAR)); 150 | names.emplace_back("topics"); 151 | // transactionHash: Hash of the transaction generating the log. 152 | return_types.emplace_back(LogicalType::VARCHAR); 153 | names.emplace_back("transaction_hash"); 154 | // transactionIndex: Transaction's position within the block. 155 | return_types.emplace_back(LogicalType::INTEGER); 156 | names.emplace_back("transaction_index"); 157 | 158 | Value result; 159 | string key = "eth_node_url"; 160 | context.TryGetCurrentSetting(key, result); 161 | auto node_url = result.GetValue(); 162 | 163 | return make_uniq(0, address, topic, from_block, to_block, 164 | blocks_per_thread, node_url, strict); 165 | } 166 | 167 | struct CurrentState { 168 | uint32_t start{}; 169 | uint32_t end{}; 170 | }; 171 | 172 | struct RCPRequest { 173 | 174 | explicit RCPRequest(const EthGetLogsRequest &bind_logs_p, idx_t request_id_p, 175 | CurrentState &state_p) 176 | : bind_logs(bind_logs_p), request_id(request_id_p), state(state_p) { 177 | 178 | // Convert the request to a JSON formatted string 179 | std::string request = ToString(); 180 | std::string url = bind_logs.rpc_url; 181 | 182 | // Perform the HTTP POST request 183 | auto res = HTTPUtil::Request(url, request); 184 | 185 | // Parse the response JSON 186 | json = nlohmann::json::parse(res->body); 187 | if (!json.contains("result")) { 188 | // This is funky, we should error 189 | throw std::runtime_error("JSON Error: " + json.dump()); 190 | } 191 | } 192 | // Method to return the JSON request as a string 193 | string ToString() const { 194 | std::ostringstream oss; 195 | oss << "{" 196 | << R"("jsonrpc":"2.0",)" 197 | << "\"id\":" << request_id << "," 198 | << R"("method":"eth_getLogs",)" 199 | << "\"params\":[{" 200 | << R"("address":")" << bind_logs.address << "\","; 201 | if (!bind_logs.topic.empty()) { 202 | oss << R"("topics":[")" << bind_logs.topic << "\"],"; 203 | } 204 | oss << R"("fromBlock":")" << HexConverter::NumericToHex(state.start) 205 | << "\"," 206 | << R"("toBlock":")" << HexConverter::NumericToHex(state.end) << "\"" 207 | << "}]" 208 | << "}"; 209 | return oss.str(); 210 | } 211 | 212 | const EthGetLogsRequest &bind_logs; 213 | idx_t request_id; 214 | CurrentState state; 215 | 216 | nlohmann::basic_json<> json; 217 | idx_t cur_row = 0; 218 | bool done = false; 219 | }; 220 | 221 | struct RPCLocalState : public LocalTableFunctionState { 222 | explicit RPCLocalState(unique_ptr rpc_request_p) 223 | : rpc_request(std::move(rpc_request_p)) {} 224 | 225 | unique_ptr rpc_request; 226 | }; 227 | 228 | //! Global State 229 | struct RPCGlobalState : public GlobalTableFunctionState { 230 | RPCGlobalState(const EthGetLogsRequest &bind_logs_p, idx_t number_of_threads, 231 | const vector projection_ids_p) 232 | : bind_logs(bind_logs_p), system_threads(number_of_threads), 233 | projection_ids(projection_ids_p) { 234 | state.start = bind_logs.from_block; 235 | if (bind_logs.blocks_per_thread == -1) { 236 | state.end = bind_logs.to_block; 237 | } else { 238 | state.end = bind_logs.blocks_per_thread > 239 | bind_logs.to_block - bind_logs.from_block 240 | ? bind_logs.to_block 241 | : bind_logs.from_block + bind_logs.blocks_per_thread; 242 | } 243 | finished = 0; 244 | } 245 | 246 | unique_ptr Next(bool init) { 247 | CurrentState cur_state; 248 | idx_t cur_request_id; 249 | { 250 | lock_guard parallel_lock(main_mutex); 251 | if (state.start > bind_logs.to_block) { 252 | ++finished; 253 | return nullptr; 254 | } 255 | if (!init) { 256 | ++finished; 257 | } 258 | cur_state = state; 259 | // we start off one position after the end 260 | state.start = state.end + 1; 261 | if (bind_logs.blocks_per_thread != -1) { 262 | if (bind_logs.blocks_per_thread > bind_logs.to_block - state.start) { 263 | state.end = bind_logs.to_block; 264 | } else { 265 | state.end = state.start + bind_logs.blocks_per_thread; 266 | } 267 | } 268 | cur_request_id = GetRequestId(); 269 | } 270 | return make_uniq(bind_logs, cur_request_id, cur_state); 271 | } 272 | 273 | idx_t GetRequestId() { return request_id++; } 274 | 275 | idx_t MaxThreads() const override { 276 | idx_t thread_iterations = (bind_logs.to_block - bind_logs.from_block) / 277 | bind_logs.blocks_per_thread; 278 | if (system_threads < thread_iterations) { 279 | return system_threads; 280 | } 281 | return thread_iterations; 282 | } 283 | 284 | const EthGetLogsRequest &bind_logs; 285 | const idx_t system_threads; 286 | CurrentState state; 287 | mutable mutex main_mutex; 288 | idx_t request_id = 0; 289 | std::atomic finished; 290 | const vector projection_ids; 291 | }; 292 | 293 | unique_ptr 294 | EthRPC::InitGlobal(ClientContext &context, TableFunctionInitInput &input) { 295 | auto &bind_data = input.bind_data->Cast(); 296 | return make_uniq(bind_data, context.db->NumberOfThreads(), 297 | input.column_ids); 298 | } 299 | 300 | unique_ptr 301 | EthRPC::InitLocal(ExecutionContext &context, TableFunctionInitInput &input, 302 | GlobalTableFunctionState *global_state_p) { 303 | if (!global_state_p) { 304 | return nullptr; 305 | } 306 | auto &global_state = global_state_p->Cast(); 307 | 308 | return make_uniq(global_state.Next(true)); 309 | } 310 | 311 | double EthRPC::ProgressBar(ClientContext &context, 312 | const FunctionData *bind_data_p, 313 | const GlobalTableFunctionState *global_state) { 314 | if (!global_state) { 315 | return 0; 316 | } 317 | auto &bind_data = bind_data_p->Cast(); 318 | auto &data = global_state->Cast(); 319 | double percentage = (double)(data.finished) * 320 | (double)bind_data.blocks_per_thread / 321 | (double)(bind_data.to_block - bind_data.from_block); 322 | return percentage * 100; 323 | } 324 | 325 | void EthRPC::Scan(ClientContext &context, TableFunctionInput &data_p, 326 | DataChunk &output) { 327 | 328 | auto &local_state = (RPCLocalState &)*data_p.local_state; 329 | auto &global_state = (RPCGlobalState &)*data_p.global_state; 330 | auto &bind_data = data_p.bind_data->Cast(); 331 | if (!local_state.rpc_request) { 332 | // We are done 333 | return; 334 | } 335 | 336 | if (local_state.rpc_request->done) { 337 | local_state.rpc_request = global_state.Next(false); 338 | if (!local_state.rpc_request) { 339 | // We are done 340 | return; 341 | } 342 | } 343 | auto &rpc_request = *local_state.rpc_request; 344 | auto &result = rpc_request.json["result"]; 345 | idx_t cur_chunk_size = 346 | result.size() - rpc_request.cur_row > STANDARD_VECTOR_SIZE 347 | ? STANDARD_VECTOR_SIZE 348 | : result.size() - rpc_request.cur_row; 349 | output.SetCardinality(cur_chunk_size); 350 | 351 | for (idx_t row_idx = 0; row_idx < cur_chunk_size; row_idx++) { 352 | auto &cur_result_row = result[rpc_request.cur_row++]; 353 | for (idx_t col_idx = 0; col_idx < global_state.projection_ids.size(); 354 | col_idx++) { 355 | vector topics = cur_result_row["topics"]; 356 | idx_t event_type = 6; 357 | if (event_signatures.find(topics[0]) != event_signatures.end()) { 358 | event_type = event_signatures.at(topics[0]).id; 359 | } 360 | switch (global_state.projection_ids[col_idx]) { 361 | case 0: 362 | // Column 0 - Address 363 | ((string_t *)output.data[col_idx].GetData())[row_idx] = 364 | StringVector::AddString(output.data[col_idx], 365 | cur_result_row["address"].dump()); 366 | break; 367 | case 1: 368 | // Column 1 - Event Type 369 | ((uint8_t *)output.data[col_idx].GetData())[row_idx] = event_type; 370 | break; 371 | case 2: 372 | // Column 2 - Block Hash 373 | ((string_t *)output.data[col_idx].GetData())[row_idx] = 374 | StringVector::AddString(output.data[col_idx], 375 | cur_result_row["blockHash"].dump()); 376 | break; 377 | case 3: 378 | // Column 3 - Block Number 379 | ((uint32_t *)output.data[col_idx].GetData())[row_idx] = 380 | stoi(cur_result_row["blockNumber"].get(), nullptr, 16); 381 | break; 382 | case 4: 383 | // Column 4 - Data 384 | { 385 | uhugeint_t u_hugeint_res{}; 386 | auto &child_vector = ListVector::GetEntry(output.data[col_idx]); 387 | auto list_content = FlatVector::GetData(child_vector); 388 | auto current_list_size = 389 | ListVector::GetListSize(output.data[col_idx]); 390 | auto current_list_capacity = 391 | ListVector::GetListCapacity(output.data[col_idx]); 392 | 393 | auto result_data = 394 | FlatVector::GetData(output.data[col_idx]); 395 | auto &list_entry = result_data[row_idx]; 396 | list_entry.offset = current_list_size; 397 | auto &child_validity = FlatVector::Validity(child_vector); 398 | if (event_type == 2) { 399 | // Sync Event 400 | std::string data = cur_result_row["data"]; 401 | std::string reserve0_hex = data.substr(2, 64); 402 | std::string reserve1_hex = data.substr(66, 64); 403 | // Make sure we have enough room for the new entries 404 | if (current_list_size + 2 >= current_list_capacity) { 405 | ListVector::Reserve(output.data[col_idx], 406 | current_list_capacity * 2); 407 | list_content = FlatVector::GetData(child_vector); 408 | } 409 | 410 | if (!HexConverter::HexToUhugeiInt(reserve0_hex, bind_data.strict, 411 | u_hugeint_res)) { 412 | child_validity.SetInvalid(current_list_size++); 413 | } else { 414 | list_content[current_list_size++] = u_hugeint_res; 415 | } 416 | if (!HexConverter::HexToUhugeiInt(reserve1_hex, bind_data.strict, 417 | u_hugeint_res)) { 418 | child_validity.SetInvalid(current_list_size++); 419 | } else { 420 | list_content[current_list_size++] = u_hugeint_res; 421 | } 422 | } else { 423 | if (current_list_size + 1 >= current_list_capacity) { 424 | ListVector::Reserve(output.data[col_idx], 425 | current_list_capacity * 2); 426 | list_content = FlatVector::GetData(child_vector); 427 | } 428 | std::string data = cur_result_row["data"]; 429 | std::string reserve0_hex = data.substr(2); 430 | if (!HexConverter::HexToUhugeiInt(reserve0_hex, bind_data.strict, 431 | u_hugeint_res)) { 432 | child_validity.SetInvalid(current_list_size++); 433 | } else { 434 | list_content[current_list_size++] = u_hugeint_res; 435 | } 436 | } 437 | list_entry.length = current_list_size - list_entry.offset; 438 | ListVector::SetListSize(output.data[col_idx], current_list_size); 439 | break; 440 | } 441 | case 5: 442 | ((uint32_t *)output.data[col_idx].GetData())[row_idx] = 443 | stoi(cur_result_row["logIndex"].get(), nullptr, 16); 444 | break; 445 | case 6: 446 | ((bool *)output.data[col_idx].GetData())[row_idx] = 447 | (int8_t)cur_result_row["removed"]; 448 | break; 449 | case 7: { 450 | // Column 7 - Topics 451 | auto &child_vector = ListVector::GetEntry(output.data[col_idx]); 452 | auto list_content = FlatVector::GetData(child_vector); 453 | auto current_list_size = ListVector::GetListSize(output.data[col_idx]); 454 | auto current_list_capacity = 455 | ListVector::GetListCapacity(output.data[col_idx]); 456 | 457 | auto result_data = 458 | FlatVector::GetData(output.data[col_idx]); 459 | auto &list_entry = result_data[row_idx]; 460 | list_entry.offset = current_list_size; 461 | // Make sure we have enough room for the new entries 462 | if (current_list_size + topics.size() - 1 >= current_list_capacity) { 463 | ListVector::Reserve(output.data[col_idx], current_list_capacity * 2); 464 | list_content = FlatVector::GetData(child_vector); 465 | } 466 | for (idx_t i = 1; i < topics.size(); i++) { 467 | list_content[current_list_size++] = 468 | StringVector::AddString(child_vector, topics[i]); 469 | } 470 | list_entry.length = current_list_size - list_entry.offset; 471 | ListVector::SetListSize(output.data[col_idx], current_list_size); 472 | 473 | break; 474 | } 475 | case 8: 476 | // Column 8 - TransactionHash 477 | ((string_t *)output.data[col_idx].GetData())[row_idx] = 478 | StringVector::AddString(output.data[col_idx], 479 | cur_result_row["transactionHash"].dump()); 480 | break; 481 | case 9: 482 | // Column 9 - Transaction Index 483 | ((int32_t *)output.data[col_idx].GetData())[row_idx] = 484 | stoi(cur_result_row["transactionIndex"].get(), nullptr, 16); 485 | break; 486 | default: 487 | break; 488 | } 489 | } 490 | } 491 | if (result.size() == rpc_request.cur_row) { 492 | // we are done 493 | rpc_request.done = true; 494 | } 495 | } 496 | } // namespace scrooge 497 | } // namespace duckdb 498 | -------------------------------------------------------------------------------- /src/scanner/yahoo_finance.cpp: -------------------------------------------------------------------------------- 1 | #include "duckdb/common/named_parameter_map.hpp" 2 | #include "duckdb/execution/operator/csv_scanner/csv_reader_options.hpp" 3 | #include "duckdb/main/relation/projection_relation.hpp" 4 | #include "duckdb/main/relation/read_csv_relation.hpp" 5 | #include "duckdb/parser/expression/constant_expression.hpp" 6 | #include "duckdb/parser/expression/star_expression.hpp" 7 | #include "functions/scanner.hpp" 8 | #include "duckdb/common/helper.hpp" 9 | 10 | namespace duckdb { 11 | namespace scrooge { 12 | 13 | int64_t IntervalInEpoch(string &interval) { 14 | // ble string checkaroo 15 | if (interval == "1d") { 16 | return Interval::SECS_PER_DAY; 17 | } 18 | if (interval == "5d") { 19 | return 5 * Interval::SECS_PER_DAY; 20 | } 21 | if (interval == "1wk") { 22 | return 7 * Interval::SECS_PER_DAY; 23 | } 24 | if (interval == "1mo") { 25 | return 30 * Interval::SECS_PER_DAY; 26 | } 27 | if (interval == "3mo") { 28 | return 90 * Interval::SECS_PER_DAY; 29 | } 30 | return 0; 31 | } 32 | 33 | struct YahooFunctionData : public TableFunctionData { 34 | YahooFunctionData(unique_ptr conn_p, vector &symbol_p, 35 | int64_t from_epoch_p, int64_t to_epoch_p, 36 | string &interval_p) 37 | : conn(std::move(conn_p)), symbols(symbol_p), from_epoch(from_epoch_p), 38 | to_epoch(to_epoch_p), interval(interval_p) { 39 | auto interval_epoch = IntervalInEpoch(interval); 40 | // We have to do this hacky thing to keep yahoo finance requests happy 41 | int expected_tuples = (to_epoch - from_epoch) / interval_epoch + 1; 42 | if (expected_tuples > 60) { 43 | increment_epoch = (to_epoch - from_epoch) / (expected_tuples / 60 + 1); 44 | } else { 45 | increment_epoch = (to_epoch - from_epoch); 46 | } 47 | from_epoch_og = from_epoch; 48 | cur_to_epoch = from_epoch + increment_epoch < to_epoch 49 | ? from_epoch + increment_epoch 50 | : to_epoch; 51 | symbol = symbols[0]; 52 | } 53 | shared_ptr plan; 54 | unique_ptr conn; 55 | vector symbols; 56 | string symbol; 57 | idx_t cur_symbol_idx = 0; 58 | int64_t from_epoch; 59 | int64_t from_epoch_og; 60 | int64_t cur_to_epoch; 61 | int64_t to_epoch; 62 | string interval; 63 | int64_t increment_epoch; 64 | }; 65 | 66 | shared_ptr GeneratePlan(YahooFunctionData &bind_data) { 67 | if (bind_data.cur_to_epoch > bind_data.to_epoch) { 68 | if (bind_data.cur_symbol_idx + 1 == bind_data.symbols.size()) { 69 | // we are done 70 | return nullptr; 71 | } 72 | bind_data.cur_symbol_idx++; 73 | bind_data.symbol = bind_data.symbols[bind_data.cur_symbol_idx]; 74 | bind_data.from_epoch = bind_data.from_epoch_og; 75 | bind_data.cur_to_epoch = 76 | bind_data.from_epoch + bind_data.increment_epoch < bind_data.to_epoch 77 | ? bind_data.from_epoch + bind_data.increment_epoch 78 | : bind_data.to_epoch; 79 | } 80 | auto from = to_string(bind_data.from_epoch); 81 | auto to = to_string(bind_data.cur_to_epoch); 82 | 83 | // Increment start 84 | bind_data.from_epoch += bind_data.increment_epoch; 85 | bind_data.cur_to_epoch += bind_data.increment_epoch; 86 | 87 | string url = "https://query2.finance.yahoo.com/v8/finance/chart/" + 88 | bind_data.symbol + "?period1=" + from + "&period2=" + to + 89 | "&interval=" + bind_data.interval + "&events=history"; 90 | string query = 91 | "SELECT '" + bind_data.symbol + 92 | "'as symbol, list_transform(chart.result[1].timestamp, x -> " 93 | "make_timestamp(x*1000000)::date) as date, " 94 | "chart.result[1].indicators.quote[1].open as open, " 95 | "chart.result[1].indicators.quote[1].high as high, " 96 | "chart.result[1].indicators.quote[1].low as low, " 97 | "chart.result[1].indicators.quote[1].close as close, " 98 | "chart.result[1].indicators.adjclose[1].adjclose as adj_close, " 99 | "chart.result[1].indicators.quote[1].volume as volume " + 100 | "FROM read_json('" + url + "');"; 101 | return bind_data.conn->RelationFromQuery(query); 102 | } 103 | 104 | void ValidInterval(string &interval) { 105 | unordered_set valid_interval{"1d", "5d", "1wk", "1mo", "3mo"}; 106 | if (valid_interval.find(interval) == valid_interval.end()) { 107 | string accepted_intervals = 108 | "1d: 1 day interval\n5d: 5 day interval\n1wk: 1 week interval\n1mo: 1 " 109 | "month interval\n3mo: 3 month interval\n"; 110 | throw InvalidInputException( 111 | "Interval is not valid, you should use one of the following valid " 112 | "intervals: \n" + 113 | accepted_intervals); 114 | } 115 | } 116 | 117 | unique_ptr YahooScanner::Bind(ClientContext &context, 118 | TableFunctionBindInput &input, 119 | vector &return_types, 120 | vector &names) { 121 | if (input.inputs[0].type() != LogicalType::VARCHAR && 122 | input.inputs[0].type() != LogicalType::LIST(LogicalType::VARCHAR)) { 123 | throw InvalidInputException( 124 | "Symbol must be either a String or a List of strings"); 125 | } 126 | if (input.inputs[1].type() != LogicalType::VARCHAR && 127 | input.inputs[1].type() != LogicalType::DATE) { 128 | throw InvalidInputException( 129 | "Start Period must be a Date or a Date-VARCHAR "); 130 | } 131 | if (input.inputs[2].type() != LogicalType::VARCHAR && 132 | input.inputs[2].type() != LogicalType::DATE) { 133 | throw InvalidInputException( 134 | "Start Period must be a Date or a Date-VARCHAR "); 135 | } 136 | vector symbols; 137 | if (input.inputs[0].type() == LogicalType::VARCHAR) { 138 | symbols.emplace_back(input.inputs[0].GetValueUnsafe()); 139 | } else { 140 | auto values = ListValue::GetChildren(input.inputs[0]); 141 | for (auto &value : values) { 142 | symbols.emplace_back(value.GetValueUnsafe()); 143 | } 144 | } 145 | auto from_date = input.inputs[1].GetValue(); 146 | auto to_date = input.inputs[2].GetValue(); 147 | auto from = Date::Epoch(input.inputs[1].GetValue()); 148 | auto to = Date::Epoch(input.inputs[2].GetValue()); 149 | auto interval = input.inputs[3].GetValue(); 150 | ValidInterval(interval); 151 | if (to_date <= from_date) { 152 | throw InvalidInputException( 153 | "The End period must be higher than the start period"); 154 | } 155 | auto result = make_uniq(make_uniq(*context.db), 156 | symbols, from, to, interval); 157 | result->plan = GeneratePlan(*result); 158 | for (auto &column : result->plan->Columns()) { 159 | return_types.emplace_back(column.Type()); 160 | names.emplace_back(column.Name()); 161 | } 162 | return std::move(result); 163 | } 164 | void YahooScanner::Scan(ClientContext &context, TableFunctionInput &data_p, 165 | DataChunk &output) { 166 | 167 | auto &data = (YahooFunctionData &)*data_p.bind_data; 168 | if (!data.plan) { 169 | return; 170 | } 171 | unique_ptr res = data.plan->Execute(); 172 | auto result_chunk = res->Fetch(); 173 | if (!result_chunk) { 174 | return; 175 | } 176 | output.Move(*result_chunk); 177 | data.plan = GeneratePlan(data); 178 | } 179 | } // namespace scrooge 180 | } // namespace duckdb -------------------------------------------------------------------------------- /src/scrooge_extension.cpp: -------------------------------------------------------------------------------- 1 | #define DUCKDB_EXTENSION_MAIN 2 | 3 | #include "scrooge_extension.hpp" 4 | #include "functions/functions.hpp" 5 | #include "functions/scanner.hpp" 6 | #include "duckdb.hpp" 7 | #include "duckdb/function/table_function.hpp" 8 | #include "duckdb/main/client_context.hpp" 9 | #include "duckdb/main/connection.hpp" 10 | #include "duckdb/parser/parsed_data/create_aggregate_function_info.hpp" 11 | #include "duckdb/parser/parsed_data/create_pragma_function_info.hpp" 12 | #include "duckdb/parser/parsed_data/create_table_function_info.hpp" 13 | #include 14 | #include "duckdb/parser/parsed_data/create_type_info.hpp" 15 | 16 | namespace duckdb { 17 | 18 | void ScroogeExtension::Load(DuckDB &db) { 19 | Connection con(db); 20 | con.BeginTransaction(); 21 | 22 | auto &catalog = Catalog::GetSystemCatalog(*con.context); 23 | scrooge::FirstScrooge::RegisterFunction(con, catalog); 24 | scrooge::LastScrooge::RegisterFunction(con, catalog); 25 | scrooge::TimeBucketScrooge::RegisterFunction(con, catalog); 26 | scrooge::Aliases::Register(con, catalog); 27 | 28 | // Create Yahoo Scanner Function 29 | TableFunction yahoo_scanner("yahoo_finance", 30 | {LogicalType::ANY, LogicalType::ANY, 31 | LogicalType::ANY, LogicalType::VARCHAR}, 32 | scrooge::YahooScanner::Scan, 33 | scrooge::YahooScanner::Bind); 34 | CreateTableFunctionInfo yahoo_scanner_info(yahoo_scanner); 35 | catalog.CreateTableFunction(*con.context, &yahoo_scanner_info); 36 | 37 | // Create Portfolio Frontier Function 38 | // FIXME: this should not be dependent of the yahoo scanner 39 | // TableFunction portfolio_frontier( 40 | // "portfolio_frontier", 41 | // {duckdb::LogicalType::LIST(duckdb::LogicalType::VARCHAR), 42 | // LogicalType::ANY, LogicalType::ANY, LogicalType::INTEGER}, 43 | // scrooge::PortfolioFrontier::Scan, scrooge::PortfolioFrontier::Bind); 44 | // CreateTableFunctionInfo portfolio_frontier_info(portfolio_frontier); 45 | // catalog.CreateTableFunction(*con.context, &portfolio_frontier_info); 46 | 47 | // Create Ethereum Scanner Function 48 | TableFunction ethereum_rpc_scanner( 49 | "read_eth", 50 | {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::BIGINT, 51 | LogicalType::BIGINT}, 52 | scrooge::EthRPC::Scan, scrooge::EthRPC::Bind, scrooge::EthRPC::InitGlobal, 53 | scrooge::EthRPC::InitLocal); 54 | ethereum_rpc_scanner.table_scan_progress = scrooge::EthRPC::ProgressBar; 55 | ethereum_rpc_scanner.projection_pushdown = true; 56 | ethereum_rpc_scanner.named_parameters["blocks_per_thread"] = 57 | LogicalType::BIGINT; 58 | ethereum_rpc_scanner.named_parameters["strict"] = LogicalType::BOOLEAN; 59 | 60 | CreateTableFunctionInfo ethereum_rpc_scanner_info(ethereum_rpc_scanner); 61 | catalog.CreateTableFunction(*con.context, ðereum_rpc_scanner_info); 62 | 63 | auto &config = DBConfig::GetConfig(*db.instance); 64 | 65 | config.AddExtensionOption("eth_node_url", 66 | "URL of Ethereum node to be queried", 67 | LogicalType::VARCHAR, "http://127.0.0.1:8545"); 68 | 69 | auto &temp_catalog = Catalog::GetCatalog(*con.context, TEMP_CATALOG); 70 | // Create CSV_ERROR_TYPE ENUM 71 | string enum_name = "ETH_EVENT"; 72 | Vector order_errors(LogicalType::VARCHAR, 7); 73 | order_errors.SetValue(0, "Transfer"); 74 | order_errors.SetValue(1, "Approval"); 75 | order_errors.SetValue(2, "Sync"); 76 | order_errors.SetValue(3, "TransferSingle"); 77 | order_errors.SetValue(4, "TransferBatch"); 78 | order_errors.SetValue(5, "ApprovalForAll"); 79 | order_errors.SetValue(6, "Unknown"); 80 | LogicalType enum_type = LogicalType::ENUM(enum_name, order_errors, 7); 81 | auto type_info = make_uniq(enum_name, enum_type); 82 | type_info->temporary = true; 83 | type_info->on_conflict = OnCreateConflict::IGNORE_ON_CONFLICT; 84 | temp_catalog.CreateType(*con.context, *type_info); 85 | con.Commit(); 86 | } 87 | 88 | std::string ScroogeExtension::Name() { return "scrooge"; } 89 | 90 | } // namespace duckdb 91 | 92 | extern "C" { 93 | 94 | DUCKDB_EXTENSION_API void scrooge_init(duckdb::DatabaseInstance &db) { 95 | duckdb::DuckDB db_wrapper(db); 96 | db_wrapper.LoadExtension(); 97 | } 98 | 99 | DUCKDB_EXTENSION_API const char *scrooge_version() { 100 | return duckdb::DuckDB::LibraryVersion(); 101 | } 102 | } -------------------------------------------------------------------------------- /test/sql/scrooge/test_eth_local.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/scrooge/test_eth_local.test 2 | # description: Test ETH scan over local node 3 | # group: [scrooge] 4 | 5 | require scrooge 6 | 7 | # We need a local node running for this test, so shouldn't run on CI 8 | mode skip 9 | 10 | # Sadly we can't support big integers yet 11 | statement error 12 | create table t as FROM read_eth( 13 | 'USDT', 14 | 'Transfer', 15 | 20034000, 16 | 20045000, 17 | blocks_per_thread = 1000, 18 | strict = true 19 | ); 20 | ---- 21 | This number requires a uint256, that is not yet supported 22 | 23 | 24 | statement ok 25 | create table t as FROM read_eth( 26 | 'USDT', 27 | 'Transfer', 28 | 20034000, 29 | 20045000, 30 | blocks_per_thread = 1000, 31 | strict = false 32 | ); 33 | 34 | query I 35 | select count(*) from t; 36 | ---- 37 | 233791 38 | 39 | query I 40 | select count(*) from T where data[1] is null 41 | ---- 42 | 130 43 | 44 | query I 45 | select count(*) FROM read_eth( 46 | 'USDT', 47 | 'Transfer', 48 | 20034000, 49 | 20045000, 50 | blocks_per_thread = 1000 51 | ); 52 | ---- 53 | 233791 -------------------------------------------------------------------------------- /test/sql/scrooge/test_eth_projection.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/scrooge/test_eth_projection.test 2 | # description: Test ETH scan 3 | # group: [scrooge] 4 | 5 | require scrooge 6 | 7 | statement ok 8 | set eth_node_url= 'https://mempool.merkle.io/rpc/eth/pk_mbs_0b647b195065b3294a5254838a33d062'; 9 | 10 | query I 11 | select address 12 | FROM read_eth( 13 | '0xBb2b8038a1640196FbE3e38816F3e67Cba72D940', 14 | 'SYNC', 15 | 20154992, 16 | 20155831 17 | ) 18 | limit 1; 19 | ---- 20 | "0xbb2b8038a1640196fbe3e38816f3e67cba72d940" 21 | 22 | query II 23 | select address, data 24 | FROM read_eth( 25 | '0xBb2b8038a1640196FbE3e38816F3e67Cba72D940', 26 | 'SYNC', 27 | 20154992, 28 | 20155831 29 | ) 30 | limit 1; 31 | ---- 32 | "0xbb2b8038a1640196fbe3e38816f3e67cba72d940" [10565836168, 1940010911013880876521] 33 | 34 | query II 35 | select data, address 36 | FROM read_eth( 37 | '0xBb2b8038a1640196FbE3e38816F3e67Cba72D940', 38 | 'SYNC', 39 | 20154992, 40 | 20155831 41 | ) 42 | limit 1; 43 | ---- 44 | [10565836168, 1940010911013880876521] "0xbb2b8038a1640196fbe3e38816f3e67cba72d940" 45 | 46 | query I 47 | select count(*) 48 | FROM read_eth( 49 | '0xBb2b8038a1640196FbE3e38816F3e67Cba72D940', 50 | 'SYNC', 51 | 20154992, 52 | 20155831 53 | ) 54 | limit 1; 55 | ---- 56 | 13 57 | 58 | -------------------------------------------------------------------------------- /test/sql/scrooge/test_eth_roi.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/scrooge/test_eth_roi.test 2 | # description: Test ETH scan 3 | # group: [scrooge] 4 | 5 | require scrooge 6 | 7 | mode skip 8 | # Best ETH Traders of 1st week of June 9 | 10 | statement ok 11 | CREATE TABLE WETH_Transfers AS 12 | SELECT 13 | block_number, 14 | data[1] / 1000000000000000000 AS amount, 15 | lower(topics[1]) AS from_address, 16 | lower(topics[2]) AS to_address 17 | FROM read_eth( 18 | 'WETH', 19 | 'Transfer', 20 | 20000000, 21 | 20040000, 22 | blocks_per_thread = 1 23 | ); 24 | 25 | 26 | statement ok 27 | CREATE TABLE WETH_Prices AS 28 | SELECT 29 | block_number, 30 | (data[2] / data[1]) * 1000000000000 AS price 31 | FROM read_eth( 32 | 'WETH_USDT', 33 | 'SYNC', 34 | 20000000, 35 | 20040000, 36 | blocks_per_thread = 1000 37 | ); 38 | 39 | query I 40 | WITH WETH_Transfer_Values AS ( 41 | SELECT 42 | t.block_number, 43 | t.amount, 44 | t.from_address, 45 | t.to_address, 46 | p.price, 47 | t.amount * p.price AS value 48 | FROM WETH_Transfers t 49 | JOIN LATERAL ( 50 | SELECT price 51 | FROM WETH_Prices p 52 | WHERE p.block_number <= t.block_number 53 | ORDER BY p.block_number DESC 54 | LIMIT 1 55 | ) p ON true 56 | ), 57 | Address_Values AS ( 58 | SELECT 59 | address, 60 | SUM(value) AS net_value, 61 | SUM(CASE WHEN type = 'in' THEN value ELSE 0 END) AS total_value_in, 62 | SUM(CASE WHEN type = 'out' THEN value ELSE 0 END) AS total_value_out, 63 | SUM(amount) AS net_amount 64 | FROM ( 65 | SELECT from_address AS address, -value AS value, 'out' AS type, -amount as amount FROM WETH_Transfer_Values 66 | UNION ALL 67 | SELECT to_address AS address, value AS value, 'in' AS type, amount as amount FROM WETH_Transfer_Values 68 | ) AS combined 69 | GROUP BY address 70 | ) 71 | SELECT 72 | address, 73 | net_value, 74 | total_value_in, 75 | total_value_out, 76 | CASE WHEN total_value_in <> 0 THEN ((total_value_out - total_value_in) / total_value_in) * 100 ELSE NULL END AS ROI, 77 | net_amount 78 | FROM Address_Values 79 | WHERE net_amount > 0 80 | ORDER BY ROI DESC NULLS LAST, 81 | net_value DESC 82 | LIMIT 4; 83 | ---- 84 | 0x0000000000000000000000009600a48ed0f931d0c422d574e3275a90d8b22745 7019197.761391707 7019197.761391707 0.0 -100.0 2000.0 85 | 0x0000000000000000000000001827f9ea98e0bf96550b2fc20f7233277fcd7e63 6791969.193029855 6791969.193029855 0.0 -100.0 1936.0 86 | 0x00000000000000000000000030604f68318be1ab1beb0d7d27c487caee9bee30 3119976.2776494976 3119976.2776494976 0.0 -100.0 911.4327243104834 87 | 0x00000000000000000000000042d019ce067f855cf62d17258ef736f85490e48c 2910310.65775324 2910310.65775324 0.0 -100.0 850.3722826837854 88 | -------------------------------------------------------------------------------- /test/sql/scrooge/test_eth_uniswap.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/scrooge/test_eth_uniswap.test 2 | # description: Test ETH scan ROI 3 | # group: [scrooge] 4 | 5 | require scrooge 6 | 7 | statement ok 8 | set eth_node_url= 'https://mempool.merkle.io/rpc/eth/pk_mbs_0b647b195065b3294a5254838a33d062'; 9 | 10 | query IIIIIIIIII 11 | FROM read_eth( 12 | '0xBb2b8038a1640196FbE3e38816F3e67Cba72D940', 13 | 'SYNC', 14 | 20155765, 15 | 20155765 16 | ) 17 | limit 1 18 | ---- 19 | "0xbb2b8038a1640196fbe3e38816f3e67cba72d940" Sync "0x5004f54a265f2dde80254c7d972516280cf886f3348aa5616b869a03d24bdecd" 20155765 [10551363886, 1942679877456358670324] 177 false [] "0xe81b2289f41c1e397ca5f9d8be4534778a92a5186aa2032500f8b6e782d02990" 34 20 | 21 | statement error 22 | select data[1], data[2] FROM read_eth( 23 | 'WBTK_WETH', 24 | 'SYNC', 25 | 20154992, 26 | 20155831 27 | ) 28 | ---- 29 | Failed to infer the address 30 | 31 | # WBTC and ETH pairing 32 | query II 33 | select data[1], data[2] FROM read_eth( 34 | '0xBb2b8038a1640196FbE3e38816F3e67Cba72D940', 35 | 'SYNC', 36 | 20154992, 37 | 20155831 38 | ) 39 | ---- 40 | 10565836168 1940010911013880876521 41 | 10565816952 1940014450056568835941 42 | 10565810494 1940015639556568835941 43 | 10565783345 1940020639556568835941 44 | 10564794691 1940202733214438415478 45 | 10558627962 1941339311449401968346 46 | 10558655111 1941334334745024700252 47 | 10554249412 1942147151681153011326 48 | 10553162954 1942347699350556225037 49 | 10551363886 1942679877456358670324 50 | 10551400278 1942673197213835942974 51 | 10551424347 1942668779049485225198 52 | 10551427731 1942668157875806339229 53 | 54 | query II 55 | select data[1], data[2] FROM read_eth( 56 | 'WBTC_WETH', 57 | 'SYNC', 58 | 20154992, 59 | 20155831 60 | ) 61 | ---- 62 | 10565836168 1940010911013880876521 63 | 10565816952 1940014450056568835941 64 | 10565810494 1940015639556568835941 65 | 10565783345 1940020639556568835941 66 | 10564794691 1940202733214438415478 67 | 10558627962 1941339311449401968346 68 | 10558655111 1941334334745024700252 69 | 10554249412 1942147151681153011326 70 | 10553162954 1942347699350556225037 71 | 10551363886 1942679877456358670324 72 | 10551400278 1942673197213835942974 73 | 10551424347 1942668779049485225198 74 | 10551427731 1942668157875806339229 -------------------------------------------------------------------------------- /test/sql/scrooge/test_scrooge_first.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/scrooge/test_scrooge_first.test 2 | # description: Test first_s function 3 | # group: [scrooge] 4 | 5 | require scrooge 6 | 7 | #Corner cases 8 | statement error 9 | select first_s() 10 | ---- 11 | 12 | query I 13 | select first_s(NULL,NULL) 14 | ---- 15 | NULL 16 | 17 | query I 18 | select first_s(NULL,'2021-11-15 02:30:00'::timestamptz) 19 | ---- 20 | NULL 21 | 22 | 23 | #Empty Table 24 | query I 25 | select first_s(0 ,'2021-11-15 02:30:00'::timestamptz ) from range (5) where 1 == 0 26 | ---- 27 | NULL 28 | 29 | foreach timestamp_type timestamp timestamptz 30 | 31 | foreach type decimal(4,0) decimal(8,0) decimal(12,0) decimal(18,0) 32 | 33 | statement ok 34 | create table test (a integer, b ${type}, c ${timestamp_type}) 35 | 36 | statement ok 37 | insert into test values 38 | (1, 8::${type}, '2021-11-14 02:30:00'::${timestamp_type}), 39 | (1, null, '2021-11-15 02:30:00'::${timestamp_type}), 40 | (null, 10::${type}, '2021-11-13 02:30:00'::${timestamp_type}), 41 | (1, 7::${type}, '2021-11-12 02:30:00'::${timestamp_type}), 42 | (2, 4::${type}, '2021-11-18 02:30:00'::${timestamp_type}), 43 | (2, 20::${type}, '2021-11-20 02:30:00'::${timestamp_type}), 44 | (2, 25::${type}, null), 45 | (2, 30::${type}, '2021-11-17 02:30:00'::${timestamp_type}), 46 | (2, 90::${type}, '2021-12-15 02:30:00'::${timestamp_type}), 47 | (2, 88::${type}, '2020-11-15 02:30:00'::${timestamp_type}), 48 | (2, 72::${type}, '2021-11-15 02:30:31'::${timestamp_type}); 49 | 50 | query I 51 | select first_s(b,c) from test; 52 | ---- 53 | 88 54 | 55 | query I 56 | select first_s(b,c) from test group by a; 57 | ---- 58 | 10 59 | 7 60 | 88 61 | 62 | # Window Function 63 | require vector_size 512 64 | 65 | query I rowsort 66 | select first_s(b,c) over (partition by a) 67 | from test; 68 | ---- 69 | 10 70 | 7 71 | 7 72 | 7 73 | 88 74 | 88 75 | 88 76 | 88 77 | 88 78 | 88 79 | 88 80 | 81 | statement ok 82 | DROP TABLE test 83 | 84 | endloop 85 | 86 | endloop -------------------------------------------------------------------------------- /test/sql/scrooge/test_scrooge_last.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/scrooge/test_scrooge_last.test 2 | # description: Test last_s function 3 | # group: [scrooge] 4 | 5 | require scrooge 6 | 7 | #Corner cases 8 | statement error 9 | select last_s() 10 | ---- 11 | 12 | query I 13 | select last_s(NULL,NULL) 14 | ---- 15 | NULL 16 | 17 | query I 18 | select last_s(NULL,'2021-11-15 02:30:00'::timestamptz) 19 | ---- 20 | NULL 21 | 22 | 23 | #Empty Table 24 | query I 25 | select last_s(0 ,'2021-11-15 02:30:00'::timestamptz ) from range (5) where 1 == 0 26 | ---- 27 | NULL 28 | 29 | 30 | foreach timestamp_type timestamp timestamptz 31 | 32 | foreach type decimal(4,0) decimal(8,0) decimal(12,0) decimal(18,0) 33 | 34 | statement ok 35 | create table test (a integer, b ${type}, c ${timestamp_type}) 36 | 37 | statement ok 38 | insert into test values 39 | (1, 8::${type}, '2021-11-14 02:30:00'::${timestamp_type}), 40 | (1, null, '2021-11-15 02:30:00'::${timestamp_type}), 41 | (null, 10::${type}, '2021-11-13 02:30:00'::${timestamp_type}), 42 | (1, 7::${type}, '2021-11-12 02:30:00'::${timestamp_type}), 43 | (2, 4::${type}, '2021-11-18 02:30:00'::${timestamp_type}), 44 | (2, 20::${type}, '2021-11-20 02:30:00'::${timestamp_type}), 45 | (2, 25::${type}, null), 46 | (2, 30::${type}, '2021-11-17 02:30:00'::${timestamp_type}), 47 | (2, 90::${type}, '2021-12-15 02:30:00'::${timestamp_type}), 48 | (2, 88::${type}, '2020-11-15 02:30:00'::${timestamp_type}), 49 | (2, 72::${type}, '2021-11-15 02:30:31'::${timestamp_type}); 50 | 51 | query I 52 | select last_s(b,c) from test; 53 | ---- 54 | 90 55 | 56 | query I 57 | select last_s(b,c) from test group by a; 58 | ---- 59 | 10 60 | 8 61 | 90 62 | 63 | # Window Function 64 | require vector_size 512 65 | 66 | query I rowsort 67 | select last_s(b,c) over (partition by a) 68 | from test; 69 | ---- 70 | 10 71 | 8 72 | 8 73 | 8 74 | 90 75 | 90 76 | 90 77 | 90 78 | 90 79 | 90 80 | 90 81 | 82 | statement ok 83 | DROP TABLE test 84 | 85 | endloop 86 | 87 | endloop -------------------------------------------------------------------------------- /test/sql/scrooge/test_scrooge_timebucket.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/scrooge/test_scrooge_timebucket.test 2 | # description: Test timebucket function 3 | # group: [scrooge] 4 | 5 | require scrooge 6 | 7 | #Corner cases 8 | statement error 9 | select timebucket() 10 | ---- 11 | 12 | query I 13 | select timebucket(NULL,NULL) 14 | ---- 15 | NULL 16 | 17 | query I 18 | select timebucket('2021-11-15 02:30:00'::timestamptz,NULL) 19 | ---- 20 | NULL 21 | 22 | query I 23 | select timebucket(NULL,'1M'::interval) 24 | ---- 25 | NULL 26 | 27 | #Empty Table 28 | query I 29 | select timebucket('2021-11-15 02:30:00'::timestamptz, '1M'::interval ) from range (5) where 1 == 0 30 | ---- 31 | 32 | statement ok 33 | create table test (a timestamptz) 34 | 35 | statement ok 36 | insert into test values 37 | ('2021-11-14 02:30:00'::timestamptz), 38 | ('2021-11-14 02:30:01'::timestamptz), 39 | ('2021-11-14 02:30:02'::timestamptz), 40 | ('2021-11-14 02:30:03'::timestamptz), 41 | ('2021-11-14 02:30:04'::timestamptz), 42 | ('2021-11-14 02:30:05'::timestamptz), 43 | ('2021-11-14 02:30:09'::timestamptz), 44 | ('2021-11-14 02:30:33'::timestamptz), 45 | ('2021-11-14 02:30:34'::timestamptz), 46 | ('2021-11-14 02:30:35'::timestamptz), 47 | (NULL); 48 | 49 | query I 50 | select timebucket(a, '5S'::interval ) from test; 51 | ---- 52 | 2021-11-14 02:30:00+00 53 | 2021-11-14 02:30:00+00 54 | 2021-11-14 02:30:00+00 55 | 2021-11-14 02:30:00+00 56 | 2021-11-14 02:30:00+00 57 | 2021-11-14 02:30:05+00 58 | 2021-11-14 02:30:05+00 59 | 2021-11-14 02:30:30+00 60 | 2021-11-14 02:30:30+00 61 | 2021-11-14 02:30:35+00 62 | NULL 63 | 64 | query I 65 | select count from (select timebucket(a, '5S'::interval ) as buckets, count(*) as count from test group by buckets) order by all; 66 | ---- 67 | 1 68 | 1 69 | 2 70 | 2 71 | 5 72 | 73 | statement ok 74 | drop table test 75 | 76 | statement ok 77 | create table test (a timestamp) 78 | 79 | statement ok 80 | insert into test values 81 | ('2021-11-14 02:30:00'::timestamp), 82 | ('2021-11-14 02:30:01'::timestamp), 83 | ('2021-11-14 02:30:02'::timestamp), 84 | ('2021-11-14 02:30:03'::timestamp), 85 | ('2021-11-14 02:30:04'::timestamp), 86 | ('2021-11-14 02:30:05'::timestamp), 87 | ('2021-11-14 02:30:09'::timestamp), 88 | ('2021-11-14 02:30:33'::timestamp), 89 | ('2021-11-14 02:30:34'::timestamp), 90 | ('2021-11-14 02:30:35'::timestamp), 91 | (NULL); 92 | 93 | query I 94 | select timebucket(a, '5S'::interval ) from test; 95 | ---- 96 | 2021-11-14 02:30:00 97 | 2021-11-14 02:30:00 98 | 2021-11-14 02:30:00 99 | 2021-11-14 02:30:00 100 | 2021-11-14 02:30:00 101 | 2021-11-14 02:30:05 102 | 2021-11-14 02:30:05 103 | 2021-11-14 02:30:30 104 | 2021-11-14 02:30:30 105 | 2021-11-14 02:30:35 106 | NULL 107 | 108 | query I 109 | select count from (select timebucket(a, '5S'::interval ) as buckets, count(*) as count from test group by buckets) order by all; 110 | ---- 111 | 1 112 | 1 113 | 2 114 | 2 115 | 5 -------------------------------------------------------------------------------- /test/sql/scrooge/test_yahoo.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/scrooge/test_yahoo.test 2 | # description: Test Yahoo Finance Scanner 3 | # group: [scrooge] 4 | 5 | require scrooge 6 | 7 | require httpfs 8 | 9 | require json 10 | 11 | statement error 12 | select * FROM yahoo_finance("^aeouhae", "2017-12-01"::DATE, "2017-12-10"::DATE, "1d") 13 | ---- 14 | 15 | statement error 16 | select * FROM yahoo_finance("^GSPC", "2017-12-11"::DATE, "2017-12-10"::DATE, "1d") 17 | ---- 18 | 19 | statement error 20 | select * FROM yahoo_finance("^GSPC", "2017-12-11"::DATE, "2017-12-10"::DATE, -1) 21 | ---- 22 | 23 | query IIIIIIII 24 | select * FROM yahoo_finance("^GSPC", "2017-12-01"::DATE, "2017-12-10"::DATE, "1d") 25 | ---- 26 | ^GSPC [2017-12-01, 2017-12-04, 2017-12-05, 2017-12-06, 2017-12-07, 2017-12-08] [2645.10009765625, 2657.18994140625, 2639.780029296875, 2626.239990234375, 2628.3798828125, 2646.2099609375] [2650.6201171875, 2665.18994140625, 2648.719970703125, 2634.409912109375, 2640.989990234375, 2651.64990234375] [2605.52001953125, 2639.030029296875, 2627.72998046875, 2624.75, 2626.530029296875, 2644.10009765625] [2642.219970703125, 2639.43994140625, 2629.570068359375, 2629.27001953125, 2636.97998046875, 2651.5] [2642.219970703125, 2639.43994140625, 2629.570068359375, 2629.27001953125, 2636.97998046875, 2651.5] [3950930000, 4025840000, 3547570000, 3253080000, 3297060000, 3126750000] 27 | 28 | # Try 5 day period 29 | query IIIIIIII 30 | select * FROM yahoo_finance("^GSPC", "2017-12-01"::DATE, "2017-12-10"::DATE, "5d") 31 | ---- 32 | ^GSPC [2017-12-01, 2017-12-06] [2645.10009765625, 2626.239990234375] [2650.6201171875, 2634.409912109375] [2605.52001953125, 2624.75] [2642.219970703125, 2629.27001953125] [2642.219970703125, 2629.27001953125] [3950930000, 3253080000] 33 | 34 | 35 | # Try 1wk period 36 | query IIIIIIII 37 | select * FROM yahoo_finance("^GSPC", "2017-12-01"::DATE, "2017-12-20"::DATE, "1wk") 38 | ---- 39 | ^GSPC [2017-11-27, 2017-12-04, 2017-12-11, 2017-12-18] [2633.929931640625, 2657.18994140625, 2652.18994140625, 2685.919921875] [2657.739990234375, 2665.18994140625, 2679.6298828125, 2694.969970703125] [2605.52001953125, 2624.75, 2651.469970703125, 2680.739990234375] [2642.219970703125, 2651.5, 2675.81005859375, 2681.469970703125] [2642.219970703125, 2651.5, 2675.81005859375, 2681.469970703125] [8916630000, 17250300000, 19458420000, 7135450000] 40 | 41 | # Try 1mo period 42 | query IIIIIIII 43 | select * FROM yahoo_finance("^GSPC", "2017-12-01"::DATE, "2018-06-20"::DATE, "1mo") 44 | ---- 45 | ^GSPC [2017-12-01, 2018-01-01, 2018-02-01, 2018-03-01, 2018-04-01, 2018-05-01, 2018-06-01] [2645.10009765625, 2683.72998046875, 2816.449951171875, 2715.219970703125, 2633.449951171875, 2642.9599609375, 2718.699951171875] [2694.969970703125, 2872.8701171875, 2835.9599609375, 2801.89990234375, 2717.489990234375, 2742.239990234375, 2791.469970703125] [2605.52001953125, 2682.360107421875, 2532.68994140625, 2585.889892578125, 2553.800048828125, 2594.6201171875, 2691.989990234375] [2673.610107421875, 2823.81005859375, 2713.830078125, 2640.8701171875, 2648.050048828125, 2705.27001953125, 2718.3701171875] [2673.610107421875, 2823.81005859375, 2713.830078125, 2640.8701171875, 2648.050048828125, 2705.27001953125, 2718.3701171875] [65531700000, 77318690000, 79933970000, 76803890000, 70194700000, 76011820000, 77891360000] 46 | 47 | # Try 3mo period 48 | query IIIIIIII 49 | select * FROM yahoo_finance("^GSPC", "2017-12-01"::DATE, "2018-06-20"::DATE, "3mo") 50 | ---- 51 | ^GSPC [2017-12-01, 2018-03-01, 2018-06-01] [2645.10009765625, 2715.219970703125, 2718.699951171875] [2872.8701171875, 2801.89990234375, 2791.469970703125] [2532.68994140625, 2553.800048828125, 2691.989990234375] [2713.830078125, 2705.27001953125, 2718.3701171875] [2713.830078125, 2705.27001953125, 2718.3701171875] [222784360000, 223010410000, 77891360000] 52 | 53 | # Try Big Data 54 | query IIIIIIII 55 | select * FROM yahoo_finance("BTC-USD", "2015-01-01"::DATE, "2015-04-01"::DATE, "1d"); 56 | ---- 57 | BTC-USD [2015-01-01, 2015-01-02, 2015-01-03, 2015-01-04, 2015-01-05, 2015-01-06, 2015-01-07, 2015-01-08, 2015-01-09, 2015-01-10, 2015-01-11, 2015-01-12, 2015-01-13, 2015-01-14, 2015-01-15, 2015-01-16, 2015-01-17, 2015-01-18, 2015-01-19, 2015-01-20, 2015-01-21, 2015-01-22, 2015-01-23, 2015-01-24, 2015-01-25, 2015-01-26, 2015-01-27, 2015-01-28, 2015-01-29, 2015-01-30, 2015-01-31, 2015-02-01, 2015-02-02, 2015-02-03, 2015-02-04, 2015-02-05, 2015-02-06, 2015-02-07, 2015-02-08, 2015-02-09, 2015-02-10, 2015-02-11, 2015-02-12, 2015-02-13, 2015-02-14, 2015-02-15] [320.43499755859375, 314.0790100097656, 314.84600830078125, 281.14599609375, 265.0840148925781, 274.6109924316406, 286.0769958496094, 294.135009765625, 282.38299560546875, 287.3030090332031, 274.6080017089844, 266.14599609375, 267.3940124511719, 223.8939971923828, 176.89700317382812, 209.07000732421875, 207.83399963378906, 200.0500030517578, 211.4709930419922, 212.90699768066406, 211.3780059814453, 227.32200622558594, 233.51699829101562, 232.6999969482422, 247.3520050048828, 254.07899475097656, 273.1669921875, 263.35101318359375, 233.34800720214844, 232.77200317382812, 226.4409942626953, 216.86700439453125, 226.49099731445312, 237.45399475097656, 227.51100158691406, 227.6649932861328, 216.92300415039062, 222.63299560546875, 227.6929931640625, 223.38900756835938, 220.28199768066406, 219.73199462890625, 219.20799255371094, 221.968994140625, 235.5279998779297, 257.5069885253906] [320.43499755859375, 315.8389892578125, 315.1499938964844, 287.2300109863281, 278.34100341796875, 287.5530090332031, 298.7539978027344, 294.135009765625, 291.114013671875, 288.12701416015625, 279.63800048828125, 272.2030029296875, 268.2770080566406, 223.8939971923828, 229.06700134277344, 221.59100341796875, 211.7310028076172, 218.69500732421875, 216.72799682617188, 215.24099731445312, 227.78799438476562, 237.0189971923828, 234.84500122070312, 248.2100067138672, 255.07400512695312, 309.3840026855469, 275.4800109863281, 266.5350036621094, 238.70599365234375, 242.8509979248047, 233.50399780273438, 231.57400512695312, 242.1750030517578, 245.95700073242188, 230.05799865722656, 239.40499877929688, 230.50999450683594, 230.2989959716797, 229.43800354003906, 223.9770050048828, 221.8070068359375, 223.406005859375, 222.19900512695312, 240.25900268554688, 259.8080139160156, 265.6109924316406] [314.00299072265625, 313.56500244140625, 281.0820007324219, 257.61199951171875, 265.0840148925781, 272.6960144042969, 283.0790100097656, 282.17498779296875, 280.5329895019531, 273.96600341796875, 265.03900146484375, 265.20001220703125, 219.906005859375, 171.50999450683594, 176.89700317382812, 199.77099609375, 194.875, 194.50599670410156, 207.3179931640625, 205.1529998779297, 211.21200561523438, 226.4340057373047, 225.1959991455078, 230.02200317382812, 243.88999938964844, 254.07899475097656, 250.6529998779297, 227.04600524902344, 220.71200561523438, 225.83900451660156, 216.3090057373047, 212.01499938964844, 222.65899658203125, 224.48300170898438, 221.11300659179688, 214.72500610351562, 216.23199462890625, 222.60699462890625, 221.07699584960938, 217.0189971923828, 215.33200073242188, 218.07400512695312, 217.61399841308594, 221.26199340820312, 235.5279998779297, 227.6840057373047] [314.2489929199219, 315.0320129394531, 281.0820007324219, 264.19500732421875, 274.4739990234375, 286.1889953613281, 294.3370056152344, 283.3489990234375, 290.4079895019531, 274.7959899902344, 265.6600036621094, 267.7959899902344, 225.86099243164062, 178.10299682617188, 209.843994140625, 208.0970001220703, 199.25999450683594, 210.33900451660156, 214.86099243164062, 211.31500244140625, 226.89700317382812, 233.406005859375, 232.87899780273438, 247.8470001220703, 253.71800231933594, 273.4729919433594, 263.4750061035156, 233.9149932861328, 233.51300048828125, 226.4250030517578, 217.46400451660156, 226.9720001220703, 238.22900390625, 227.26800537109375, 226.85299682617188, 217.11099243164062, 222.26600646972656, 227.75399780273438, 223.41200256347656, 220.11000061035156, 219.83900451660156, 219.18499755859375, 221.76400756835938, 235.427001953125, 257.3210144042969, 234.8249969482422] [314.2489929199219, 315.0320129394531, 281.0820007324219, 264.19500732421875, 274.4739990234375, 286.1889953613281, 294.3370056152344, 283.3489990234375, 290.4079895019531, 274.7959899902344, 265.6600036621094, 267.7959899902344, 225.86099243164062, 178.10299682617188, 209.843994140625, 208.0970001220703, 199.25999450683594, 210.33900451660156, 214.86099243164062, 211.31500244140625, 226.89700317382812, 233.406005859375, 232.87899780273438, 247.8470001220703, 253.71800231933594, 273.4729919433594, 263.4750061035156, 233.9149932861328, 233.51300048828125, 226.4250030517578, 217.46400451660156, 226.9720001220703, 238.22900390625, 227.26800537109375, 226.85299682617188, 217.11099243164062, 222.26600646972656, 227.75399780273438, 223.41200256347656, 220.11000061035156, 219.83900451660156, 219.18499755859375, 221.76400756835938, 235.427001953125, 257.3210144042969, 234.8249969482422] [8036550, 7860650, 33054400, 55629100, 43962800, 23245700, 24866800, 19982500, 18718600, 15264300, 18200800, 18880300, 72843904, 97638704, 81773504, 38421000, 23469700, 30085100, 18658300, 24051100, 29924600, 33544600, 24621700, 24782500, 33582700, 106794000, 44399000, 44352200, 32213400, 26605200, 23348200, 29128500, 30612100, 40783700, 26594300, 22516400, 24435300, 21604200, 17145200, 27791300, 21115100, 17201900, 15206200, 42744400, 49732500, 56552400] 58 | BTC-USD [2015-02-15, 2015-02-16, 2015-02-17, 2015-02-18, 2015-02-19, 2015-02-20, 2015-02-21, 2015-02-22, 2015-02-23, 2015-02-24, 2015-02-25, 2015-02-26, 2015-02-27, 2015-02-28, 2015-03-01, 2015-03-02, 2015-03-03, 2015-03-04, 2015-03-05, 2015-03-06, 2015-03-07, 2015-03-08, 2015-03-09, 2015-03-10, 2015-03-11, 2015-03-12, 2015-03-13, 2015-03-14, 2015-03-15, 2015-03-16, 2015-03-17, 2015-03-18, 2015-03-19, 2015-03-20, 2015-03-21, 2015-03-22, 2015-03-23, 2015-03-24, 2015-03-25, 2015-03-26, 2015-03-27, 2015-03-28, 2015-03-29, 2015-03-30, 2015-03-31, 2015-04-01] [257.5069885253906, 234.8249969482422, 233.4219970703125, 243.77999877929688, 236.41000366210938, 240.25100708007812, 243.7519989013672, 244.54400634765625, 235.9949951171875, 238.9980010986328, 238.88999938964844, 237.33700561523438, 236.43600463867188, 253.52000427246094, 254.2830047607422, 260.35699462890625, 275.0459899902344, 281.989990234375, 272.739013671875, 275.6000061035156, 272.29400634765625, 276.4330139160156, 274.81201171875, 289.86199951171875, 291.5249938964844, 296.12701416015625, 294.1180114746094, 284.4419860839844, 281.42498779296875, 285.68499755859375, 290.5950012207031, 285.0669860839844, 255.8800048828125, 260.95599365234375, 261.6440124511719, 259.9169921875, 267.8949890136719, 266.5769958496094, 247.4720001220703, 246.2760009765625, 248.5659942626953, 246.97500610351562, 252.74000549316406, 242.87899780273438, 247.45399475097656, 244.22300720214844] [265.6109924316406, 239.52099609375, 245.77499389648438, 244.25100708007812, 242.6719970703125, 247.1009979248047, 255.32000732421875, 246.39199829101562, 240.10899353027344, 239.9010009765625, 239.33999633789062, 237.7100067138672, 256.65301513671875, 254.69200134277344, 261.6600036621094, 276.3009948730469, 285.7959899902344, 284.2250061035156, 281.6669921875, 277.6080017089844, 277.85400390625, 277.8580017089844, 292.70098876953125, 300.04400634765625, 297.3909912109375, 297.0880126953125, 294.49798583984375, 286.3420104980469, 286.52899169921875, 294.11199951171875, 292.364990234375, 285.33599853515625, 264.2439880371094, 264.8479919433594, 262.1960144042969, 269.74700927734375, 277.2969970703125, 267.00299072265625, 249.19000244140625, 254.35400390625, 256.8110046386719, 254.2050018310547, 253.13900756835938, 249.24200439453125, 248.72999572753906, 247.54100036621094] [227.6840057373047, 229.02200317382812, 232.31399536132812, 232.33999633789062, 235.5919952392578, 239.2989959716797, 243.1840057373047, 233.8509979248047, 232.42100524902344, 236.40199279785156, 235.52999877929688, 234.2570037841797, 236.43600463867188, 249.47900390625, 245.93299865722656, 258.31298828125, 268.1610107421875, 268.1260070800781, 264.7690124511719, 270.0150146484375, 270.13299560546875, 272.56500244140625, 273.89300537109375, 289.7430114746094, 290.50799560546875, 292.4129943847656, 285.3370056152344, 280.97601318359375, 280.9960021972656, 285.68499755859375, 284.3739929199219, 249.8699951171875, 248.63600158691406, 259.1619873046875, 255.64999389648438, 259.5899963378906, 261.7449951171875, 244.15499877929688, 236.51499938964844, 244.90499877929688, 245.21299743652344, 246.97500610351562, 240.85000610351562, 239.21400451660156, 242.73899841308594, 241.16000366210938] [234.8249969482422, 233.84300231933594, 243.61000061035156, 236.3260040283203, 240.2830047607422, 243.7790069580078, 244.53399658203125, 235.9770050048828, 238.89199829101562, 238.73500061035156, 237.47000122070312, 236.42599487304688, 253.8280029296875, 254.26300048828125, 260.2019958496094, 275.6700134277344, 281.7019958496094, 273.0920104980469, 276.1780090332031, 272.7229919433594, 276.260986328125, 274.35400390625, 289.60699462890625, 291.760009765625, 296.3789978027344, 294.35400390625, 285.3370056152344, 281.885009765625, 286.39300537109375, 290.5929870605469, 285.5050048828125, 256.29901123046875, 260.9280090332031, 261.7489929199219, 260.0249938964844, 267.9599914550781, 266.739990234375, 245.59500122070312, 246.19700622558594, 248.53199768066406, 247.0290069580078, 252.79800415039062, 242.71299743652344, 247.5260009765625, 244.2239990234375, 247.27200317382812] [234.8249969482422, 233.84300231933594, 243.61000061035156, 236.3260040283203, 240.2830047607422, 243.7790069580078, 244.53399658203125, 235.9770050048828, 238.89199829101562, 238.73500061035156, 237.47000122070312, 236.42599487304688, 253.8280029296875, 254.26300048828125, 260.2019958496094, 275.6700134277344, 281.7019958496094, 273.0920104980469, 276.1780090332031, 272.7229919433594, 276.260986328125, 274.35400390625, 289.60699462890625, 291.760009765625, 296.3789978027344, 294.35400390625, 285.3370056152344, 281.885009765625, 286.39300537109375, 290.5929870605469, 285.5050048828125, 256.29901123046875, 260.9280090332031, 261.7489929199219, 260.0249938964844, 267.9599914550781, 266.739990234375, 245.59500122070312, 246.19700622558594, 248.53199768066406, 247.0290069580078, 252.79800415039062, 242.71299743652344, 247.5260009765625, 244.2239990234375, 247.27200317382812] [56552400, 28153700, 27363100, 25200800, 18270500, 23876700, 12284200, 19527000, 16400000, 14200400, 11496200, 13619400, 44013900, 13949300, 25213700, 40465700, 50461300, 41383000, 41302400, 28918900, 17825900, 22067900, 59178200, 67770800, 33963900, 32585200, 31421500, 22612300, 11970100, 21516100, 21497200, 57008000, 52732000, 18456700, 17130100, 18438100, 22811900, 40073700, 35866900, 25730000, 17274900, 16040900, 21699400, 23009600, 22672000, 22877200] 59 | 60 | statement error 61 | select * FROM yahoo_finance(["oejoeaijeao"], "2015-01-01"::DATE, "2015-04-01"::DATE, "1d"); 62 | ---- 63 | 64 | statement error 65 | select * FROM yahoo_finance([], "2015-01-01"::DATE, "2015-04-01"::DATE, "1d"); 66 | ---- 67 | 68 | # Try Big Data 69 | query IIIIIIII 70 | select * FROM yahoo_finance(["BTC-USD", "^GSPC"], "2015-01-01"::DATE, "2015-04-01"::DATE, "1d"); 71 | ---- 72 | BTC-USD [2015-01-01, 2015-01-02, 2015-01-03, 2015-01-04, 2015-01-05, 2015-01-06, 2015-01-07, 2015-01-08, 2015-01-09, 2015-01-10, 2015-01-11, 2015-01-12, 2015-01-13, 2015-01-14, 2015-01-15, 2015-01-16, 2015-01-17, 2015-01-18, 2015-01-19, 2015-01-20, 2015-01-21, 2015-01-22, 2015-01-23, 2015-01-24, 2015-01-25, 2015-01-26, 2015-01-27, 2015-01-28, 2015-01-29, 2015-01-30, 2015-01-31, 2015-02-01, 2015-02-02, 2015-02-03, 2015-02-04, 2015-02-05, 2015-02-06, 2015-02-07, 2015-02-08, 2015-02-09, 2015-02-10, 2015-02-11, 2015-02-12, 2015-02-13, 2015-02-14, 2015-02-15] [320.43499755859375, 314.0790100097656, 314.84600830078125, 281.14599609375, 265.0840148925781, 274.6109924316406, 286.0769958496094, 294.135009765625, 282.38299560546875, 287.3030090332031, 274.6080017089844, 266.14599609375, 267.3940124511719, 223.8939971923828, 176.89700317382812, 209.07000732421875, 207.83399963378906, 200.0500030517578, 211.4709930419922, 212.90699768066406, 211.3780059814453, 227.32200622558594, 233.51699829101562, 232.6999969482422, 247.3520050048828, 254.07899475097656, 273.1669921875, 263.35101318359375, 233.34800720214844, 232.77200317382812, 226.4409942626953, 216.86700439453125, 226.49099731445312, 237.45399475097656, 227.51100158691406, 227.6649932861328, 216.92300415039062, 222.63299560546875, 227.6929931640625, 223.38900756835938, 220.28199768066406, 219.73199462890625, 219.20799255371094, 221.968994140625, 235.5279998779297, 257.5069885253906] [320.43499755859375, 315.8389892578125, 315.1499938964844, 287.2300109863281, 278.34100341796875, 287.5530090332031, 298.7539978027344, 294.135009765625, 291.114013671875, 288.12701416015625, 279.63800048828125, 272.2030029296875, 268.2770080566406, 223.8939971923828, 229.06700134277344, 221.59100341796875, 211.7310028076172, 218.69500732421875, 216.72799682617188, 215.24099731445312, 227.78799438476562, 237.0189971923828, 234.84500122070312, 248.2100067138672, 255.07400512695312, 309.3840026855469, 275.4800109863281, 266.5350036621094, 238.70599365234375, 242.8509979248047, 233.50399780273438, 231.57400512695312, 242.1750030517578, 245.95700073242188, 230.05799865722656, 239.40499877929688, 230.50999450683594, 230.2989959716797, 229.43800354003906, 223.9770050048828, 221.8070068359375, 223.406005859375, 222.19900512695312, 240.25900268554688, 259.8080139160156, 265.6109924316406] [314.00299072265625, 313.56500244140625, 281.0820007324219, 257.61199951171875, 265.0840148925781, 272.6960144042969, 283.0790100097656, 282.17498779296875, 280.5329895019531, 273.96600341796875, 265.03900146484375, 265.20001220703125, 219.906005859375, 171.50999450683594, 176.89700317382812, 199.77099609375, 194.875, 194.50599670410156, 207.3179931640625, 205.1529998779297, 211.21200561523438, 226.4340057373047, 225.1959991455078, 230.02200317382812, 243.88999938964844, 254.07899475097656, 250.6529998779297, 227.04600524902344, 220.71200561523438, 225.83900451660156, 216.3090057373047, 212.01499938964844, 222.65899658203125, 224.48300170898438, 221.11300659179688, 214.72500610351562, 216.23199462890625, 222.60699462890625, 221.07699584960938, 217.0189971923828, 215.33200073242188, 218.07400512695312, 217.61399841308594, 221.26199340820312, 235.5279998779297, 227.6840057373047] [314.2489929199219, 315.0320129394531, 281.0820007324219, 264.19500732421875, 274.4739990234375, 286.1889953613281, 294.3370056152344, 283.3489990234375, 290.4079895019531, 274.7959899902344, 265.6600036621094, 267.7959899902344, 225.86099243164062, 178.10299682617188, 209.843994140625, 208.0970001220703, 199.25999450683594, 210.33900451660156, 214.86099243164062, 211.31500244140625, 226.89700317382812, 233.406005859375, 232.87899780273438, 247.8470001220703, 253.71800231933594, 273.4729919433594, 263.4750061035156, 233.9149932861328, 233.51300048828125, 226.4250030517578, 217.46400451660156, 226.9720001220703, 238.22900390625, 227.26800537109375, 226.85299682617188, 217.11099243164062, 222.26600646972656, 227.75399780273438, 223.41200256347656, 220.11000061035156, 219.83900451660156, 219.18499755859375, 221.76400756835938, 235.427001953125, 257.3210144042969, 234.8249969482422] [314.2489929199219, 315.0320129394531, 281.0820007324219, 264.19500732421875, 274.4739990234375, 286.1889953613281, 294.3370056152344, 283.3489990234375, 290.4079895019531, 274.7959899902344, 265.6600036621094, 267.7959899902344, 225.86099243164062, 178.10299682617188, 209.843994140625, 208.0970001220703, 199.25999450683594, 210.33900451660156, 214.86099243164062, 211.31500244140625, 226.89700317382812, 233.406005859375, 232.87899780273438, 247.8470001220703, 253.71800231933594, 273.4729919433594, 263.4750061035156, 233.9149932861328, 233.51300048828125, 226.4250030517578, 217.46400451660156, 226.9720001220703, 238.22900390625, 227.26800537109375, 226.85299682617188, 217.11099243164062, 222.26600646972656, 227.75399780273438, 223.41200256347656, 220.11000061035156, 219.83900451660156, 219.18499755859375, 221.76400756835938, 235.427001953125, 257.3210144042969, 234.8249969482422] [8036550, 7860650, 33054400, 55629100, 43962800, 23245700, 24866800, 19982500, 18718600, 15264300, 18200800, 18880300, 72843904, 97638704, 81773504, 38421000, 23469700, 30085100, 18658300, 24051100, 29924600, 33544600, 24621700, 24782500, 33582700, 106794000, 44399000, 44352200, 32213400, 26605200, 23348200, 29128500, 30612100, 40783700, 26594300, 22516400, 24435300, 21604200, 17145200, 27791300, 21115100, 17201900, 15206200, 42744400, 49732500, 56552400] 73 | BTC-USD [2015-02-15, 2015-02-16, 2015-02-17, 2015-02-18, 2015-02-19, 2015-02-20, 2015-02-21, 2015-02-22, 2015-02-23, 2015-02-24, 2015-02-25, 2015-02-26, 2015-02-27, 2015-02-28, 2015-03-01, 2015-03-02, 2015-03-03, 2015-03-04, 2015-03-05, 2015-03-06, 2015-03-07, 2015-03-08, 2015-03-09, 2015-03-10, 2015-03-11, 2015-03-12, 2015-03-13, 2015-03-14, 2015-03-15, 2015-03-16, 2015-03-17, 2015-03-18, 2015-03-19, 2015-03-20, 2015-03-21, 2015-03-22, 2015-03-23, 2015-03-24, 2015-03-25, 2015-03-26, 2015-03-27, 2015-03-28, 2015-03-29, 2015-03-30, 2015-03-31, 2015-04-01] [257.5069885253906, 234.8249969482422, 233.4219970703125, 243.77999877929688, 236.41000366210938, 240.25100708007812, 243.7519989013672, 244.54400634765625, 235.9949951171875, 238.9980010986328, 238.88999938964844, 237.33700561523438, 236.43600463867188, 253.52000427246094, 254.2830047607422, 260.35699462890625, 275.0459899902344, 281.989990234375, 272.739013671875, 275.6000061035156, 272.29400634765625, 276.4330139160156, 274.81201171875, 289.86199951171875, 291.5249938964844, 296.12701416015625, 294.1180114746094, 284.4419860839844, 281.42498779296875, 285.68499755859375, 290.5950012207031, 285.0669860839844, 255.8800048828125, 260.95599365234375, 261.6440124511719, 259.9169921875, 267.8949890136719, 266.5769958496094, 247.4720001220703, 246.2760009765625, 248.5659942626953, 246.97500610351562, 252.74000549316406, 242.87899780273438, 247.45399475097656, 244.22300720214844] [265.6109924316406, 239.52099609375, 245.77499389648438, 244.25100708007812, 242.6719970703125, 247.1009979248047, 255.32000732421875, 246.39199829101562, 240.10899353027344, 239.9010009765625, 239.33999633789062, 237.7100067138672, 256.65301513671875, 254.69200134277344, 261.6600036621094, 276.3009948730469, 285.7959899902344, 284.2250061035156, 281.6669921875, 277.6080017089844, 277.85400390625, 277.8580017089844, 292.70098876953125, 300.04400634765625, 297.3909912109375, 297.0880126953125, 294.49798583984375, 286.3420104980469, 286.52899169921875, 294.11199951171875, 292.364990234375, 285.33599853515625, 264.2439880371094, 264.8479919433594, 262.1960144042969, 269.74700927734375, 277.2969970703125, 267.00299072265625, 249.19000244140625, 254.35400390625, 256.8110046386719, 254.2050018310547, 253.13900756835938, 249.24200439453125, 248.72999572753906, 247.54100036621094] [227.6840057373047, 229.02200317382812, 232.31399536132812, 232.33999633789062, 235.5919952392578, 239.2989959716797, 243.1840057373047, 233.8509979248047, 232.42100524902344, 236.40199279785156, 235.52999877929688, 234.2570037841797, 236.43600463867188, 249.47900390625, 245.93299865722656, 258.31298828125, 268.1610107421875, 268.1260070800781, 264.7690124511719, 270.0150146484375, 270.13299560546875, 272.56500244140625, 273.89300537109375, 289.7430114746094, 290.50799560546875, 292.4129943847656, 285.3370056152344, 280.97601318359375, 280.9960021972656, 285.68499755859375, 284.3739929199219, 249.8699951171875, 248.63600158691406, 259.1619873046875, 255.64999389648438, 259.5899963378906, 261.7449951171875, 244.15499877929688, 236.51499938964844, 244.90499877929688, 245.21299743652344, 246.97500610351562, 240.85000610351562, 239.21400451660156, 242.73899841308594, 241.16000366210938] [234.8249969482422, 233.84300231933594, 243.61000061035156, 236.3260040283203, 240.2830047607422, 243.7790069580078, 244.53399658203125, 235.9770050048828, 238.89199829101562, 238.73500061035156, 237.47000122070312, 236.42599487304688, 253.8280029296875, 254.26300048828125, 260.2019958496094, 275.6700134277344, 281.7019958496094, 273.0920104980469, 276.1780090332031, 272.7229919433594, 276.260986328125, 274.35400390625, 289.60699462890625, 291.760009765625, 296.3789978027344, 294.35400390625, 285.3370056152344, 281.885009765625, 286.39300537109375, 290.5929870605469, 285.5050048828125, 256.29901123046875, 260.9280090332031, 261.7489929199219, 260.0249938964844, 267.9599914550781, 266.739990234375, 245.59500122070312, 246.19700622558594, 248.53199768066406, 247.0290069580078, 252.79800415039062, 242.71299743652344, 247.5260009765625, 244.2239990234375, 247.27200317382812] [234.8249969482422, 233.84300231933594, 243.61000061035156, 236.3260040283203, 240.2830047607422, 243.7790069580078, 244.53399658203125, 235.9770050048828, 238.89199829101562, 238.73500061035156, 237.47000122070312, 236.42599487304688, 253.8280029296875, 254.26300048828125, 260.2019958496094, 275.6700134277344, 281.7019958496094, 273.0920104980469, 276.1780090332031, 272.7229919433594, 276.260986328125, 274.35400390625, 289.60699462890625, 291.760009765625, 296.3789978027344, 294.35400390625, 285.3370056152344, 281.885009765625, 286.39300537109375, 290.5929870605469, 285.5050048828125, 256.29901123046875, 260.9280090332031, 261.7489929199219, 260.0249938964844, 267.9599914550781, 266.739990234375, 245.59500122070312, 246.19700622558594, 248.53199768066406, 247.0290069580078, 252.79800415039062, 242.71299743652344, 247.5260009765625, 244.2239990234375, 247.27200317382812] [56552400, 28153700, 27363100, 25200800, 18270500, 23876700, 12284200, 19527000, 16400000, 14200400, 11496200, 13619400, 44013900, 13949300, 25213700, 40465700, 50461300, 41383000, 41302400, 28918900, 17825900, 22067900, 59178200, 67770800, 33963900, 32585200, 31421500, 22612300, 11970100, 21516100, 21497200, 57008000, 52732000, 18456700, 17130100, 18438100, 22811900, 40073700, 35866900, 25730000, 17274900, 16040900, 21699400, 23009600, 22672000, 22877200] 74 | ^GSPC [2015-01-02, 2015-01-05, 2015-01-06, 2015-01-07, 2015-01-08, 2015-01-09, 2015-01-12, 2015-01-13, 2015-01-14, 2015-01-15, 2015-01-16, 2015-01-20, 2015-01-21, 2015-01-22, 2015-01-23, 2015-01-26, 2015-01-27, 2015-01-28, 2015-01-29, 2015-01-30, 2015-02-02, 2015-02-03, 2015-02-04, 2015-02-05, 2015-02-06, 2015-02-09, 2015-02-10, 2015-02-11, 2015-02-12, 2015-02-13] [2058.89990234375, 2054.43994140625, 2022.1500244140625, 2005.550048828125, 2030.6099853515625, 2063.449951171875, 2046.1300048828125, 2031.5799560546875, 2018.4000244140625, 2013.75, 1992.25, 2020.760009765625, 2020.18994140625, 2034.300048828125, 2062.97998046875, 2050.419921875, 2047.8599853515625, 2032.3399658203125, 2002.449951171875, 2019.3499755859375, 1996.6700439453125, 2022.7099609375, 2048.860107421875, 2043.449951171875, 2062.280029296875, 2053.469970703125, 2049.3798828125, 2068.550048828125, 2069.97998046875, 2088.780029296875] [2072.360107421875, 2054.43994140625, 2030.25, 2029.6099853515625, 2064.080078125, 2064.429931640625, 2049.300048828125, 2056.929931640625, 2018.4000244140625, 2021.3499755859375, 2020.4599609375, 2028.93994140625, 2038.2900390625, 2064.6201171875, 2062.97998046875, 2057.6201171875, 2047.8599853515625, 2042.489990234375, 2024.6400146484375, 2023.3199462890625, 2021.6600341796875, 2050.300048828125, 2054.739990234375, 2063.550048828125, 2072.39990234375, 2056.159912109375, 2070.860107421875, 2073.47998046875, 2088.530029296875, 2097.030029296875] [2046.0400390625, 2017.3399658203125, 1992.43994140625, 2005.550048828125, 2030.6099853515625, 2038.3299560546875, 2022.5799560546875, 2008.25, 1988.43994140625, 1991.469970703125, 1988.1199951171875, 2004.489990234375, 2012.0400390625, 2026.3800048828125, 2050.5400390625, 2040.969970703125, 2019.9100341796875, 2001.489990234375, 1989.1800537109375, 1993.3800048828125, 1980.9000244140625, 2022.7099609375, 2036.719970703125, 2043.449951171875, 2049.969970703125, 2041.8800048828125, 2048.6201171875, 2057.989990234375, 2069.97998046875, 2086.699951171875] [2058.199951171875, 2020.5799560546875, 2002.6099853515625, 2025.9000244140625, 2062.139892578125, 2044.81005859375, 2028.260009765625, 2023.030029296875, 2011.27001953125, 1992.6700439453125, 2019.4200439453125, 2022.550048828125, 2032.1199951171875, 2063.14990234375, 2051.820068359375, 2057.090087890625, 2029.550048828125, 2002.1600341796875, 2021.25, 1994.989990234375, 2020.8499755859375, 2050.030029296875, 2041.510009765625, 2062.52001953125, 2055.469970703125, 2046.739990234375, 2068.590087890625, 2068.530029296875, 2088.47998046875, 2096.989990234375] [2058.199951171875, 2020.5799560546875, 2002.6099853515625, 2025.9000244140625, 2062.139892578125, 2044.81005859375, 2028.260009765625, 2023.030029296875, 2011.27001953125, 1992.6700439453125, 2019.4200439453125, 2022.550048828125, 2032.1199951171875, 2063.14990234375, 2051.820068359375, 2057.090087890625, 2029.550048828125, 2002.1600341796875, 2021.25, 1994.989990234375, 2020.8499755859375, 2050.030029296875, 2041.510009765625, 2062.52001953125, 2055.469970703125, 2046.739990234375, 2068.590087890625, 2068.530029296875, 2088.47998046875, 2096.989990234375] [2708700000, 3799120000, 4460110000, 3805480000, 3934010000, 3364140000, 3456460000, 4107300000, 4378680000, 4276720000, 4056410000, 3944340000, 3730070000, 4176050000, 3573560000, 3465760000, 3329810000, 4067530000, 4127140000, 4568650000, 4008330000, 4615900000, 4141920000, 3821990000, 4232970000, 3549540000, 3669850000, 3596860000, 3788350000, 3527450000] 75 | ^GSPC [2015-02-17, 2015-02-18, 2015-02-19, 2015-02-20, 2015-02-23, 2015-02-24, 2015-02-25, 2015-02-26, 2015-02-27, 2015-03-02, 2015-03-03, 2015-03-04, 2015-03-05, 2015-03-06, 2015-03-09, 2015-03-10, 2015-03-11, 2015-03-12, 2015-03-13, 2015-03-16, 2015-03-17, 2015-03-18, 2015-03-19, 2015-03-20, 2015-03-23, 2015-03-24, 2015-03-25, 2015-03-26, 2015-03-27, 2015-03-30, 2015-03-31] [2096.469970703125, 2099.159912109375, 2099.25, 2097.64990234375, 2109.830078125, 2109.10009765625, 2115.300048828125, 2113.909912109375, 2110.8798828125, 2105.22998046875, 2115.760009765625, 2107.719970703125, 2098.5400390625, 2100.909912109375, 2072.25, 2076.139892578125, 2044.68994140625, 2041.0999755859375, 2064.56005859375, 2055.35009765625, 2080.590087890625, 2072.840087890625, 2098.68994140625, 2090.320068359375, 2107.989990234375, 2103.93994140625, 2093.10009765625, 2059.93994140625, 2055.780029296875, 2064.110107421875, 2084.050048828125] [2101.300048828125, 2100.22998046875, 2102.1298828125, 2110.610107421875, 2110.050048828125, 2117.93994140625, 2119.590087890625, 2113.909912109375, 2112.739990234375, 2117.52001953125, 2115.760009765625, 2107.719970703125, 2104.25, 2100.909912109375, 2083.489990234375, 2076.139892578125, 2050.080078125, 2066.409912109375, 2064.56005859375, 2081.409912109375, 2080.590087890625, 2106.85009765625, 2098.68994140625, 2113.919921875, 2114.860107421875, 2107.6298828125, 2097.429931640625, 2067.14990234375, 2062.830078125, 2088.969970703125, 2084.050048828125] [2089.800048828125, 2092.14990234375, 2090.7900390625, 2085.43994140625, 2103.0, 2105.8701171875, 2109.889892578125, 2103.760009765625, 2103.75, 2104.5, 2098.260009765625, 2094.489990234375, 2095.219970703125, 2067.27001953125, 2072.2099609375, 2044.1600341796875, 2039.68994140625, 2041.0999755859375, 2041.1700439453125, 2055.35009765625, 2065.080078125, 2061.22998046875, 2085.56005859375, 2090.320068359375, 2104.419921875, 2091.5, 2061.050048828125, 2045.5, 2052.9599609375, 2064.110107421875, 2067.0400390625] [2100.340087890625, 2099.679931640625, 2097.449951171875, 2110.300048828125, 2109.659912109375, 2115.47998046875, 2113.860107421875, 2110.739990234375, 2104.5, 2117.389892578125, 2107.780029296875, 2098.530029296875, 2101.0400390625, 2071.260009765625, 2079.429931640625, 2044.1600341796875, 2040.239990234375, 2065.949951171875, 2053.39990234375, 2081.18994140625, 2074.280029296875, 2099.5, 2089.27001953125, 2108.10009765625, 2104.419921875, 2091.5, 2061.050048828125, 2056.14990234375, 2061.02001953125, 2086.239990234375, 2067.889892578125] [2100.340087890625, 2099.679931640625, 2097.449951171875, 2110.300048828125, 2109.659912109375, 2115.47998046875, 2113.860107421875, 2110.739990234375, 2104.5, 2117.389892578125, 2107.780029296875, 2098.530029296875, 2101.0400390625, 2071.260009765625, 2079.429931640625, 2044.1600341796875, 2040.239990234375, 2065.949951171875, 2053.39990234375, 2081.18994140625, 2074.280029296875, 2099.5, 2089.27001953125, 2108.10009765625, 2104.419921875, 2091.5, 2061.050048828125, 2056.14990234375, 2061.02001953125, 2086.239990234375, 2067.889892578125] [3361750000, 3370020000, 3247100000, 3281600000, 3093680000, 3199840000, 3312340000, 3408690000, 3547380000, 3409490000, 3262300000, 3421110000, 3103030000, 3853570000, 3349090000, 3668900000, 3406570000, 3405860000, 3498560000, 3295600000, 3221840000, 4128210000, 3305220000, 5554120000, 3267960000, 3189820000, 3521140000, 3510670000, 3008550000, 2917690000, 3376550000] 76 | 77 | # Interval must be a string accepted by the yahoo api 78 | statement error 79 | select * FROM yahoo_finance("^GSPC", "2017-12-11"::DATE, "2017-12-12"::DATE, "bla") 80 | ---- 81 | 82 | # Periods can't be equal 83 | statement error 84 | select * FROM yahoo_finance("^GSPC", "2017-12-11"::DATE, "2017-12-11"::DATE, "1d") 85 | ---- 86 | 87 | # Interval can't be bigger than period 88 | statement error 89 | select * FROM yahoo_finance("^GSPC", "2017-12-11"::DATE, "2017-12-15"::DATE, "1mo") 90 | ---- 91 | 92 | 93 | # Start period must be castable to date 94 | statement error 95 | select * FROM yahoo_finance("^GSPC", "bla", "2017-12-15"::DATE, "1d") 96 | ---- 97 | 98 | # End period must be castable to date 99 | statement error 100 | select * FROM yahoo_finance("^GSPC", "2017-12-15"::DATE, "bla", "1d") 101 | ---- 102 | 103 | query I 104 | select list_aggregate(close, 'volatility') FROM yahoo_finance("^GSPC", "2017-12-01"::DATE, "2017-12-10"::DATE, "1d") 105 | ---- 106 | 7.640770797901039 107 | 108 | query I 109 | select list_aggregate(close, 'volatility') FROM yahoo_finance("^GSPC", "2017-12-01", "2017-12-10", "1d") 110 | ---- 111 | 7.640770797901039 112 | 113 | query I 114 | select list_aggregate(close, 'sma') FROM yahoo_finance("^GSPC", "2017-12-01"::DATE, "2017-12-10"::DATE, "1d") 115 | ---- 116 | 2638.16333 117 | 118 | query I 119 | select list_aggregate(close, 'volatility') FROM yahoo_finance("^GSPC", "2023-01-01"::DATE, "2023-02-18"::DATE, "1d") 120 | ---- 121 | 101.31567537720929 122 | 123 | query I 124 | select list_aggregate(close, 'volatility') FROM yahoo_finance("BTC-USD", "2023-01-01"::DATE, "2023-02-18"::DATE, "1d"); 125 | ---- 126 | 2535.839600150706 127 | -------------------------------------------------------------------------------- /vcpkg.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": [ 3 | "openssl" 4 | ] 5 | } --------------------------------------------------------------------------------