├── tests └── input_simple.json ├── .gitignore ├── package.json ├── .github └── workflows │ └── deploy.yml ├── LICENSE ├── serverless.yml ├── handler.js └── README.md /tests/input_simple.json: -------------------------------------------------------------------------------- 1 | { 2 | "body": "{ \"data\": [ [ 0, \"https://google.com/\" ] ] }" 3 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # package directories 2 | node_modules 3 | jspm_packages 4 | 5 | # Serverless directories 6 | .serverless 7 | 8 | snowflake-conf.sh -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "axios": "^0.21.1", 4 | "lodash-core": "^4.17.19", 5 | "serverless-snowflake-external-function-plugin": "^0.1.5" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: Deploy master branch 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | 9 | jobs: 10 | deploy: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - uses: actions/setup-node@v1 15 | with: 16 | node-version: '12.x' 17 | - name: Install Serverless Framework 18 | run: npm install -g serverless 19 | - name: Install NPM dependencies 20 | run: npm ci 21 | - name: Test deployment and removing 22 | run: sls deploy -s ci && sls remove -s ci 23 | env: 24 | AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} 25 | AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 26 | SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} 27 | SNOWFLAKE_USERNAME: ci_serverless 28 | SNOWFLAKE_DATABASE: test_db 29 | SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} 30 | SNOWFLAKE_ROLE: CI_TEST_DB 31 | SNOWFLAKE_SCHEMA: dev -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2020, Starschema Limited 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /serverless.yml: -------------------------------------------------------------------------------- 1 | service: starsnow-request 2 | app: starsnow-request 3 | #org: starschema 4 | 5 | plugins: 6 | - serverless-snowflake-external-function-plugin 7 | 8 | 9 | provider: 10 | name: aws 11 | runtime: nodejs12.x 12 | apiGateway: 13 | shouldStartNameWithService: true 14 | 15 | stage: dev 16 | region: us-west-2 17 | 18 | 19 | custom: 20 | snowflake: 21 | role: ${env:SNOWFLAKE_ROLE} 22 | account: ${env:SNOWFLAKE_ACCOUNT} 23 | username: ${env:SNOWFLAKE_USERNAME} 24 | password: ${env:SNOWFLAKE_PASSWORD} 25 | warehouse: "" 26 | database: ${env:SNOWFLAKE_DATABASE} 27 | schema: ${env:SNOWFLAKE_SCHEMA} 28 | 29 | # you can add statements to the Lambda function's IAM Role here 30 | # iamRoleStatements: 31 | # - Effect: "Allow" 32 | # Action: 33 | # - "s3:ListBucket" 34 | # Resource: { "Fn::Join" : ["", ["arn:aws:s3:::", { "Ref" : "ServerlessDeploymentBucket" } ] ] } 35 | # - Effect: "Allow" 36 | # Action: 37 | # - "s3:PutObject" 38 | # Resource: 39 | # Fn::Join: 40 | # - "" 41 | # - - "arn:aws:s3:::" 42 | # - "Ref" : "ServerlessDeploymentBucket" 43 | # - "/*" 44 | 45 | # you can define service wide environment variables here 46 | # environment: 47 | # variable1: value1 48 | 49 | # you can add packaging information here 50 | package: 51 | patterns: 52 | - "!snowflake-conf.sh" 53 | 54 | functions: 55 | 56 | starsnow_request: 57 | handler: handler.starsnowRequest 58 | snowflake: 59 | argument_signature: (url VARCHAR2, params OBJECT) 60 | data_type: variant 61 | events: 62 | - http: 63 | path: starsnow_request 64 | method: post 65 | authorizer: aws_iam 66 | 67 | 68 | starsnow_request_get: 69 | handler: handler.starsnowRequestGet 70 | snowflake: 71 | argument_signature: (url VARCHAR) 72 | data_type: VARCHAR 73 | events: 74 | - http: 75 | path: starsnow_request_get 76 | method: post 77 | authorizer: aws_iam 78 | 79 | # - websocket: $connect 80 | # - s3: ${env:BUCKET} 81 | # - schedule: rate(10 minutes) 82 | # - sns: greeter-topic 83 | # - stream: arn:aws:dynamodb:region:XXXXXX:table/foo/stream/1970-01-01T00:00:00.000 84 | # - alexaSkill: amzn1.ask.skill.xx-xx-xx-xx 85 | # - alexaSmartHome: amzn1.ask.skill.xx-xx-xx-xx 86 | # - iot: 87 | # sql: "SELECT * FROM 'some_topic'" 88 | # - cloudwatchEvent: 89 | # event: 90 | # source: 91 | # - "aws.ec2" 92 | # detail-type: 93 | # - "EC2 Instance State-change Notification" 94 | # detail: 95 | # state: 96 | # - pending 97 | # - cloudwatchLog: '/aws/lambda/hello' 98 | # - cognitoUserPool: 99 | # pool: MyUserPool 100 | # trigger: PreSignUp 101 | # - alb: 102 | # listenerArn: arn:aws:elasticloadbalancing:us-east-1:XXXXXX:listener/app/my-load-balancer/50dc6c495c0c9188/ 103 | # priority: 1 104 | # conditions: 105 | # host: example.com 106 | # path: /hello 107 | 108 | 109 | # you can add CloudFormation resource templates here 110 | #resources: 111 | # Resources: 112 | # NewResource: 113 | # Type: AWS::S3::Bucket 114 | # Properties: 115 | # BucketName: my-new-bucket 116 | # Outputs: 117 | # NewOutput: 118 | # Description: "Description for the output" 119 | # Value: "Some output value" 120 | -------------------------------------------------------------------------------- /handler.js: -------------------------------------------------------------------------------- 1 | // BSD 3-Clause License 2 | 3 | // Copyright (c) 2020, Starschema Limited 4 | // All rights reserved. 5 | 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are met: 8 | 9 | // 1. Redistributions of source code must retain the above copyright notice, this 10 | // list of conditions and the following disclaimer. 11 | 12 | // 2. Redistributions in binary form must reproduce the above copyright notice, 13 | // this list of conditions and the following disclaimer in the documentation 14 | // and/or other materials provided with the distribution. 15 | 16 | // 3. Neither the name of the copyright holder nor the names of its 17 | // contributors may be used to endorse or promote products derived from 18 | // this software without specific prior written permission. 19 | 20 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | 'use strict'; 32 | 33 | var _ = require('lodash/core'); 34 | const axios = require('axios'); 35 | 36 | /** 37 | * 38 | * Call axios with the user supplied url and parameters. 39 | * 40 | * More informaion about axios config (params) can be found [here](https://github.com/axios/axios#request-config) 41 | * 42 | * @param {string} url URL to invoke 43 | * @param {object} params axios config object 44 | * @returns {Promise} Promise object that contains the entire axios response 45 | */ 46 | const makeRequest = (url, params) => { 47 | const config = _.defaults(params, { method: 'get' }, { url: url }) 48 | 49 | if (!config['url']) { 50 | return Promise.reject({ errorCode: 1, message: `URL parameter cannot be empty.` }) 51 | } 52 | 53 | return axios(config) 54 | .then(response => { 55 | return _.pick(response, ["headers", "status", "statusText", "data"]) 56 | }) 57 | .catch(error => { 58 | return { 59 | errorCode: 2 60 | , message: `cannot retrieve url "${url}": ${error}.` 61 | , request: params 62 | } 63 | }) 64 | 65 | } 66 | 67 | module.exports.starsnowRequest = async event => { 68 | const body = JSON.parse(event.body); 69 | 70 | 71 | return await Promise.all( 72 | 73 | body.data.map((row) => makeRequest(row[1], row[2])) 74 | 75 | ).then((ret) => { 76 | 77 | return { 78 | statusCode: 200, 79 | body: JSON.stringify( 80 | { 81 | data: ret.map((v, idx) => [idx, v]) 82 | } 83 | ) 84 | } 85 | 86 | }).catch(error => { 87 | return { 88 | statusCode: 500, 89 | body: JSON.stringify(error) 90 | } 91 | }) 92 | 93 | } 94 | 95 | module.exports.starsnowRequestGet = async event => { 96 | const body = JSON.parse(event.body); 97 | 98 | 99 | return await Promise.all( 100 | 101 | body.data.map((row) => makeRequest(row[1], {})) 102 | 103 | ).then((ret) => { 104 | 105 | return { 106 | statusCode: 200, 107 | body: JSON.stringify( 108 | { 109 | data: ret.map((v, idx) => [idx, v.data]) 110 | }, 111 | ) 112 | } 113 | 114 | }).catch(error => { 115 | return { 116 | statusCode: 500, 117 | body: JSON.stringify(error) 118 | } 119 | }) 120 | 121 | } 122 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # StarSnow Request ![Integration tests](https://github.com/starschema/starsnow_request/workflows/Deploy%20master%20branch/badge.svg) 2 | 3 | HTTP Client for Snowflake database (HTTP get/post from SQL) 4 | 5 | The idea is simple: invoke HTTP/HTTPS web servers directly from Snowflake Database SQL statements using pre-deployed, generic external functions. Built on top of [Axios](https://github.com/axios/axios). 6 | 7 | Detailed blog post on usage: https://medium.com/starschema-blog/starsnow-http-client-for-snowflake-sql-e1b329293fc6 8 | 9 | ## Examples 10 | 11 | Simple HTTP get. varchar input (URL), varchar return (content of the site): 12 | 13 | ```sql 14 | select STARSNOW_REQUEST_GET('https://google.com/') as google_content; 15 | ``` 16 | 17 | Get Snowflake system status. Select value from the result `variant`: 18 | 19 | ```sql 20 | select STARSNOW_REQUEST('https://status.snowflake.com/api/v2/status.json', NULL):data:status:description as snowflake_status; 21 | ``` 22 | ![](https://user-images.githubusercontent.com/82426/103464558-df01b000-4d34-11eb-8228-4e7d16e81875.png) 23 | 24 | Get Snowflake historical stock values: 25 | 26 | ```sql 27 | select key, value:"4. close"::float 28 | from TABLE (FLATTEN( 29 | input => STARSNOW_REQUEST('https://www.alphavantage.co/query', 30 | object_construct('method', 'get', 31 | 'params', object_construct( 32 | 'function', 'TIME_SERIES_DAILY_ADJUSTED', 33 | 'symbol', 'SNOW', 34 | 'apikey', '')) 35 | ): data:"Time Series (Daily)")); 36 | ``` 37 | ![](https://user-images.githubusercontent.com/82426/103464555-db6e2900-4d34-11eb-82f5-cc54112243cb.png) 38 | 39 | ## StarSnow Request API 40 | 41 | The package contains two functions: `STARSNOW_REQUEST_GET` and `STARSNOW_REQUEST`. 42 | 43 | **`varchar STARSNOW_REQUEST_GET(url VARCHAR)`** 44 | 45 | The function takes one string argument (url) and returns with the content of that web address. It supports only `get` method and no custom headers. 46 | 47 | **`variant STARSNOW_REQUEST(url VARCHAR, params OBJECT)`** 48 | 49 | The function takes and url and an parameters object that passed as an [axios configuration](https://github.com/axios/axios#request-config) to request. The following configuration properties are supported: 50 | 51 | 52 | ```sql 53 | 54 | -- params object: 55 | 56 | object_construct( 57 | -- `url` is the server URL that will be used for the request 58 | 'url', '/user', 59 | 60 | -- `method` is the request method to be used when making the request 61 | 'method', 'get', -- default 62 | 63 | -- `headers` are custom headers to be sent 64 | 'headers', object_construct('X-Requested-With', 'XMLHttpRequest'), 65 | 66 | -- `params` are the URL parameters to be sent with the request 67 | -- Must be a plain object 68 | 'params', object_construct( 69 | 'ID', 12345 70 | ), 71 | 72 | -- `data` is the data to be sent as the request body 73 | -- Only applicable for request methods 'PUT', 'POST', 'DELETE , and 'PATCH' 74 | 'data', object_construct( 75 | 'firstName', 'Fred' 76 | ), 77 | 78 | -- syntax alternative to send data into the body 79 | -- method post 80 | -- only the value is sent, not the key 81 | 'data', 'Country=Brasil&City=Belo Horizonte', 82 | 83 | -- `timeout` specifies the number of milliseconds before the request times out. 84 | -- If the request takes longer than `timeout`, the request will be aborted. 85 | 'timeout', 1000, -- default is `0` (no timeout) 86 | 87 | -- `auth` indicates that HTTP Basic auth should be used, and supplies credentials. 88 | -- This will set an `Authorization` header, overwriting any existing 89 | -- `Authorization` custom headers you have set using `headers`. 90 | -- Please note that only HTTP Basic auth is configurable through this parameter. 91 | -- For Bearer tokens and such, use `Authorization` custom headers instead. 92 | 'auth', object_construct( 93 | 'username', 'janedoe', 94 | 'password', 's00pers3cret' 95 | ), 96 | 97 | -- `responseType` indicates the type of data that the server will respond with 98 | -- options are: 'arraybuffer', 'document', 'json', 'text', 99 | 'responseType', 'json', -- default 100 | 101 | -- `responseEncoding` indicates encoding to use for decoding responses 102 | 'responseEncoding', 'utf8', -- default 103 | 104 | -- `maxContentLength` defines the max size of the http response content in bytes allowed in node.js 105 | 'maxContentLength', 2000, 106 | 107 | -- `maxBodyLength` defines the max size of the http request content in bytes allowed 108 | 'maxBodyLength', 2000, 109 | 110 | -- `maxRedirects` defines the maximum number of redirects to follow in node.js. 111 | -- If set to 0, no redirects will be followed. 112 | 'maxRedirects', 5, -- default 113 | 114 | -- `proxy` defines the hostname, port, and protocol of the proxy server. 115 | -- You can also define your proxy using the conventional `http_proxy` and 116 | -- `https_proxy` environment variables. If you are using environment variables 117 | -- for your proxy configuration, you can also define a `no_proxy` environment 118 | -- variable as a comma-separated list of domains that should not be proxied. 119 | -- Use `false` to disable proxies, ignoring environment variables. 120 | -- `auth` indicates that HTTP Basic auth should be used to connect to the proxy, and 121 | -- supplies credentials. 122 | -- This will set an `Proxy-Authorization` header, overwriting any existing 123 | -- `Proxy-Authorization` custom headers you have set using `headers`. 124 | -- If the proxy server uses HTTPS, then you must set the protocol to `https`. 125 | 'proxy', object_construct( 126 | 'protocol', 'https', 127 | 'host', '127.0.0.1', 128 | 'port', 9000, 129 | 'auth', object_construct( 130 | 'username', 'mikeymike', 131 | 'password', 'rapunz3l' 132 | ), 133 | ), 134 | 135 | 136 | -- `decompress` indicates whether or not the response body should be decompressed 137 | -- automatically. If set to `true` will also remove the 'content-encoding' header 138 | -- from the responses objects of all decompressed responses 139 | 'decompress', true -- default 140 | ) 141 | ``` 142 | 143 | See examples on how to construct snowflake objects from SQL statements. 144 | 145 | On successful execution, the status code, headers and data are returned in single variant. 146 | 147 | 148 | ## Deploying 149 | 150 | To deploy, clone the repo, then: 151 | 152 | ``` 153 | $ npm install -g serverless 154 | $ npm install 155 | $ vim serverless.yml # edit your snowflake account details 156 | $ sls deploy 157 | ``` 158 | 159 | Then, the functions should be available in your snowflake account. 160 | 161 | ## License, Copyright 162 | 163 | BSD License, Starschema Inc, 2020 164 | --------------------------------------------------------------------------------