├── DeploymentTemplates ├── aws │ ├── BasicSetup.yaml │ └── README.md └── azure │ ├── BasicSetup.json │ └── README.md ├── ExtractEntitiesFromArticle ├── 1_entityExtraction.py ├── 2_entityExtraction.sql ├── entityExtractio_WithStockPrices.sql ├── example.png └── readme.md ├── FetchHTTPData ├── 1_handler.py ├── 2_fetchHTTPData.sql └── readme.md ├── FetchStockPricesAndIndicators ├── config.py ├── examplePayload.json ├── priceData.py ├── priceData.sql └── readme.md ├── LICENSE ├── MicrosoftTranslatorAPI_Python ├── ef_translate.py ├── ef_translate.sql ├── function.json ├── readme.md ├── requirements.txt └── sample.dat ├── ParseUserAgentString ├── 1_Setup.sql ├── 2_parseUserAgent.py ├── 3_parseUserAgent.sql └── readme.md ├── README.md ├── SendSNS ├── 1_Setup.sql ├── 2_LambdaCode.py ├── 3_Invocation.sql └── readme.md └── _Template_Python ├── example.sql ├── readme.md ├── template_python.py └── testData.json /DeploymentTemplates/aws/BasicSetup.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | Description: "Template for creating API Gateway and Lambda function for Snowflake external functions" 3 | 4 | Parameters: 5 | apiGatewayStageName: 6 | Type: "String" 7 | AllowedPattern: "^[-a-z0-9]+$" 8 | Default: "ext-func-stage" 9 | Description: "API deployment stage" 10 | 11 | lambdaName: 12 | Type: "String" 13 | AllowedPattern: "^[a-zA-Z0-9]+[-a-zA-Z0-9-]+[-a-zA-Z0-9]+$" 14 | Default: "ext-func-lambda" 15 | Description: "Lambda instance name" 16 | 17 | apiGatewayType: 18 | Type: "String" 19 | Default: "REGIONAL" 20 | AllowedValues: 21 | - "REGIONAL" 22 | - "PRIVATE" 23 | Description: "API Gateway type to create" 24 | 25 | apiGatewayName: 26 | Type: "String" 27 | AllowedPattern: "^[a-zA-Z0-9]+[-a-zA-Z0-9-]+[-a-zA-Z0-9]+$" 28 | Default: "ext-func-api" 29 | Description: "API Gateway instance name" 30 | 31 | apiGatewayIAMRoleName: 32 | Type: "String" 33 | AllowedPattern: "^[a-zA-Z0-9]+[-a-zA-Z0-9-]+[-a-zA-Z0-9]+$" 34 | Description: "Role used with Snowflake API Integration" 35 | 36 | lambdaExecutionRoleName: 37 | Type: "String" 38 | AllowedPattern: "^[a-zA-Z0-9]+[-a-zA-Z0-9-]+[-a-zA-Z0-9]+$" 39 | Description: "Role used by the Lambda." 40 | 41 | sourceVpcId: 42 | Type: "String" 43 | Default: "" 44 | Description: "Snowflake VPC that has access to private API Gateway. Used only when creating a private API Gateway" 45 | 46 | Conditions: 47 | shouldCreateRegionalGateway: 48 | !Equals [!Ref apiGatewayType, "REGIONAL"] 49 | 50 | Resources: 51 | apiGateway: 52 | Type: "AWS::ApiGateway::RestApi" 53 | DependsOn : apiIAMRole 54 | Properties: 55 | Name: !Ref "apiGatewayName" 56 | Description: "Snowflake external functions Gateway" 57 | Policy: !Sub 58 | - '{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "AWS": "arn:aws:sts::${AWS::AccountId}:assumed-role/${apiGatewayIAMRoleName}/snowflake" }, "Action": "execute-api:Invoke", "Resource": "${resourceArn}" , "Condition": { ${vpcCondition} } } ] }' 59 | - resourceArn : !Join [ "", [ "execute-api:/", "*" ] ] 60 | vpcCondition: !If 61 | - shouldCreateRegionalGateway 62 | - "" 63 | - !Sub '"StringEquals": { "aws:sourceVpc": "${sourceVpcId}"}' 64 | EndpointConfiguration: 65 | Types: 66 | - !Ref apiGatewayType 67 | 68 | apiResource: 69 | Type: 'AWS::ApiGateway::Resource' 70 | Properties: 71 | RestApiId: !Ref apiGateway 72 | ParentId: !GetAtt 73 | - apiGateway 74 | - RootResourceId 75 | PathPart: echo 76 | 77 | apiGatewayRootMethod: 78 | Type: "AWS::ApiGateway::Method" 79 | Properties: 80 | AuthorizationType: "AWS_IAM" 81 | HttpMethod: "POST" 82 | Integration: 83 | IntegrationHttpMethod: "POST" 84 | Type: "AWS_PROXY" 85 | Uri: !Sub 86 | - "arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${lambdaArn}/invocations" 87 | - lambdaArn: !GetAtt "lambdaFunction.Arn" 88 | ResourceId: !Ref "apiResource" 89 | RestApiId: !Ref "apiGateway" 90 | 91 | apiGatewayDeployment: 92 | Type: "AWS::ApiGateway::Deployment" 93 | DependsOn: 94 | - "apiGatewayRootMethod" 95 | Properties: 96 | RestApiId: !Ref "apiGateway" 97 | StageName: !Ref "apiGatewayStageName" 98 | 99 | lambdaFunction: 100 | Type: "AWS::Lambda::Function" 101 | DependsOn : lambdaExecutionIAMRole 102 | Properties: 103 | Code: 104 | ZipFile: | 105 | import json 106 | 107 | def handler(event, context): 108 | status_code = 200 109 | array_of_rows_to_return = [] 110 | 111 | try: 112 | event_body = event["body"] 113 | 114 | payload = json.loads(event_body) 115 | rows = payload["data"] 116 | 117 | for row in rows: 118 | row_number = row[0] 119 | 120 | input_value_1 = row[1] 121 | 122 | input_value_2 = row[2] 123 | 124 | output_value = ["Echoing inputs:", input_value_1, input_value_2] 125 | 126 | row_to_return = [row_number, output_value] 127 | 128 | array_of_rows_to_return.append(row_to_return) 129 | 130 | json_compatible_string_to_return = json.dumps({"data" : array_of_rows_to_return}) 131 | 132 | except Exception as err: 133 | status_code = 400 134 | json_compatible_string_to_return = event_body 135 | 136 | return { 137 | 'statusCode': status_code, 138 | 'body': json_compatible_string_to_return 139 | } 140 | Description: "Echo Lambda created from CloudFormation template" 141 | FunctionName: !Ref "lambdaName" 142 | Handler: "index.handler" 143 | Role: !GetAtt "lambdaExecutionIAMRole.Arn" 144 | Runtime: "python3.7" 145 | Timeout: 10 146 | 147 | lambdaApiGatewayInvoke: 148 | Type: "AWS::Lambda::Permission" 149 | Properties: 150 | Action: "lambda:InvokeFunction" 151 | FunctionName: !GetAtt "lambdaFunction.Arn" 152 | Principal: "apigateway.amazonaws.com" 153 | SourceArn: !Sub "arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${apiGateway}/*/*/*" 154 | 155 | apiIAMRole: 156 | Type: "AWS::IAM::Role" 157 | Properties: 158 | RoleName : !Ref "apiGatewayIAMRoleName" 159 | AssumeRolePolicyDocument: 160 | Version: "2012-10-17" 161 | Statement: 162 | - Action: 163 | - "sts:AssumeRole" 164 | Effect: "Allow" 165 | Principal: 166 | AWS: 167 | - !Sub "arn:aws:iam::${AWS::AccountId}:root" 168 | 169 | lambdaExecutionIAMRole: 170 | Type: "AWS::IAM::Role" 171 | Properties: 172 | RoleName : !Ref "lambdaExecutionRoleName" 173 | AssumeRolePolicyDocument: 174 | Version: "2012-10-17" 175 | Statement: 176 | - Action: 177 | - "sts:AssumeRole" 178 | Effect: "Allow" 179 | Principal: 180 | Service: 181 | - "lambda.amazonaws.com" 182 | ManagedPolicyArns: 183 | - "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" 184 | 185 | Outputs: 186 | resourceInvocationUrl: 187 | Value: !Sub "https://${apiGateway}.execute-api.${AWS::Region}.amazonaws.com/${apiGatewayStageName}/echo" 188 | 189 | awsRoleArn: 190 | Value: !GetAtt "apiIAMRole.Arn" -------------------------------------------------------------------------------- /DeploymentTemplates/aws/README.md: -------------------------------------------------------------------------------- 1 | # External Functions 2 | 3 | [AWS CloudFormation](https://aws.amazon.com/cloudformation/) template for setting up AWS API Gateway and Lambda for Snowflake external functions. 4 | 5 | Steps for creating Snowflake external function using this template: 6 | 1. Go to AWS cloudformation and create a stack using this template. 7 | 2. Note the Gateway IAM role and URL of the "echo" method created in the API Gateway. 8 | 3. Create API integration in Snowflake using the Gatway URL and Gateway Role ARN. 9 | 4. Update the API Gateway role trust relation with API integration's API_AWS_IAM_USER_ARN and API_AWS_EXTERNAL_ID. 10 | 5. Create and run the external function. -------------------------------------------------------------------------------- /DeploymentTemplates/azure/BasicSetup.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "apiManagementServiceName": { 6 | "type": "String" 7 | }, 8 | "functionAppName": { 9 | "type": "String" 10 | }, 11 | "storageAccountName": { 12 | "defaultValue": "[concat('sa', uniqueString(resourceGroup().id))]", 13 | "type": "String" 14 | }, 15 | "publisherEmail": { 16 | "type": "String" 17 | }, 18 | "azureadApplicationId": { 19 | "type": "String" 20 | } 21 | }, 22 | "variables": { 23 | "functionAppId": "[resourceId('Microsoft.Web/sites', parameters('functionAppName'))]" 24 | }, 25 | "resources": [ 26 | { 27 | "type": "Microsoft.ApiManagement/service", 28 | "apiVersion": "2019-12-01", 29 | "name": "[parameters('apiManagementServiceName')]", 30 | "location": "[resourceGroup().location]", 31 | "dependsOn": [ 32 | "[resourceId('Microsoft.Web/sites', parameters('functionAppName'))]" 33 | ], 34 | "sku": { 35 | "name": "Developer", 36 | "capacity": 1 37 | }, 38 | "properties": { 39 | "publisherEmail": "[parameters('publisherEmail')]", 40 | "publisherName": "[parameters('publisherEmail')]", 41 | "notificationSenderEmail": "apimgmt-noreply@mail.windowsazure.com", 42 | "hostnameConfigurations": [ 43 | { 44 | "type": "Proxy", 45 | "hostName": "[concat(parameters('apiManagementServiceName'), '.azure-api.net')]", 46 | "negotiateClientCertificate": false, 47 | "defaultSslBinding": true 48 | } 49 | ], 50 | "customProperties": { 51 | "Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Protocols.Tls10": "False", 52 | "Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Protocols.Tls11": "False", 53 | "Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Protocols.Ssl30": "False", 54 | "Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Ciphers.TripleDes168": "False", 55 | "Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Backend.Protocols.Tls10": "False", 56 | "Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Backend.Protocols.Tls11": "False", 57 | "Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Backend.Protocols.Ssl30": "False", 58 | "Microsoft.WindowsAzure.ApiManagement.Gateway.Protocols.Server.Http2": "False" 59 | }, 60 | "virtualNetworkType": "None", 61 | "disableGateway": false 62 | } 63 | }, 64 | { 65 | "type": "Microsoft.ApiManagement/service/apis", 66 | "apiVersion": "2019-12-01", 67 | "name": "[concat(parameters('apiManagementServiceName'), '/', parameters('apiManagementServiceName'))]", 68 | "dependsOn": [ 69 | "[resourceId('Microsoft.ApiManagement/service', parameters('apiManagementServiceName'))]" 70 | ], 71 | "properties": { 72 | "displayName": "ext-func-api", 73 | "apiRevision": "1", 74 | "description": "[concat('Import from \"', parameters('functionAppName'), '\" Function App')]", 75 | "subscriptionRequired": false, 76 | "path": "ext-func-api", 77 | "protocols": [ 78 | "https" 79 | ], 80 | "isCurrent": true 81 | } 82 | }, 83 | { 84 | "type": "Microsoft.ApiManagement/service/backends", 85 | "apiVersion": "2019-12-01", 86 | "name": "[concat(parameters('apiManagementServiceName'), '/', parameters('apiManagementServiceName'))]", 87 | "dependsOn": [ 88 | "[resourceId('Microsoft.ApiManagement/service', parameters('apiManagementServiceName'))]" 89 | ], 90 | "properties": { 91 | "description": "[concat('\"', parameters('functionAppName'), '\" Function App')]", 92 | "url": "[concat('https://', parameters('functionAppName'), '.azurewebsites.net/api')]", 93 | "protocol": "http", 94 | "resourceId": "[concat('https://management.azure.com/subscriptions/', subscription().subscriptionId, '/resourceGroups/', resourceGroup().name, '/providers/Microsoft.Web/sites/', parameters('functionAppName'))]", 95 | "credentials": { 96 | "header": { 97 | "x-functions-key": [ 98 | "[listkeys(concat(variables('functionAppId'), '/host/default/'),'2019-08-01').functionKeys.default]" 99 | ] 100 | } 101 | } 102 | } 103 | }, 104 | { 105 | "type": "Microsoft.ApiManagement/service/policies", 106 | "apiVersion": "2019-12-01", 107 | "name": "[concat(parameters('apiManagementServiceName'), '/policy')]", 108 | "dependsOn": [ 109 | "[resourceId('Microsoft.ApiManagement/service', parameters('apiManagementServiceName'))]" 110 | ], 111 | "properties": { 112 | "value": "\r\n\r\n \r\n \r\n \r\n \r\n \r\n", 113 | "format": "xml" 114 | } 115 | }, 116 | { 117 | "type": "Microsoft.ApiManagement/service/apis/operations", 118 | "apiVersion": "2019-12-01", 119 | "name": "[concat(parameters('apiManagementServiceName'), '/', parameters('apiManagementServiceName'), '/post-echo')]", 120 | "dependsOn": [ 121 | "[resourceId('Microsoft.ApiManagement/service/apis', parameters('apiManagementServiceName'), parameters('apiManagementServiceName'))]", 122 | "[resourceId('Microsoft.ApiManagement/service', parameters('apiManagementServiceName'))]" 123 | ], 124 | "properties": { 125 | "displayName": "echo", 126 | "method": "POST", 127 | "urlTemplate": "/echo", 128 | "templateParameters": [], 129 | "responses": [] 130 | } 131 | }, 132 | { 133 | "type": "Microsoft.ApiManagement/service/apis/operations/policies", 134 | "apiVersion": "2019-12-01", 135 | "name": "[concat(parameters('apiManagementServiceName'), '/', parameters('apiManagementServiceName'), '/post-echo/policy')]", 136 | "dependsOn": [ 137 | "[resourceId('Microsoft.ApiManagement/service/apis/operations', parameters('apiManagementServiceName'), parameters('apiManagementServiceName'), 'post-echo')]", 138 | "[resourceId('Microsoft.ApiManagement/service/apis', parameters('apiManagementServiceName'), parameters('apiManagementServiceName'))]", 139 | "[resourceId('Microsoft.ApiManagement/service', parameters('apiManagementServiceName'))]" 140 | ], 141 | "properties": { 142 | "value": "[concat('\r\n \r\n \r\n ', ' \r\n \r\n \r\n \r\n SNOWFLAKE_SERVICE_PRINCIPAL_ID \r\n \r\n \r\n ', parameters('azureadApplicationId') ,' \r\n \r\n \r\n ' ,'\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n')]", 143 | "format": "xml" 144 | } 145 | }, 146 | { 147 | "type": "Microsoft.Web/sites", 148 | "apiVersion": "2018-11-01", 149 | "name": "[parameters('functionAppName')]", 150 | "location": "[resourceGroup().location]", 151 | "dependsOn": [ 152 | "[concat('Microsoft.Storage/storageAccounts/', parameters('storageAccountName'))]" 153 | ], 154 | "kind": "functionapp", 155 | "properties": { 156 | "name": "[parameters('functionAppName')]", 157 | "siteConfig": { 158 | "appSettings": [ 159 | { 160 | "name": "FUNCTIONS_EXTENSION_VERSION", 161 | "value": "~3" 162 | }, 163 | { 164 | "name": "FUNCTIONS_WORKER_RUNTIME", 165 | "value": "node" 166 | }, 167 | { 168 | "name": "WEBSITE_NODE_DEFAULT_VERSION", 169 | "value": "~12" 170 | }, 171 | { 172 | "name": "AzureWebJobsStorage", 173 | "value": "[concat('DefaultEndpointsProtocol=https;AccountName=',parameters('storageAccountName'),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2019-06-01').keys[0].value,';EndpointSuffix=','core.windows.net')]" 174 | }, 175 | { 176 | "name": "AzureWebJobsDashboard", 177 | "value": "[concat('DefaultEndpointsProtocol=https;AccountName=',parameters('storageAccountName'),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2019-06-01').keys[0].value,';EndpointSuffix=','core.windows.net')]" 178 | }, 179 | { 180 | "name": "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING", 181 | "value": "[concat('DefaultEndpointsProtocol=https;AccountName=',parameters('storageAccountName'),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2019-06-01').keys[0].value,';EndpointSuffix=','core.windows.net')]" 182 | }, 183 | { 184 | "name": "WEBSITE_CONTENTSHARE", 185 | "value": "[toLower(parameters('functionAppName'))]" 186 | } 187 | ] 188 | }, 189 | "clientAffinityEnabled": true 190 | }, 191 | "resources": [ 192 | { 193 | "name": "[concat(parameters('functionAppName'), '/authsettings')]", 194 | "apiVersion": "2018-11-01", 195 | "type": "Microsoft.Web/sites/config", 196 | "location": "[resourceGroup().location]", 197 | "dependsOn": [ 198 | "[resourceId('Microsoft.Web/sites', parameters('functionAppName'))]" 199 | ], 200 | "properties": { 201 | "enabled": true, 202 | "unauthenticatedClientAction": "RedirectToLoginPage", 203 | "tokenStoreEnabled": true, 204 | "defaultProvider": "AzureActiveDirectory", 205 | "clientId": "[parameters('azureadApplicationId')]", 206 | "issuer": "[concat('https://sts.windows.net/', subscription().tenantId, '/')]" 207 | } 208 | }, 209 | { 210 | "apiVersion": "2019-08-01", 211 | "name": "[concat(parameters('functionAppName'),'/', 'echo')]", 212 | "type": "Microsoft.Web/sites/functions", 213 | "dependsOn": [ 214 | "[resourceId('Microsoft.Web/sites', parameters('functionAppName'))]" 215 | ], 216 | "properties": { 217 | "config": { 218 | "bindings": [ 219 | { 220 | "name": "req", 221 | "webHookType": "genericJson", 222 | "direction": "in", 223 | "type": "httpTrigger", 224 | "authLevel": "function" 225 | }, 226 | { 227 | "name": "res", 228 | "direction": "out", 229 | "type": "http" 230 | } 231 | ] 232 | }, 233 | "files": { 234 | "index.js": "module.exports = async function (context, req) { \r\n if (req.body) { \r\n var rows = req.body.data; \r\n var results = [];\r\n var index = 0;\r\n rows.forEach(row => {\r\n results.push([index++, row[1]]);\r\n }); \r\n results = {\"data\": results} \r\n context.res = { \r\n body: JSON.stringify(results) \r\n }; \r\n } \r\n else { \r\n context.res = {\r\n status: 400, \r\n body: \"Please pass data in the request body\" \r\n }; \r\n } \r\n}" 235 | } 236 | } 237 | } 238 | ] 239 | }, 240 | { 241 | "type": "Microsoft.Storage/storageAccounts", 242 | "apiVersion": "2019-06-01", 243 | "name": "[parameters('storageAccountName')]", 244 | "location": "[resourceGroup().location]", 245 | "sku": { 246 | "name": "Standard_LRS" 247 | } 248 | } 249 | ], 250 | "outputs": { 251 | "API Management URL": { 252 | "value": "[concat('https://', parameters('apiManagementServiceName'), '.azure-api.net')]", 253 | "type": "string" 254 | }, 255 | "Azure Function Http Trigger URL": { 256 | "value": "[concat('https://', parameters('apiManagementServiceName'), '.azure-api.net', '/ext-func-api/echo')]", 257 | "type": "string" 258 | } 259 | } 260 | } 261 | -------------------------------------------------------------------------------- /DeploymentTemplates/azure/README.md: -------------------------------------------------------------------------------- 1 | # External Functions 2 | 3 | [Azure Resource Manager](https://docs.microsoft.com/en-us/azure/azure-resource-manager/templates/quickstart-create-templates-use-the-portal#edit-and-deploy-the-template) template for setting up Azure API Management instance and Azure Function App for Snowflake external functions. 4 | 5 | Steps for creating Snowflake external function using this template: 6 | 1. Create a new [app registration](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app) that represents the API gateway and Azure Function app. 7 | 2. Use the application ID of the app from step #1 for azureadApplicationId parameter and deploy this template. 8 | 3. Note the URL of the API management instance and the Function App. 9 | 4. Create API integration in Snowflake using the API management URL and the Azure AD application ID. 10 | 5. Get the service principal of the Snowflake application created using the consent flow. 11 | 6. Replace SNOWFLAKE_SERVICE_PRINCIPAL_ID with the above value in the validate-jwt policy. 12 | 7. Create and run the external function. -------------------------------------------------------------------------------- /ExtractEntitiesFromArticle/1_entityExtraction.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | 4 | def ExtractEntitiesFromArticle(event, context): 5 | retVal= {} 6 | retVal["data"] = [] 7 | 8 | # Data is sent to Lambda via a HTTPS POST call. We want to get to the payload send by Snowflake 9 | event_body = event["body"] 10 | payload = json.loads(event_body) 11 | 12 | for row in payload["data"]: 13 | sflkRowRef = row[0] # This is how Snowflake keeps track of data as it gets returned 14 | inputText = row[1] 15 | 16 | client = boto3.client('comprehend') 17 | comprehendResponse = client.detect_entities( 18 | Text=inputText, 19 | LanguageCode='en' 20 | ) 21 | 22 | retVal["data"].append([sflkRowRef,comprehendResponse]) 23 | 24 | 25 | return retVal -------------------------------------------------------------------------------- /ExtractEntitiesFromArticle/2_entityExtraction.sql: -------------------------------------------------------------------------------- 1 | ------------------ 2 | -- ML Inference -- 3 | ------------------ 4 | use extfuncdemodb.extfuncs; 5 | 6 | 7 | create or replace external function EXT_UDF_entity_extraction(bodyext string) 8 | returns variant 9 | api_integration = api_int_pricedata --Tell the External Function how to connect to my AWS account 10 | as 'https://{domain}.execute-api.{region}.amazonaws.com/prod/{endpoint}'; 11 | 12 | 13 | --Test the external function. 14 | --Result is the full response of the Lambda. An array of entities 15 | SELECT EXT_UDF_entity_extraction('Richard Morwood (Aged 35) works at Snowflake. He loves working there!'); 16 | 17 | 18 | create or replace table articleText 19 | ( 20 | sourceUrl string, 21 | bodyText string 22 | ); 23 | 24 | insert into articleText (sourceUrl, bodyText) 25 | values ('https://www.snowflake.com/news/snowflake-announces-availability-aws-marketplace/', 'SAN MATEO, Calif. – Nov. 29, 2016 – Snowflake Computing, the data warehouse built for the cloud, today announced immediate availability of the Snowflake Elastic Data Warehouse through AWS Marketplace in conjunction with the launch of SaaS Subscriptions on AWS Marketplace. AWS Marketplace is an online store that helps customers find, buy, and immediately start using the software and services they need to build products and run their businesses. Visitors to the marketplace can quickly access ready-to-use software and pay only for what they use. “AWS Marketplace strives to offer customers the best possible selection of software products from ISVs to enable customer choice,” said Barry Russell, GM of Global Business Development, AWS Marketplace and Catalog Services, Amazon Web Services. “Our customers want solutions like Snowflake that are built on AWS and designed with the cloud in mind, and their support of SaaS Subscriptions on AWS Marketplace makes it even easier for customers to procure and deploy Snowflake for their data and analytics needs.” Snowflake, the data warehouse built for the cloud, was founded with the vision of eliminating the barriers to data analytics. Snowflake’s data warehouse built for the cloud delivers the performance, concurrency, and simplicity needed to support today’s diverse data and analytic') 26 | , ('https://www.snowflake.com/news/snowflake-announces-general-availability-on-google-cloud-in-london-u-k/', 'SAN MATEO, Calif. – June 23, 2020 – Snowflake, the cloud data platform, today announced general availability on Google Cloud in London, U.K.. The expansion follows Snowflake’s general availability on Google Cloud in the US and Netherlands earlier this year and reflects a continued rise in customer demand from organizations with a Google Cloud or multi-cloud strategy. Snowflake accelerated its London launch to empower organizations, such as the U.K.’s National Health Service, with a scalable, powerful data platform capable of generating rapid data insights that can help organizations as they respond to the COVID-19 pandemic. For Greater Manchester Health and Social Care Partnership, having a U.K.-hosted cloud data platform was essential, due to the sensitive nature of their data. With Snowflake, a number of key health and social care organisations in Greater Manchester will now be able to access a multi-cloud data platform for data science and analytics that scales to support any number of users and is secure by design. Matt Hennessey, Chief Intelligence and Analytics Officer for the Greater Manchester Health and Social Care Partnership, said: “Snowflake is a valuable addition to the suite of digital technologies that comprise the Greater Manchester Digital Platform. Snowflake will provide a powerful mechanism for generating quick data insights and driving decision-making that ultimately supports the health and wellbeing of our citizens.”') 27 | , ('https://www.snowflake.com/news/snowflake-announces-availability-on-microsoft-azure/', 'SAN MATEO, Calif. – July 12, 2018 – Snowflake Computing, the data warehouse built for the cloud, today announced immediate availability of its data warehouse-as-a-service on Microsoft Azure for preview. Customer demand for Azure, and the need for large organizations to have flexibility across their cloud strategy, has prompted Snowflake to offer Azure as a cloud computing option to run Snowflake’s cloud-built data warehouse. Nielsen, the global measurement and data analytics company, built their next-generation Connected System as a cloud-native platform. “We strongly believe that advancements in computing will happen more rapidly in the Cloud. We are proactively building the future of our business by leveraging Snowflake and Microsoft Azure,” Nielsen Buy CTO, Srini Varadarajan said. Nielsen’s Connected System operates on large volumes of global, retail, point-of-sale data to produce analytics that enable FMCG and retail companies around the world to achieve sustained, profitable growth in today’s ever-evolving industries. “Snowflake and Azure deliver the scale and performance we need to enable modern data analytics so we can deliver our customers the product and consumer insights they need,” Varadarajan said. “We look forward to what’s on the horizon with Azure and Snowflake.” Snowflake CEO, Bob Muglia said: “Organizations continue to move their data analytics to the cloud at an increasing pace, with the cloud data warehouse at the core of their strategy. Customer demand for an Azure-based data warehouse is also on the rise. We’re working with Microsoft to provide the performance, concurrency and flexibility that Azure customers require from a modern, cloud-built data warehouse.” Corporate Vice President for Azure Compute at Microsoft Corp., Corey Sanders said: “Migration of an enterprise data warehouse to the cloud is a key requirement for Azure customers. We look forward to partnering with Snowflake to enable these data warehouse migrations for enterprise customers moving onto Microsoft Azure. I am excited to have Snowflake available on the Azure platform.”') 28 | ; 29 | 30 | 31 | --Response per input row 32 | select EXT_UDF_entity_extraction(bodyText) as results 33 | from articleText; 34 | 35 | 36 | --Let's flatten the entities 37 | with cte_strings as 38 | ( 39 | select EXT_UDF_entity_extraction(bodyText) as results 40 | from articleText 41 | ) 42 | 43 | select f.value as EntityVariant, 44 | f.value:Type::string as EntityType, 45 | f.value:Text::string as Entity, 46 | f.value:Score::number(11, 10) as Score 47 | from cte_strings, 48 | lateral flatten(input => results:Entities) f 49 | ; -------------------------------------------------------------------------------- /ExtractEntitiesFromArticle/entityExtractio_WithStockPrices.sql: -------------------------------------------------------------------------------- 1 | ------------------ 2 | -- ML Inference -- 3 | ------------------ 4 | use extfuncdemodb.elt; 5 | 6 | 7 | create or replace external function EXT_UDF_entity_extraction(bodyext string) 8 | returns variant 9 | api_integration = api_int_pricedata --Tell the External Function how to connect to my AWS account 10 | as 'https://{domain}.execute-api.{region}.amazonaws.com/prod/{endpoint}'; 11 | 12 | 13 | --Test the external function. 14 | --Result is the full response of the Lambda. An array of entities 15 | SELECT EXT_UDF_entity_extraction('Richard Morwood (Aged 35) works at Snowflake. He loves working there!'); 16 | 17 | 18 | 19 | create or replace table articleText 20 | ( 21 | sourceUrl string, 22 | bodyText string 23 | ); 24 | 25 | 26 | truncate table articleText; 27 | 28 | insert into articleText (sourceUrl, bodyText) 29 | values ('https://www.snowflake.com/news/snowflake-announces-availability-aws-marketplace/', 'SAN MATEO, Calif. – Nov. 29, 2016 – Snowflake Computing, the data warehouse built for the cloud, today announced immediate availability of the Snowflake Elastic Data Warehouse through AWS Marketplace in conjunction with the launch of SaaS Subscriptions on AWS Marketplace. AWS Marketplace is an online store that helps customers find, buy, and immediately start using the software and services they need to build products and run their businesses. Visitors to the marketplace can quickly access ready-to-use software and pay only for what they use. “AWS Marketplace strives to offer customers the best possible selection of software products from ISVs to enable customer choice,” said Barry Russell, GM of Global Business Development, AWS Marketplace and Catalog Services, Amazon Web Services. “Our customers want solutions like Snowflake that are built on AWS and designed with the cloud in mind, and their support of SaaS Subscriptions on AWS Marketplace makes it even easier for customers to procure and deploy Snowflake for their data and analytics needs.” Snowflake, the data warehouse built for the cloud, was founded with the vision of eliminating the barriers to data analytics. Snowflake’s data warehouse built for the cloud delivers the performance, concurrency, and simplicity needed to support today’s diverse data and analytic') 30 | , ('https://www.snowflake.com/news/snowflake-announces-general-availability-on-google-cloud-in-london-u-k/', 'SAN MATEO, Calif. – June 23, 2020 – Snowflake, the cloud data platform, today announced general availability on Google Cloud in London, U.K.. The expansion follows Snowflake’s general availability on Google Cloud in the US and Netherlands earlier this year and reflects a continued rise in customer demand from organizations with a Google Cloud or multi-cloud strategy. Snowflake accelerated its London launch to empower organizations, such as the U.K.’s National Health Service, with a scalable, powerful data platform capable of generating rapid data insights that can help organizations as they respond to the COVID-19 pandemic. For Greater Manchester Health and Social Care Partnership, having a U.K.-hosted cloud data platform was essential, due to the sensitive nature of their data. With Snowflake, a number of key health and social care organisations in Greater Manchester will now be able to access a multi-cloud data platform for data science and analytics that scales to support any number of users and is secure by design. Matt Hennessey, Chief Intelligence and Analytics Officer for the Greater Manchester Health and Social Care Partnership, said: “Snowflake is a valuable addition to the suite of digital technologies that comprise the Greater Manchester Digital Platform. Snowflake will provide a powerful mechanism for generating quick data insights and driving decision-making that ultimately supports the health and wellbeing of our citizens.”') 31 | , ('https://www.snowflake.com/news/snowflake-announces-availability-on-microsoft-azure/', 'SAN MATEO, Calif. – July 12, 2018 – Snowflake Computing, the data warehouse built for the cloud, today announced immediate availability of its data warehouse-as-a-service on Microsoft Azure for preview. Customer demand for Azure, and the need for large organizations to have flexibility across their cloud strategy, has prompted Snowflake to offer Azure as a cloud computing option to run Snowflake’s cloud-built data warehouse. Nielsen, the global measurement and data analytics company, built their next-generation Connected System as a cloud-native platform. “We strongly believe that advancements in computing will happen more rapidly in the Cloud. We are proactively building the future of our business by leveraging Snowflake and Microsoft Azure,” Nielsen Buy CTO, Srini Varadarajan said. Nielsen’s Connected System operates on large volumes of global, retail, point-of-sale data to produce analytics that enable FMCG and retail companies around the world to achieve sustained, profitable growth in today’s ever-evolving industries. “Snowflake and Azure deliver the scale and performance we need to enable modern data analytics so we can deliver our customers the product and consumer insights they need,” Varadarajan said. “We look forward to what’s on the horizon with Azure and Snowflake.” Snowflake CEO, Bob Muglia said: “Organizations continue to move their data analytics to the cloud at an increasing pace, with the cloud data warehouse at the core of their strategy. Customer demand for an Azure-based data warehouse is also on the rise. We’re working with Microsoft to provide the performance, concurrency and flexibility that Azure customers require from a modern, cloud-built data warehouse.” Corporate Vice President for Azure Compute at Microsoft Corp., Corey Sanders said: “Migration of an enterprise data warehouse to the cloud is a key requirement for Azure customers. We look forward to partnering with Snowflake to enable these data warehouse migrations for enterprise customers moving onto Microsoft Azure. I am excited to have Snowflake available on the Azure platform.”') 32 | ; 33 | 34 | 35 | 36 | 37 | --Response per input row 38 | select EXT_UDF_entity_extraction(bodyText) as results 39 | from articleText; 40 | 41 | 42 | 43 | 44 | 45 | --Let's flatten the entities 46 | with cte_strings as 47 | ( 48 | select EXT_UDF_entity_extraction(bodyText) as results 49 | from articleText 50 | ) 51 | 52 | select f.value as EntityVariant, 53 | f.value:Type::string as EntityType, 54 | f.value:Text::string as Entity, 55 | f.value:Score::number(11, 10) as Score 56 | from cte_strings, 57 | lateral flatten(input => results:Entities) f 58 | ; 59 | 60 | 61 | 62 | 63 | 64 | create or replace table mappings_EntityToStockSymbol 65 | ( 66 | entity string, 67 | entitytype string, 68 | symbol string 69 | ); 70 | 71 | 72 | insert into mappings_entityToStockSymbol ( entity, entitytype, symbol ) 73 | values ('Amazon', 'ORGANIZATION', 'AMZN'), 74 | ('Amazon Web Services', 'ORGANIZATION', 'AMZN'), 75 | ('AWS Marketplace', 'ORGANIZATION', 'AMZN'), 76 | ('Microsoft', 'ORGANIZATION', 'MSFT'), 77 | ('Microsoft Corp.', 'ORGANIZATION', 'MSFT'), 78 | ('Azure', 'ORGANIZATION', 'MSFT'), 79 | ('Google', 'ORGANIZATION', 'GOOGL'), 80 | ('Splunk', 'ORGANIZATION', 'SPLK'), 81 | ('MongoDB', 'ORGANIZATION', 'MBG'), 82 | ('New Relic', 'ORGANIZATION', 'NEWR'), 83 | ('Alteryx', 'ORGANIZATION', 'AYX'), 84 | ('Elastic', 'ORGANIZATION', 'ESTC'), 85 | ('Cloudera', 'ORGANIZATION', 'CLDR'), 86 | ('Talend', 'ORGANIZATION', 'TLND'), 87 | ('IBM', 'ORGANIZATION', 'IBM'), 88 | ('Oracle', 'ORGANIZATION', 'ORCL'), 89 | ('VMWare', 'ORGANIZATION', 'VMW'), 90 | ('SAP', 'ORGANIZATION', 'SAP'); 91 | 92 | 93 | --Let's join this with out entities to monitor for! 94 | select * from mappings_entitytostocksymbol; 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | --Extract entities, join with the mapping table 103 | with cte_entityExtraction as 104 | ( 105 | select EXT_UDF_entity_extraction(bodyText) as results 106 | from articleText 107 | ) 108 | 109 | , cte_flattened as 110 | ( 111 | select f.value as EntityVariant, 112 | f.value:Type::string as EntityType, 113 | f.value:Text::string as Entity, 114 | f.value:Score::number(11, 10) as Score 115 | from cte_entityExtraction, 116 | lateral flatten(input => results:Entities) f 117 | // where entitytype = 'ORGANIZATION'; 118 | ) 119 | 120 | --Join with the mapping table 121 | //select distinct f.*, m.Symbol 122 | select distinct m.Symbol 123 | from cte_flattened f 124 | join mappings_entitytostocksymbol m 125 | on f.entity = m.entity 126 | and f.entitytype = m.entitytype 127 | ; 128 | 129 | 130 | 131 | create view extfuncdemodb.elt.vw_extractEntitiesFromText 132 | as 133 | --Extract entities, join with the mapping table 134 | with cte_entityExtraction as 135 | ( 136 | select EXT_UDF_entity_extraction(bodyText) as results 137 | from articleText 138 | ) 139 | 140 | , cte_flattened as 141 | ( 142 | select f.value as EntityVariant, 143 | f.value:Type::string as EntityType, 144 | f.value:Text::string as Entity, 145 | f.value:Score::number(11, 10) as Score 146 | from cte_entityExtraction, 147 | lateral flatten(input => results:Entities) f 148 | // where entitytype = 'ORGANIZATION'; 149 | ) 150 | 151 | --Join with the mapping table 152 | //select distinct f.*, m.Symbol 153 | select distinct m.Symbol 154 | from cte_flattened f 155 | join mappings_entitytostocksymbol m 156 | on f.entity = m.entity 157 | and f.entitytype = m.entitytype 158 | ; 159 | 160 | 161 | select * from extfuncdemodb.elt.vw_extractEntitiesFromText; -------------------------------------------------------------------------------- /ExtractEntitiesFromArticle/example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Snowflake-Labs/sfguide-external-functions-examples/cb38a3e18076b467c1cb79c7e59766b1be03f638/ExtractEntitiesFromArticle/example.png -------------------------------------------------------------------------------- /ExtractEntitiesFromArticle/readme.md: -------------------------------------------------------------------------------- 1 | External Functions can be used for Machine Learning Inference. 2 | In this example, we'll pass text through AWS Comprehend to extract the entities mentioned. 3 | 4 | ![Example](example.png) 5 | -------------------------------------------------------------------------------- /FetchHTTPData/1_handler.py: -------------------------------------------------------------------------------- 1 | import json 2 | import requests 3 | 4 | def FetchHttpData(event, context): 5 | retVal= {} 6 | retVal["data"] = [] 7 | 8 | # Data is sent to Lambda via a HTTPS POST call. We want to get to the payload send by Snowflake 9 | event_body = event["body"] 10 | payload = json.loads(event_body) 11 | 12 | for row in payload["data"]: 13 | sflkRowRef = row[0] # This is how Snowflake keeps track of data as it gets returned 14 | URL = row[1] # The data passed in from Snowflake that the input row contains. 15 | # If the passed in data was a Variant, it reaches Python as a dictionary. Handy! 16 | # URL = 'https://gbfs.citibikenyc.com/gbfs/en/system_regions.json' 17 | # r = requests.get(url = URL) 18 | # response = r.json() 19 | httpData = requests.get(url = URL).json() 20 | response = {} 21 | response["url"] = URL 22 | response["response"] = httpData 23 | 24 | retVal["data"].append([sflkRowRef,response]) 25 | 26 | return retVal -------------------------------------------------------------------------------- /FetchHTTPData/2_fetchHTTPData.sql: -------------------------------------------------------------------------------- 1 | use extfuncdemodb.extfuncs; 2 | 3 | create or replace external function EXT_UDF_fetchHTTPData(url string) 4 | returns variant 5 | api_integration = api_int_notifications --Tell the External Function how to connect to my AWS account 6 | as 'https://{domain}.execute-api.{region}.amazonaws.com/prod/{endpoint}'; 7 | 8 | 9 | --Test the external function. 10 | with cte_endpoints 11 | as 12 | ( 13 | select 'https://gbfs.citibikenyc.com/gbfs/en/system_regions.json' as URL 14 | union 15 | select 'https://domain.com/dataEndpoint' as URL 16 | ) 17 | 18 | SELECT EXT_UDF_fetchHTTPData(URL) 19 | from cte_endpoints; -------------------------------------------------------------------------------- /FetchHTTPData/readme.md: -------------------------------------------------------------------------------- 1 | External Functions can be used to get data from a HTTP endpoint. 2 | Pricing data provided by FinnHub - https://finnhub.io/ 3 | 4 | You'll need an API from FinnHub, nice and free 5 | 6 | Shoutouts to bukosabino for the fantastic TechnicalAanlysis Python library used in this example! 7 | Technical Analysis - https://github.com/bukosabino/ta 8 | 9 | ``` 10 | $ pip install --upgrade ta 11 | $ pip install --upgrade pandas 12 | $ pip install --upgrade requests 13 | ``` 14 | 15 | 16 | At the time of writing, the Pandas Python Package is too large to go directly to Lamdba. To get around this, I use the Serverless Framework. It takes care of zipping, compiling, uploading, creating the Lambda, and linking the API Gateway with one CLI call 17 | `sls deploy` 18 | 19 | If this is new to you, here's a great guide! 20 | 21 | https://www.serverless.com/blog/serverless-python-packaging/ 22 | 23 | If you do use the Serverless Framework, you'll need to uncomment the first line in priceData.py 24 | `import unzip_requirements # Use this when deploying through Serverless` 25 | -------------------------------------------------------------------------------- /FetchStockPricesAndIndicators/config.py: -------------------------------------------------------------------------------- 1 | class FinnHubAPI: 2 | TOKEN="yourTokenHere" 3 | 4 | # Get a FinHubb API from 5 | # https://finnhub.io/ -------------------------------------------------------------------------------- /FetchStockPricesAndIndicators/examplePayload.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": [ 3 | [ 4 | 0, 5 | "SNOW", 6 | 1593475200, 7 | 1596067200 8 | ], 9 | [ 10 | 1, 11 | "AMZN", 12 | 1593475200, 13 | 1596067200 14 | ] 15 | ] 16 | } -------------------------------------------------------------------------------- /FetchStockPricesAndIndicators/priceData.py: -------------------------------------------------------------------------------- 1 | # import unzip_requirements # Use this when deploying through Serverless 2 | import ta 3 | from ta.volatility import BollingerBands 4 | from ta.utils import dropna 5 | import json 6 | import boto3 7 | import pandas as pd 8 | import requests 9 | from config import FinnHubAPI 10 | 11 | def GetPriceDataFromExchangeFinnHub(event, context): 12 | retVal= {} 13 | retVal["data"] = [] 14 | 15 | # For debugging the input, write the EVENT to CloudWatch logs 16 | print(json.dumps(event)) 17 | 18 | # Data is sent to Lambda via a HTTPS POST call. We want to get to the payload send by Snowflake 19 | event_body = event["body"] 20 | payload = json.loads(event_body) 21 | 22 | for row in payload["data"]: 23 | sflkRowRef = row[0] # This is how Snowflake keeps track of data as it gets returned 24 | symbol = row[1] # The data passed in from Snowflake that the input row contains. 25 | fromDate = row[2] 26 | toDate = row[3] 27 | 28 | # Will return URL without token to Snowflake for tracking 29 | URL = f'https://finnhub.io/api/v1/stock/candle?symbol={symbol}&resolution=D&from={fromDate}&to={toDate}' 30 | 31 | # Add our FinnHubAPI Key to the end of the URL. 32 | # This is in a new variable which will not be returned to Snowflake 33 | URLWithToken = f'{URL}&token={FinnHubAPI.TOKEN}' 34 | 35 | # GET data from the API 36 | httpData = requests.get(url = URLWithToken).json() 37 | 38 | # Convert to Pandas DataFrame 39 | df = pd.DataFrame(httpData) 40 | 41 | # Add the column names 42 | print("Adding column names") 43 | df.columns = ["Close", "High", "Low", "Open", "Status", "OpenTime", "Volume"] 44 | 45 | # Set DateTime columns to correct type 46 | df['OpenTime'] = pd.to_datetime(df['OpenTime'], unit='ms') 47 | df['Open'] = df['Open'].astype(float) 48 | df['High'] = df['High'].astype(float) 49 | df['Low'] = df['Low'].astype(float) 50 | df['Close'] = df['Close'].astype(float) 51 | df['Volume'] = df['Volume'].astype(float) 52 | 53 | # Clean NaN values 54 | print("Cleaning NA values") 55 | df = dropna(df) 56 | 57 | # Calculate the Bollinger Bands indicator 58 | indicator_bb = BollingerBands(close=df["Close"], n=20, ndev=2) 59 | df['bb_bbm'] = indicator_bb.bollinger_mavg() 60 | df['bb_bbh'] = indicator_bb.bollinger_hband() 61 | df['bb_bbl'] = indicator_bb.bollinger_lband() 62 | df['bb_bbhi'] = indicator_bb.bollinger_hband_indicator() 63 | df['bb_bbli'] = indicator_bb.bollinger_lband_indicator() 64 | df['bb_bbw'] = indicator_bb.bollinger_wband() 65 | df['bb_bbp'] = indicator_bb.bollinger_pband() 66 | 67 | print("converting OHLC pandas to JSON. This does it as a string") 68 | buffer = df.to_json(orient = "records") 69 | 70 | print("Interpret the JSON string into a dictionary for output") 71 | jsonResponse = json.loads(buffer) 72 | 73 | # Prepare the output response 74 | response = {} 75 | response["url"] = URL 76 | response["response"] = jsonResponse 77 | 78 | retVal["data"].append([sflkRowRef,response]) 79 | 80 | # For debugging the output, write the RETurn VALue to CloudWatch logs 81 | # print(json.dumps(retVal)) 82 | 83 | return retVal -------------------------------------------------------------------------------- /FetchStockPricesAndIndicators/priceData.sql: -------------------------------------------------------------------------------- 1 | --------------------------- 2 | -- Getting External Data -- 3 | --------------------------- 4 | //create or replace external function EXT_UDF_fetch_asset_price_data(url string) 5 | create or replace external function EXT_UDF_fetch_asset_price_data(symbol string, fromDate int, toDate int) 6 | returns variant 7 | api_integration = api_int_pricedata --Tell the External Function how to connect to my AWS account 8 | as 'https://{domain}.execute-api.ap-southeast-2.amazonaws.com/prod/{endpoint}'; 9 | 10 | 11 | // The FinHubb API would like the time in UnixSeconds 12 | // https://www.epochconverter.com/ 13 | select to_timestamp(1590969600) as startDate, 14 | to_timestamp(1595808000) as startDate; 15 | 16 | // Simple call to get Snowflakes stock price 17 | select EXT_UDF_fetch_asset_price_data('SNOW', 1590969600, 1595808000) as apiData; 18 | 19 | 20 | // Now lets make it dynamically fetch the last 7 days history 21 | set daysHistory = 7; 22 | select $daysHistory; 23 | 24 | //Convert a datetime to unix seconds 25 | select dateadd(day, -$daysHistory, current_date())::timestamp 26 | , DATE_PART('EPOCH_SECOND', dateadd(day, -$daysHistory, current_date())::timestamp); 27 | 28 | 29 | // Which stock symbols should we get? 30 | with cte_stocksymbols 31 | as 32 | ( 33 | select 'SNOW' as symbol 34 | union 35 | select 'AMZN' as symbol 36 | union 37 | select 'MSFT' as symbol 38 | union 39 | select 'GOOGL' as symbol 40 | ) 41 | 42 | // Fetch the data from FinHubb 43 | , cte_stockprices 44 | as 45 | ( 46 | select Symbol 47 | , DATE_PART('EPOCH_SECOND', dateadd(day, -$daysHistory, current_date())::timestamp) as fromDate 48 | , DATE_PART('EPOCH_SECOND', current_date()) as toDate 49 | , EXT_UDF_fetch_asset_price_data(symbol, fromDate, toDate) as apiData 50 | from cte_stocksymbols 51 | ) 52 | 53 | //Read the results. Each set of price data comes back as an array, which we can flatten 54 | //"Hey! The Bollinger Bands are null!" 55 | //These are calculated using the last X number of candles (20 if you didn't adjust the python code) 56 | //apiData:response.entry[20] and onwards have populated data 57 | select symbol 58 | , d.value as entry 59 | , to_timestamp((entry:OpenTime::int)) as OpenTime 60 | , entry:OpenTime::int as OpenTime 61 | , entry:Open::number(20, 5) as OpenPrice 62 | , entry:High::number(20, 5) as HighPrice 63 | , entry:Low::number(20, 5) as LowPrice 64 | , entry:Close::number(20, 5) as ClosePrice 65 | , entry:Volume::int as Volume 66 | from cte_stockprices, 67 | lateral flatten(input => apiData:response) d 68 | ; -------------------------------------------------------------------------------- /FetchStockPricesAndIndicators/readme.md: -------------------------------------------------------------------------------- 1 | This example is an extension of the FetchHHTPData example. 2 | We'll get stock price data from a HTTP endpoint and enhance it with Technical Indicators. 3 | Pricing data provided by FinnHub - https://finnhub.io/ (You'll need an API from FinnHub) 4 | 5 | Shoutouts to bukosabino for the fantastic TechnicalAanlysis Python library used in this example! 6 | Technical Analysis - https://github.com/bukosabino/ta 7 | 8 | ``` 9 | $ pip install --upgrade ta 10 | $ pip install --upgrade pandas 11 | $ pip install --upgrade requests 12 | ``` 13 | 14 | 15 | At the time of writing, the Pandas Python Package is too large to go directly to Lamdba. To get around this you can use the Serverless Framework. It takes care of zipping, compiling, uploading, creating the Lambda, and linking the API Gateway with one CLI call 16 | `sls deploy` 17 | 18 | If you are new to the Serverless Framework then here is a great guide! 19 | https://www.serverless.com/blog/serverless-python-packaging/ 20 | 21 | If you do use the Serverless Framework, you'll need to uncomment the first line in priceData.py 22 | `import unzip_requirements # Use this when deploying through Serverless` 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /MicrosoftTranslatorAPI_Python/ef_translate.py: -------------------------------------------------------------------------------- 1 | import os, requests, json 2 | import logging, httpx 3 | import azure.functions as func 4 | 5 | async def main(req: func.HttpRequest) -> func.HttpResponse: 6 | endpoint = "https://api.cognitive.microsofttranslator.com/" 7 | path = "/translate?api-version=3.0" 8 | params = "&to=it" 9 | constructed_url = endpoint + path + params 10 | 11 | headers = { 12 | "Ocp-Apim-Subscription-Key": "", 13 | "Ocp-apim-subscription-region": "", 14 | "Content-Type": "application/json" 15 | } 16 | 17 | req_body = req.get_json() 18 | 19 | if req_body : 20 | translated = [] 21 | body = [] 22 | i = 0 23 | 24 | # Format JSON data passed from Snowflake to what Translator API expects. 25 | for row in req_body["data"]: 26 | body.append({"text": row[1]}) 27 | 28 | # Microsoft recommends using asynchronous APIs for network IO. 29 | # This example uses httpx library to make async calls to the API. 30 | client = httpx.AsyncClient() 31 | response = await client.post(constructed_url, headers = headers, json = body) 32 | response_json = response.json() 33 | 34 | # Process and format response into Snowflake expected JSON. 35 | for row in response_json: 36 | translations = row["translations"][0] 37 | translated_text = translations["text"] 38 | translated.append([req_body["data"][i][0], translated_text]) 39 | i += 1 40 | 41 | output = {"data": translated} 42 | return func.HttpResponse(json.dumps(output)) 43 | 44 | else: 45 | return func.HttpResponse( 46 | "Please pass data to translate in the request body", 47 | Status_code = 400 48 | ) 49 | -------------------------------------------------------------------------------- /MicrosoftTranslatorAPI_Python/ef_translate.sql: -------------------------------------------------------------------------------- 1 | 2 | -- create API integration 3 | 4 | create or replace api integration external_api_integration 5 | api_provider = azure_api_management 6 | azure_tenant_id = '' 7 | azure_ad_application_id = '' 8 | api_allowed_prefixes = ('https://.azure-api.net/') 9 | enabled = true; 10 | 11 | -- create external function 12 | create or replace external function translate_en_italian(input string) 13 | returns variant 14 | api_integration = external_api_integration 15 | as 'https://.azure-api.net//' 16 | ; 17 | -------------------------------------------------------------------------------- /MicrosoftTranslatorAPI_Python/function.json: -------------------------------------------------------------------------------- 1 | { 2 | "scriptFile": "ef_translate.py", 3 | "bindings": [ 4 | { 5 | "authLevel": "function", 6 | "type": "httpTrigger", 7 | "direction": "in", 8 | "name": "req", 9 | "methods": [ 10 | "get", 11 | "post" 12 | ] 13 | }, 14 | { 15 | "type": "http", 16 | "direction": "out", 17 | "name": "$return" 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /MicrosoftTranslatorAPI_Python/readme.md: -------------------------------------------------------------------------------- 1 | In the following example, we demonstrate how to use external functions to invoke an API via Azure API Management that will trigger an Azure function. The Azure function is written in Python and invokes the Microsoft Translator API provided by Azure Cognitive services to return the Italian translation for a given input text string. This solution with external functions eliminates the need to manually export the data out of Snowflake, translate it, and then reimport it, greatly simplifying the workflow. You can find more deatils on the solution in the following blog post: 2 | https://www.snowflake.com/blog/support-for-calling-external-functions-via-azure-api-management-now-in-public-preview/ 3 | -------------------------------------------------------------------------------- /MicrosoftTranslatorAPI_Python/requirements.txt: -------------------------------------------------------------------------------- 1 | # DO NOT include azure-functions-worker in this file 2 | # The Python Worker is managed by Azure Functions platform 3 | # Manually managing azure-functions-worker may cause unexpected issues 4 | 5 | azure-functions 6 | requests==2.19.1 7 | -------------------------------------------------------------------------------- /MicrosoftTranslatorAPI_Python/sample.dat: -------------------------------------------------------------------------------- 1 | { 2 | "data": [ 3 | [ 4 | 0, 5 | "good morning" 6 | ], 7 | [ 8 | 1, 9 | "hello" 10 | ] 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /ParseUserAgentString/1_Setup.sql: -------------------------------------------------------------------------------- 1 | //DOCO on ExternalFucntions 2 | //https://docs.snowflake.com/en/sql-reference/external-functions.html 3 | 4 | create database if not exists extfuncdemodb; 5 | create schema if not exists extfuncs; 6 | use extfuncdemodb.extfuncs; 7 | 8 | show integrations; 9 | 10 | //// 1) Create the integration 11 | create or replace api integration api_int_notifications 12 | api_provider = aws_api_gateway 13 | api_aws_role_arn = 'arn:aws:iam::{{AWSAccountID}}:role/{{AWSRole}}' --Role created in AWS account 14 | enabled = true 15 | api_allowed_prefixes = ('{{API Gateway Endpoint}}') --API endpoint 16 | ; 17 | 18 | //// 2) Take the API_AWS_IAM_USER_ARN, and API_AWS_EXTERNAL_ID, put them into the roles TrustRelationship 19 | //---- API_AWS_IAM_USER_ARN goes into; 20 | //"Principal": { 21 | // "AWS": "{{API_AWS_IAM_USER_ARN here}}" 22 | // } 23 | // 24 | //----API_AWS_EXTERNAL_ID goes into; 25 | // "Condition": { 26 | // "StringEquals": { 27 | // "sts:ExternalId": "{{API_AWS_EXTERNAL_ID here}}" 28 | // } 29 | // } 30 | 31 | describe integration api_int_notifications; 32 | 33 | show external functions; 34 | 35 | 36 | create or replace external function extfuncdemodb.extfuncs.EXT_UDF_parse_useragent(useragent string) 37 | returns variant 38 | api_integration = api_int_pricedata --Tell the External Function how to connect to my AWS account 39 | as 'https://{apigatewaydomain}.execute-api.{region}.amazonaws.com/prod/{endpoint}'; -------------------------------------------------------------------------------- /ParseUserAgentString/2_parseUserAgent.py: -------------------------------------------------------------------------------- 1 | import json 2 | from user_agents import parse # https://pypi.org/project/user-agents/#description 3 | 4 | def parseUserAgent(event, context): 5 | retVal= {} # The value/object to return to Snowflake 6 | retVal["data"] = [] 7 | 8 | # Data is sent to Lambda via a HTTPS POST call. We want to get to the payload send by Snowflake 9 | event_body = event["body"] 10 | payload = json.loads(event_body) 11 | 12 | for row in payload["data"]: 13 | sflkRowRef = row[0] # This is how Snowflake keeps track of data as it gets returned 14 | userAgentInput = row[1] # The data passed in from Snowflake that the input row contains. 15 | # If the passed in data was a Variant, it lands here as a dictionary. Handy! 16 | 17 | user_agent = parse(userAgentInput) # user_agent object is not serializable 18 | 19 | # Create a Dictionary from the user_agent properties 20 | userAgentOutput = {} 21 | userAgentOutput["Browser"] = {} 22 | userAgentOutput["Browser"]["Family"] = user_agent.browser.family 23 | userAgentOutput["Browser"]["Version"] = user_agent.browser.version 24 | userAgentOutput["Browser"]["VersionString"] = user_agent.browser.version_string 25 | 26 | userAgentOutput["OS"] = {} 27 | userAgentOutput["OS"]["Family"] = user_agent.os.family 28 | userAgentOutput["OS"]["Version"] = user_agent.os.version 29 | userAgentOutput["OS"]["VersionString"] = user_agent.os.version_string 30 | 31 | userAgentOutput["OS"] = {} 32 | userAgentOutput["OS"]["Family"] = user_agent.device.family 33 | userAgentOutput["OS"]["Brand"] = user_agent.device.brand 34 | userAgentOutput["OS"]["Model"] = user_agent.device.model 35 | 36 | userAgentOutput["IsMobile"] = user_agent.is_mobile 37 | userAgentOutput["IsTablet"] = user_agent.is_tablet 38 | userAgentOutput["IsTouchCapable"] = user_agent.is_touch_capable 39 | userAgentOutput["IsPC"] = user_agent.is_pc 40 | userAgentOutput["IsBot"] = user_agent.is_bot 41 | 42 | # prepare this rows response 43 | response = {} 44 | response["UserAgentInput"] = userAgentInput 45 | response["ParsedUserAgent"] = userAgentOutput 46 | 47 | # add this row to the full returned-to-snowflake response 48 | # It must have the row identifier 49 | retVal["data"].append([sflkRowRef,response]) 50 | 51 | return retVal -------------------------------------------------------------------------------- /ParseUserAgentString/3_parseUserAgent.sql: -------------------------------------------------------------------------------- 1 | use extfuncdemodb.extfuncs; 2 | 3 | -- Here's a few example UserAgent strings to send to the External Function 4 | with cte_useragents 5 | as 6 | ( 7 | select 'Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36' as useragent 8 | union 9 | select 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Mobile Safari/537.36' as useragent 10 | union 11 | select 'Mozilla/5.0 (Linux; Android 7.0; SM-G930VC Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/58.0.3029.83 Mobile Safari/537.36' as useragent 12 | union 13 | select 'Mozilla/5.0 (Linux; Android 6.0; HTC One X10 Build/MRA58K; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/61.0.3163.98 Mobile Safari/537.36' as useragent 14 | union 15 | select 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1' as useragent 16 | union 17 | select 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/13.2b11866 Mobile/16A366 Safari/605.1.15' as useragent 18 | union 19 | select 'Mozilla/5.0 (CrKey armv7l 1.5.16041) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.0 Safari/537.36' as useragent 20 | union 21 | select 'AppleTV6,2/11.1' as useragent 22 | union 23 | select 'Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Xbox; Xbox One) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Mobile Safari/537.36 Edge/13.10586' as useragent 24 | union 25 | select 'Mozilla/5.0 (PlayStation 4 3.11) AppleWebKit/537.73 (KHTML, like Gecko)' as useragent 26 | ) 27 | 28 | select useragent as input, 29 | EXT_UDF_parse_useragent(useragent) as output 30 | from cte_useragents; -------------------------------------------------------------------------------- /ParseUserAgentString/readme.md: -------------------------------------------------------------------------------- 1 | This example uses the user_agents Python library 2 | https://pypi.org/project/user-agents 3 | 4 | ``` 5 | $ pip install user-agents; 6 | ``` -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ExternalFunctions 2 | 3 | This repository provides examples on leveraging Snowflake's External Functions feature. 4 | 5 | Please navigate to each subdirectory to learn more about the example External Function, and how to use it. 6 | 7 | [Snowflake documentation - External Functions](https://docs.snowflake.com/en/sql-reference/external-functions.html) 8 | -------------------------------------------------------------------------------- /SendSNS/1_Setup.sql: -------------------------------------------------------------------------------- 1 | //DOCO on ExternalFucntions 2 | //https://docs.snowflake.com/en/sql-reference/external-functions.html 3 | 4 | create database if not exists admin; 5 | create schema if not exists extfuncs; 6 | use admin.extfuncs; 7 | 8 | show integrations; 9 | 10 | //// 1) Create the integration 11 | create or replace api integration api_int_notifications 12 | api_provider = aws_api_gateway 13 | api_aws_role_arn = 'arn:aws:iam::{{AWSAccountID}}:role/{{AWSRole}}' --Role created in AWS account 14 | enabled = true 15 | api_allowed_prefixes = ('{{API Gateway Endpoint}}') --API endpoint 16 | ; 17 | 18 | //// 2) Take the API_AWS_IAM_USER_ARN, and API_AWS_EXTERNAL_ID, put them into the roles TrustRelationship 19 | //---- API_AWS_IAM_USER_ARN goes into; 20 | //"Principal": { 21 | // "AWS": "{{API_AWS_IAM_USER_ARN here}}" 22 | // } 23 | // 24 | //----API_AWS_EXTERNAL_ID goes into; 25 | // "Condition": { 26 | // "StringEquals": { 27 | // "sts:ExternalId": "{{API_AWS_EXTERNAL_ID here}}" 28 | // } 29 | // } 30 | 31 | describe integration api_int_notifications; 32 | 33 | show external functions; 34 | 35 | create or replace external function EXT_UDF_SendNotification(notificationDetails variant) 36 | returns variant 37 | api_integration = api_int_notifications 38 | as '{{full URL of API Gateway Endpoint}}'; -------------------------------------------------------------------------------- /SendSNS/2_LambdaCode.py: -------------------------------------------------------------------------------- 1 | import json 2 | import boto3 3 | 4 | def SendNotification(event, context): 5 | # print(event) # Debug, writes input to CloudWatch log 6 | 7 | retVal= {} 8 | retVal["data"] = [] 9 | 10 | # Data is sent to Lambda via a HTTPS POST call. We want to get to the payload send by Snowflake 11 | event_body = event["body"] 12 | payload = json.loads(event_body) 13 | 14 | for row in payload["data"]: 15 | sflkRowRef = row[0] # This is how Snowflake keeps track of data as it gets returned 16 | content = row[1] # The data passed in from Snowflake that the input row contains. 17 | # If the passed in data was a Variant, it lands here as a dictionary. Handy! 18 | 19 | # Extract anything needed from the row 20 | emailSubject = content['Subject'] 21 | emailBody = content['Body'] 22 | 23 | message = {"foo": "bar"} # SNS doesn't use this part for emails, but you MUST HAVE IT or the publish call will error 24 | client = boto3.client('sns') 25 | snsResponse = client.publish( 26 | TargetArn='arn:aws:sns:{your SNS ARN here}', 27 | Message=json.dumps({'default': json.dumps(message), 28 | 'email': emailBody}), 29 | Subject=emailSubject, 30 | MessageStructure='json' 31 | ) 32 | 33 | sflkResponse={} 34 | sflkResponse["snsResponse"] = snsResponse #['snsResponse']['messageId'] 35 | 36 | retVal["data"].append([sflkRowRef,sflkResponse]) 37 | 38 | ## Debug, writes output to CloudWatch log 39 | # print('--- RESPONSE FROM LAMBDA ---') 40 | # print(retVal) 41 | 42 | return retVal -------------------------------------------------------------------------------- /SendSNS/3_Invocation.sql: -------------------------------------------------------------------------------- 1 | use admin.extfuncs; 2 | 3 | set notificationSubject = 'Job completed successfully'; 4 | set notificationBody = 'ELT job completed! No error'; 5 | 6 | select 7 | admin.extfuncs.EXT_UDF_SendNotification( 8 | parse_json('{"Subject":"'|| $notificationSubject 9 | ||'","Body":"'|| $notificationBody 10 | ||'"}') 11 | ); -------------------------------------------------------------------------------- /SendSNS/readme.md: -------------------------------------------------------------------------------- 1 | Enables Notifications from Snowflake. 2 | 3 | Be aware that Snowflake does **NOT** guarantee an ExternalFunction will be called only once. This may result in multiple notifications being sent. 4 | 5 | To minimise this; 6 | * Increase your Lambda timeout to more than the default 3 seconds 7 | * Only send one notification in your ExternalFunction call. Let SNS do the email-many-recipients -------------------------------------------------------------------------------- /_Template_Python/example.sql: -------------------------------------------------------------------------------- 1 | create or replace database extfuncdemodb; 2 | create or replace schema elt; 3 | 4 | create or replace external function extfuncdemodb.elt.EXT_UDF_add_three(sourceNumber int) 5 | returns int 6 | api_integration = api_int_pricedata --Tell the External Function how to connect to my AWS account 7 | as 'https://{apigatewaydomain}.{region}.amazonaws.com/prod/{endpoint}'; 8 | 9 | 10 | --This will work nicely 11 | with cte_numbers as 12 | ( 13 | select 5 as input 14 | union 15 | select 23 as input 16 | ) 17 | 18 | select EXT_UDF_add_three(input) 19 | from cte_numbers; -------------------------------------------------------------------------------- /_Template_Python/readme.md: -------------------------------------------------------------------------------- 1 | Python template code for the Lamdba in an External Function, with per-row try/catch 2 | 3 | Use testData.json as the test payload in Lamdba to ensure your Lambda is setup correctly. -------------------------------------------------------------------------------- /_Template_Python/template_python.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | def lambda_handler(event, context): 4 | # Need to see what your INPUT looks like? 5 | print(json.dumps(event)) # This will be written to CloudWatch log 6 | 7 | retVal= {} 8 | retVal["data"] = [] 9 | 10 | # Data is sent to Lambda via a HTTPS POST call. We want to get to the payload send by Snowflake 11 | event_body = event["body"] 12 | payload = json.loads(event_body) 13 | 14 | for row in payload["data"]: 15 | sflkRowRef = row[0] # This is how Snowflake keeps track of data as it gets returned 16 | input = row[1] 17 | 18 | # Pass each row through a processRow function 19 | response = processRow(input) 20 | 21 | # Top-level element must be "data". 22 | # Each item in the array should say which rowIndex it was sources from - sflkRowRef 23 | retVal["data"].append([sflkRowRef,response]) 24 | 25 | print(json.dumps(retVal)) 26 | return retVal 27 | 28 | 29 | 30 | def processRow(input): 31 | retVal = 0 32 | 33 | # Wrap the processing in a try/except 34 | # If a single row errors, it should not fail the entire ExternalFunction 35 | try: 36 | retVal = input + 3 37 | except: 38 | retVal = "Error processing this row..." 39 | 40 | return retVal -------------------------------------------------------------------------------- /_Template_Python/testData.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": [ 3 | [ 4 | 0, 5 | 5 6 | ], 7 | [ 8 | 1, 9 | "I am not a number!" 10 | ] 11 | ] 12 | } --------------------------------------------------------------------------------