├── .github └── main.workflow ├── .gitignore ├── app ├── README.md ├── api │ ├── gateway_deployment │ │ ├── main.tf │ │ ├── outputs.tf │ │ └── variables.tf │ ├── gateway_rest_api │ │ ├── main.tf │ │ ├── output.tf │ │ └── variables.tf │ └── swagger.json ├── cloudwatch │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── cognito │ └── user_pool │ │ ├── main.tf │ │ ├── outputs.tf │ │ └── variables.tf ├── dynamodb │ ├── README.md │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── kms │ ├── main.tf │ ├── output.tf │ └── variables.tf ├── lambda │ ├── README.md │ ├── artifacts │ │ ├── crud_handler.zip │ │ ├── producerjobs.zip │ │ └── surveyjobs.zip │ ├── functions │ │ ├── crud_handler │ │ │ ├── app.py │ │ │ ├── requirements.txt │ │ │ └── template.yml │ │ ├── producerjobs │ │ │ ├── dyno2sqs.py │ │ │ └── requirements.txt │ │ └── surveyjobs │ │ │ ├── SV_cGXWxvADgIihxrf-Example Qualtrics Output.csv │ │ │ ├── qualtrics.py │ │ │ ├── requirements.txt │ │ │ ├── sentiment.py │ │ │ └── tests │ │ │ ├── __init__.py │ │ │ ├── surveys_endpoint_response.json │ │ │ ├── test_qualtrics.py │ │ │ └── test_question_choices.py │ ├── lambda_role_policy.tpl │ ├── main.tf │ ├── outputs.tf │ └── variables.tf └── sqs │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── cicd └── plan │ ├── Dockerfile │ ├── README.md │ └── entrypoint.sh ├── collaboration ├── backend_aws.hcl ├── backend_remote.hcl ├── inputs.tf ├── main.tf └── outputs.tf ├── modules ├── terraform-google-ip-range-datasource │ ├── README.md │ ├── examples │ │ └── example_1 │ │ │ └── main.tf │ ├── main.tf │ ├── outputs.tf │ ├── scripts │ │ ├── __pycache__ │ │ │ └── datasource.cpython-37.pyc │ │ ├── datasource.py │ │ └── requirements.txt │ └── variables.tf └── trivial │ ├── examples │ └── main.tf │ ├── main.tf │ ├── outputs.tf │ └── variables.tf └── slides └── Take Terraform To The Next Level.pptx /.github/main.workflow: -------------------------------------------------------------------------------- 1 | workflow "Terraform" { 2 | resolves = ["terraform-apply-api-deployment"] 3 | on = "pull_request" 4 | } 5 | 6 | action "Debug" { 7 | uses = "actions/bin/sh@master" 8 | args = ["env"] 9 | } 10 | 11 | action "filter-to-pr-open-synced" { 12 | uses = "actions/bin/filter@master" 13 | args = "action 'opened|synchronize'" 14 | } 15 | 16 | action "terraform-init-cognito" { 17 | uses = "hashicorp/terraform-github-actions/init@v0.3.4" 18 | needs = "filter-to-pr-open-synced" 19 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 20 | env = { 21 | TF_ACTION_WORKING_DIR = "app/cognito/user_pool" 22 | } 23 | } 24 | 25 | action "terraform-validate-cognito" { 26 | uses = "hashicorp/terraform-github-actions/validate@v0.3.4" 27 | needs = "terraform-init-cognito" 28 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 29 | env = { 30 | TF_ACTION_WORKING_DIR = "app/cognito/user_pool" 31 | } 32 | } 33 | 34 | action "terraform-plan-cognito" { 35 | uses = "hashicorp/terraform-github-actions/plan@v0.3.4" 36 | needs = "terraform-validate-cognito" 37 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 38 | args = ["-out", "plan.tfplan"] 39 | env = { 40 | TF_ACTION_WORKING_DIR = "app/cognito/user_pool" 41 | } 42 | } 43 | 44 | action "terraform-apply-cognito" { 45 | uses = "hashicorp/terraform-github-actions/apply@v0.3.4" 46 | needs = "terraform-plan-cognito" 47 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 48 | args = ["plan.tfplan"] 49 | env = { 50 | TF_ACTION_WORKING_DIR = "app/cognito/user_pool" 51 | } 52 | } 53 | 54 | action "terraform-init-dynamodb" { 55 | uses = "hashicorp/terraform-github-actions/init@v0.3.4" 56 | needs = "filter-to-pr-open-synced" 57 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 58 | env = { 59 | TF_ACTION_WORKING_DIR = "app/dynamodb" 60 | } 61 | } 62 | 63 | action "terraform-validate-dynamodb" { 64 | uses = "hashicorp/terraform-github-actions/validate@v0.3.4" 65 | needs = "terraform-init-dynamodb" 66 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 67 | env = { 68 | TF_ACTION_WORKING_DIR = "app/dynamodb" 69 | } 70 | } 71 | 72 | action "terraform-workspace-dynamodb" { 73 | uses = "hashicorp/terraform-github-actions/plan@v0.3.4" 74 | needs = "terraform-validate-dynamodb" 75 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 76 | runs = ["sh", "-c", "cd $TF_ACTION_WORKING_DIR; terraform workspace select $GITHUB_HEAD_REF || terraform workspace new $GITHUB_HEAD_REF"] 77 | env = { 78 | TF_ACTION_WORKING_DIR = "app/dynamodb" 79 | } 80 | } 81 | 82 | action "terraform-plan-dynamodb" { 83 | uses = "hashicorp/terraform-github-actions/plan@v0.3.4" 84 | needs = "terraform-workspace-dynamodb" 85 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 86 | args = ["-out", "plan.tfplan"] 87 | env = { 88 | TF_ACTION_WORKING_DIR = "app/dynamodb" 89 | } 90 | } 91 | 92 | action "terraform-apply-dynamodb" { 93 | uses = "hashicorp/terraform-github-actions/apply@v0.3.4" 94 | needs = "terraform-plan-dynamodb" 95 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 96 | args = ["plan.tfplan"] 97 | env = { 98 | TF_ACTION_WORKING_DIR = "app/dynamodb" 99 | } 100 | } 101 | 102 | action "terraform-init-sqs" { 103 | uses = "hashicorp/terraform-github-actions/init@v0.3.4" 104 | needs = "filter-to-pr-open-synced" 105 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 106 | env = { 107 | TF_ACTION_WORKING_DIR = "app/sqs" 108 | } 109 | } 110 | 111 | action "terraform-validate-sqs" { 112 | uses = "hashicorp/terraform-github-actions/validate@v0.3.4" 113 | needs = "terraform-init-sqs" 114 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 115 | env = { 116 | TF_ACTION_WORKING_DIR = "app/sqs" 117 | } 118 | } 119 | 120 | action "terraform-workspace-sqs" { 121 | uses = "hashicorp/terraform-github-actions/plan@v0.3.4" 122 | needs = "terraform-validate-sqs" 123 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 124 | runs = ["sh", "-c", "cd $TF_ACTION_WORKING_DIR; terraform workspace select $GITHUB_HEAD_REF || terraform workspace new $GITHUB_HEAD_REF"] 125 | env = { 126 | TF_ACTION_WORKING_DIR = "app/sqs" 127 | } 128 | } 129 | 130 | action "terraform-plan-sqs" { 131 | uses = "hashicorp/terraform-github-actions/plan@v0.3.4" 132 | needs = "terraform-workspace-sqs" 133 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 134 | args = ["-out", "plan.tfplan"] 135 | env = { 136 | TF_ACTION_WORKING_DIR = "app/sqs" 137 | } 138 | } 139 | 140 | action "terraform-apply-sqs" { 141 | uses = "hashicorp/terraform-github-actions/apply@v0.3.4" 142 | needs = "terraform-plan-sqs" 143 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 144 | args = ["plan.tfplan"] 145 | env = { 146 | TF_ACTION_WORKING_DIR = "app/sqs" 147 | } 148 | } 149 | 150 | action "terraform-init-kms" { 151 | uses = "hashicorp/terraform-github-actions/init@v0.3.4" 152 | needs = "filter-to-pr-open-synced" 153 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 154 | env = { 155 | TF_ACTION_WORKING_DIR = "app/kms" 156 | } 157 | } 158 | 159 | action "terraform-validate-kms" { 160 | uses = "hashicorp/terraform-github-actions/validate@v0.3.4" 161 | needs = "terraform-init-kms" 162 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 163 | env = { 164 | TF_ACTION_WORKING_DIR = "app/kms" 165 | } 166 | } 167 | 168 | action "terraform-workspace-kms" { 169 | uses = "hashicorp/terraform-github-actions/plan@v0.3.4" 170 | needs = "terraform-validate-kms" 171 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 172 | runs = ["sh", "-c", "cd $TF_ACTION_WORKING_DIR; terraform workspace select $GITHUB_HEAD_REF || terraform workspace new $GITHUB_HEAD_REF"] 173 | env = { 174 | TF_ACTION_WORKING_DIR = "app/kms" 175 | } 176 | } 177 | 178 | action "terraform-plan-kms" { 179 | uses = "hashicorp/terraform-github-actions/plan@v0.3.4" 180 | needs = "terraform-workspace-kms" 181 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 182 | args = ["-out", "plan.tfplan"] 183 | env = { 184 | TF_ACTION_WORKING_DIR = "app/kms" 185 | } 186 | } 187 | 188 | action "terraform-apply-kms" { 189 | uses = "hashicorp/terraform-github-actions/apply@v0.3.4" 190 | needs = "terraform-plan-kms" 191 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 192 | args = ["plan.tfplan"] 193 | env = { 194 | TF_ACTION_WORKING_DIR = "app/kms" 195 | } 196 | } 197 | 198 | action "terraform-init-lambda" { 199 | uses = "hashicorp/terraform-github-actions/init@v0.3.4" 200 | needs = ["terraform-apply-kms", "terraform-apply-dynamodb", "terraform-apply-sqs", "terraform-apply-cognito",] 201 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 202 | env = { 203 | TF_ACTION_WORKING_DIR = "app/lambda" 204 | } 205 | } 206 | 207 | action "terraform-validate-lambda" { 208 | uses = "hashicorp/terraform-github-actions/validate@v0.3.4" 209 | needs = "terraform-init-lambda" 210 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 211 | env = { 212 | TF_ACTION_WORKING_DIR = "app/lambda" 213 | } 214 | } 215 | 216 | action "terraform-workspace-lambda" { 217 | uses = "hashicorp/terraform-github-actions/plan@v0.3.4" 218 | needs = "terraform-validate-lambda" 219 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 220 | runs = ["sh", "-c", "cd $TF_ACTION_WORKING_DIR; terraform workspace select $GITHUB_HEAD_REF || terraform workspace new $GITHUB_HEAD_REF"] 221 | env = { 222 | TF_ACTION_WORKING_DIR = "app/lambda" 223 | } 224 | } 225 | 226 | action "terraform-plan-lambda" { 227 | #uses = "hashicorp/terraform-github-actions/plan@v0.3.4" 228 | uses = "./cicd/plan" 229 | needs = "terraform-workspace-lambda" 230 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 231 | args = ["-out", "plan.tfplan"] 232 | env = { 233 | TF_ACTION_WORKING_DIR = "app/lambda" 234 | } 235 | } 236 | 237 | action "terraform-apply-lambda" { 238 | uses = "hashicorp/terraform-github-actions/apply@v0.3.4" 239 | needs = "terraform-plan-lambda" 240 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 241 | args = ["plan.tfplan"] 242 | env = { 243 | TF_ACTION_WORKING_DIR = "app/lambda" 244 | } 245 | } 246 | 247 | action "terraform-init-cloudwatch" { 248 | uses = "hashicorp/terraform-github-actions/init@v0.3.4" 249 | needs = ["terraform-apply-lambda"] 250 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 251 | env = { 252 | TF_ACTION_WORKING_DIR = "app/cloudwatch" 253 | } 254 | } 255 | 256 | action "terraform-validate-cloudwatch" { 257 | uses = "hashicorp/terraform-github-actions/validate@v0.3.4" 258 | needs = "terraform-init-cloudwatch" 259 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 260 | env = { 261 | TF_ACTION_WORKING_DIR = "app/cloudwatch" 262 | } 263 | } 264 | 265 | action "terraform-workspace-cloudwatch" { 266 | uses = "hashicorp/terraform-github-actions/plan@v0.3.4" 267 | needs = "terraform-validate-cloudwatch" 268 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 269 | runs = ["sh", "-c", "cd $TF_ACTION_WORKING_DIR; terraform workspace select $GITHUB_HEAD_REF || terraform workspace new $GITHUB_HEAD_REF"] 270 | env = { 271 | TF_ACTION_WORKING_DIR = "app/cloudwatch" 272 | } 273 | } 274 | 275 | action "terraform-plan-cloudwatch" { 276 | uses = "hashicorp/terraform-github-actions/plan@v0.3.4" 277 | needs = "terraform-workspace-cloudwatch" 278 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 279 | args = ["-out", "plan.tfplan"] 280 | env = { 281 | TF_ACTION_WORKING_DIR = "app/cloudwatch" 282 | } 283 | } 284 | 285 | action "terraform-apply-cloudwatch" { 286 | uses = "hashicorp/terraform-github-actions/apply@v0.3.4" 287 | needs = "terraform-plan-cloudwatch" 288 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 289 | args = ["plan.tfplan"] 290 | env = { 291 | TF_ACTION_WORKING_DIR = "app/cloudwatch" 292 | } 293 | } 294 | 295 | action "terraform-init-rest-api" { 296 | uses = "hashicorp/terraform-github-actions/init@v0.3.4" 297 | needs = ["terraform-apply-cloudwatch"] 298 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 299 | env = { 300 | TF_ACTION_WORKING_DIR = "app/api/gateway_rest_api" 301 | } 302 | } 303 | 304 | action "terraform-validate-rest-api" { 305 | uses = "hashicorp/terraform-github-actions/validate@v0.3.4" 306 | needs = "terraform-init-rest-api" 307 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 308 | env = { 309 | TF_ACTION_WORKING_DIR = "app/api/gateway_rest_api" 310 | } 311 | } 312 | 313 | action "terraform-plan-rest-api" { 314 | uses = "hashicorp/terraform-github-actions/plan@v0.3.4" 315 | needs = "terraform-validate-rest-api" 316 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 317 | args = ["-out", "plan.tfplan"] 318 | env = { 319 | TF_ACTION_WORKING_DIR = "app/api/gateway_rest_api" 320 | } 321 | } 322 | 323 | action "terraform-apply-rest-api" { 324 | uses = "hashicorp/terraform-github-actions/apply@v0.3.4" 325 | needs = "terraform-plan-rest-api" 326 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 327 | args = ["plan.tfplan"] 328 | env = { 329 | TF_ACTION_WORKING_DIR = "app/api/gateway_rest_api" 330 | } 331 | } 332 | 333 | action "terraform-init-api-deployment" { 334 | uses = "hashicorp/terraform-github-actions/init@v0.3.4" 335 | needs = ["terraform-apply-rest-api"] 336 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 337 | env = { 338 | TF_ACTION_WORKING_DIR = "app/api/gateway_deployment" 339 | } 340 | } 341 | 342 | action "terraform-validate-api-deployment" { 343 | uses = "hashicorp/terraform-github-actions/validate@v0.3.4" 344 | needs = "terraform-init-api-deployment" 345 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 346 | env = { 347 | TF_ACTION_WORKING_DIR = "app/api/gateway_deployment" 348 | } 349 | } 350 | 351 | action "terraform-workspace-api-deployment" { 352 | uses = "hashicorp/terraform-github-actions/plan@v0.3.4" 353 | needs = "terraform-validate-api-deployment" 354 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 355 | runs = ["sh", "-c", "cd $TF_ACTION_WORKING_DIR; terraform workspace select $GITHUB_HEAD_REF || terraform workspace new $GITHUB_HEAD_REF"] 356 | env = { 357 | TF_ACTION_WORKING_DIR = "app/api/gateway_deployment" 358 | } 359 | } 360 | 361 | action "terraform-plan-api-deployment" { 362 | uses = "hashicorp/terraform-github-actions/plan@v0.3.4" 363 | needs = "terraform-workspace-api-deployment" 364 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 365 | args = ["-out", "plan.tfplan"] 366 | env = { 367 | TF_ACTION_WORKING_DIR = "app/api/gateway_deployment" 368 | } 369 | } 370 | 371 | action "terraform-apply-api-deployment" { 372 | uses = "hashicorp/terraform-github-actions/apply@v0.3.4" 373 | needs = "terraform-plan-api-deployment" 374 | secrets = ["GITHUB_TOKEN", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] 375 | args = ["plan.tfplan"] 376 | env = { 377 | TF_ACTION_WORKING_DIR = "app/api/gateway_deployment" 378 | } 379 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Local .terraform directories 2 | **/.terraform/* 3 | 4 | # .tfstate files 5 | *.tfstate 6 | *.tfstate.* 7 | 8 | # .tfvars files 9 | *.tfvars 10 | -------------------------------------------------------------------------------- /app/README.md: -------------------------------------------------------------------------------- 1 | # Take-Terraform-to-the-Next-Level 2 | Examples and demos for the course 'Take Terraform to the Next Level' (https://learning.oreilly.com/search/?query=take%20terraform%20to%20the%20next%20level&extended_publisher_data=true&highlight=true&include_assessments=false&include_case_studies=true&include_courses=true&include_orioles=true&include_playlists=true&include_collections=false&include_notebooks=false&is_academic_institution_account=false&sort=relevance&facet_json=true&page=0) 3 | -------------------------------------------------------------------------------- /app/api/gateway_deployment/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.11.8" 3 | backend "s3" { 4 | bucket = "rojopolis-tf" 5 | key = "rojopolis-api-deployment" 6 | region = "us-east-1" 7 | dynamodb_table = "rojopolis-terraform-lock" 8 | } 9 | } 10 | 11 | provider "aws" { 12 | version = "~> 2.7" 13 | region = "us-east-1" 14 | } 15 | 16 | data "aws_caller_identity" "current" {} 17 | data "aws_region" "current" {} 18 | 19 | locals { 20 | aws_region = "${data.aws_region.current.name}" 21 | environment_slug = "${lower(terraform.workspace)}" 22 | } 23 | 24 | data "terraform_remote_state" "gateway_rest_api" { 25 | backend = "s3" 26 | workspace = "default" 27 | config = { 28 | bucket = "rojopolis-tf" 29 | key = "rojopolis-api-rest-api" 30 | region = "us-east-1" 31 | dynamodb_table = "rojopolis-terraform-lock" 32 | } 33 | } 34 | 35 | data "terraform_remote_state" "lambda" { 36 | backend = "s3" 37 | workspace = "${local.environment_slug}" 38 | config = { 39 | bucket = "rojopolis-tf" 40 | key = "rojopolis-lambda" 41 | region = "us-east-1" 42 | dynamodb_table = "rojopolis-terraform-lock" 43 | } 44 | } 45 | 46 | 47 | resource "aws_api_gateway_deployment" "rojopolis_api_gateway_deployment" { 48 | rest_api_id = "${data.terraform_remote_state.gateway_rest_api.outputs.rojopolis_rest_api_id}" 49 | description = "${local.environment_slug}" 50 | stage_name = "${local.environment_slug}" 51 | stage_description = "${local.environment_slug}" 52 | depends_on = ["aws_lambda_permission.crud_handler_lambda_permission"] 53 | } 54 | 55 | resource "aws_lambda_permission" "crud_handler_lambda_permission" { 56 | action = "lambda:InvokeFunction" 57 | function_name = "${data.terraform_remote_state.lambda.outputs.crud_handler_lambda_arn}" 58 | principal = "apigateway.amazonaws.com" 59 | 60 | source_arn = "${data.terraform_remote_state.gateway_rest_api.outputs.rojopolis_rest_api_execution_arn}/*/*/*" 61 | } -------------------------------------------------------------------------------- /app/api/gateway_deployment/outputs.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/app/api/gateway_deployment/outputs.tf -------------------------------------------------------------------------------- /app/api/gateway_deployment/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/app/api/gateway_deployment/variables.tf -------------------------------------------------------------------------------- /app/api/gateway_rest_api/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.11.8" 3 | backend "s3" { 4 | bucket = "rojopolis-tf" 5 | key = "rojopolis-api-rest-api" 6 | region = "us-east-1" 7 | dynamodb_table = "rojopolis-terraform-lock" 8 | } 9 | } 10 | 11 | provider "aws" { 12 | version = "~> 2.7" 13 | region = "us-east-1" 14 | } 15 | 16 | data "aws_caller_identity" "current" {} 17 | data "aws_region" "current" {} 18 | 19 | locals { 20 | aws_account_id = "${data.aws_caller_identity.current.account_id}" 21 | aws_region = "${data.aws_region.current.name}" 22 | environment_slug = "${var.git_branch}" 23 | } 24 | 25 | data "terraform_remote_state" "lambda" { 26 | backend = "s3" 27 | workspace = "${local.environment_slug}" 28 | config = { 29 | bucket = "rojopolis-tf" 30 | key = "rojopolis-lambda" 31 | region = "us-east-1" 32 | dynamodb_table = "rojopolis-terraform-lock" 33 | } 34 | } 35 | 36 | data "terraform_remote_state" "cognito_user_pool" { 37 | backend = "s3" 38 | workspace = "${local.environment_slug}" 39 | config = { 40 | bucket = "rojopolis-tf" 41 | key = "rojopolis-cognito-user-pool" 42 | region = "us-east-1" 43 | dynamodb_table = "rojopolis-terraform-lock" 44 | } 45 | } 46 | 47 | data "template_file" "swagger_json" { 48 | template = "${file("${path.module}/../swagger.json")}" 49 | vars = { 50 | crud_handler_lambda_qualified_arn = "${data.terraform_remote_state.lambda.outputs.crud_handler_lambda_qualified_arn}" 51 | rojopolis_user_pool_arn = "${data.terraform_remote_state.cognito_user_pool.outputs.rojopolis_user_pool_arn}" 52 | rojopolis_user_pool_resource_server_identifier = "${data.terraform_remote_state.cognito_user_pool.outputs.rojopolis_user_pool_resource_server_identifier}" 53 | } 54 | } 55 | 56 | resource "aws_api_gateway_rest_api" "rojopolis_api" { 57 | name = "rojopolis API" 58 | description = "rojopolis API" 59 | body = "${data.template_file.swagger_json.rendered}" 60 | } -------------------------------------------------------------------------------- /app/api/gateway_rest_api/output.tf: -------------------------------------------------------------------------------- 1 | output "rojopolis_rest_api_id" { 2 | description = "API Gateway resource" 3 | value = "${aws_api_gateway_rest_api.rojopolis_api.id}" 4 | } 5 | 6 | output "rojopolis_rest_api_execution_arn" { 7 | description = "API Gateway execution arn" 8 | value = "${aws_api_gateway_rest_api.rojopolis_api.execution_arn}" 9 | } -------------------------------------------------------------------------------- /app/api/gateway_rest_api/variables.tf: -------------------------------------------------------------------------------- 1 | variable "git_branch" { 2 | description = "The current branch of the GIT repo" 3 | type = "string" 4 | } -------------------------------------------------------------------------------- /app/cloudwatch/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.11.8" 3 | backend "s3" { 4 | bucket = "rojopolis-tf" 5 | key = "rojopolis-cloudwatch" 6 | region = "us-east-1" 7 | dynamodb_table = "rojopolis-terraform-lock" 8 | } 9 | } 10 | 11 | provider "aws" { 12 | version = "~> 2.7" 13 | region = "us-east-1" 14 | } 15 | 16 | data "aws_caller_identity" "current" {} 17 | data "aws_region" "current" {} 18 | 19 | locals { 20 | aws_account_id = "${data.aws_caller_identity.current.account_id}" 21 | aws_region = "${data.aws_region.current.name}" 22 | environment_slug = "${terraform.workspace}" 23 | } 24 | 25 | data "terraform_remote_state" "lambda" { 26 | backend = "s3" 27 | workspace = "${local.environment_slug}" 28 | config = { 29 | bucket = "rojopolis-tf" 30 | key = "rojopolis-lambda" 31 | region = "us-east-1" 32 | dynamodb_table = "rojopolis-terraform-lock" 33 | } 34 | } 35 | 36 | resource "aws_cloudwatch_event_rule" "producer_timer" { 37 | name = "producer-timer-${local.environment_slug}" 38 | description = "Execute producer Lambda" 39 | schedule_expression = "rate(${var.poll_interval} minutes)" 40 | } 41 | 42 | resource "aws_cloudwatch_event_target" "producer_target" { 43 | rule = "${aws_cloudwatch_event_rule.producer_timer.name}" 44 | arn = "${data.terraform_remote_state.lambda.outputs.producerjobs_handler_lambda_arn}" 45 | } 46 | 47 | resource "aws_lambda_permission" "producerjobs_handler_lambda_permission" { 48 | action = "lambda:InvokeFunction" 49 | function_name = "${data.terraform_remote_state.lambda.outputs.producerjobs_handler_lambda_arn}" 50 | principal = "events.amazonaws.com" 51 | 52 | source_arn = "arn:aws:events:${local.aws_region}:${local.aws_account_id}:rule/*" 53 | } -------------------------------------------------------------------------------- /app/cloudwatch/outputs.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/app/cloudwatch/outputs.tf -------------------------------------------------------------------------------- /app/cloudwatch/variables.tf: -------------------------------------------------------------------------------- 1 | variable "poll_interval" { 2 | description = "Number of minutes to wait between polling" 3 | type = "string" 4 | } 5 | -------------------------------------------------------------------------------- /app/cognito/user_pool/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.12.0" 3 | backend "s3" { 4 | bucket = "rojopolis-tf" 5 | key = "rojopolis-cognito-user-pool" 6 | region = "us-east-1" 7 | dynamodb_table = "rojopolis-terraform-lock" 8 | } 9 | } 10 | 11 | provider "aws" { 12 | version = "~> 2.7" 13 | region = "us-east-1" 14 | } 15 | 16 | data "aws_caller_identity" "current" {} 17 | data "aws_region" "current" {} 18 | 19 | locals { 20 | aws_account_id = data.aws_caller_identity.current.account_id 21 | aws_region = data.aws_region.current.name 22 | } 23 | 24 | resource "aws_cognito_user_pool" "rojopolis_user_pool" { 25 | name = "rojopolis User Pool" 26 | schema { 27 | name = "orgIds" 28 | attribute_data_type = "String" 29 | mutable = "true" 30 | string_attribute_constraints { 31 | min_length = "0" 32 | max_length = "2048" 33 | } 34 | } 35 | } 36 | 37 | resource "aws_cognito_user_pool_client" "rojopolis_user_pool_client" { 38 | name = "rojopolis App Client" 39 | user_pool_id = "${aws_cognito_user_pool.rojopolis_user_pool.id}" 40 | allowed_oauth_flows = ["implicit", "code"] 41 | explicit_auth_flows = ["USER_PASSWORD_AUTH"] 42 | allowed_oauth_scopes = concat([ 43 | "openid", 44 | "phone", 45 | "email", 46 | "profile", 47 | "aws.cognito.signin.user.admin"], 48 | aws_cognito_resource_server.rojopolis_user_pool_resource_server.scope_identifiers 49 | ) 50 | callback_urls = [ 51 | "https://staging.textrojopolis.com", 52 | "https://secure.textrojopolis.com", 53 | "http://localhost:6075"] 54 | logout_urls = ["https://localhost:6075"] # TODO: change this 55 | supported_identity_providers = ["COGNITO"] 56 | allowed_oauth_flows_user_pool_client = "true" 57 | } 58 | 59 | resource "aws_cognito_user_pool_domain" "rojopolis_user_pool_domain" { 60 | domain = "rojopolis-${local.aws_account_id}" 61 | user_pool_id = "${aws_cognito_user_pool.rojopolis_user_pool.id}" 62 | } 63 | 64 | resource "aws_cognito_resource_server" "rojopolis_user_pool_resource_server" { 65 | identifier = "https://api.rojopolis.com" 66 | name = "My 90 API" 67 | 68 | 69 | scope { 70 | scope_name = "agency.read" 71 | scope_description = "Users can read agency resources" 72 | } 73 | scope { 74 | scope_name = "questionResponses.read" 75 | scope_description = "Users can read questionResponses resources" 76 | } /* 77 | { 78 | scope_name = "questions.read" 79 | scope_description = "Users can read questions resources" 80 | }, 81 | { 82 | scope_name = "responsesSentimentMetadata.read" 83 | scope_description = "Users can read responsesSentimentMetadatare sources" 84 | }, 85 | { 86 | scope_name = "topics.read" 87 | scope_description = "Users can read topics resources" 88 | }, 89 | { 90 | scope_name = "responses.read" 91 | scope_description = "Users can read responses resources" 92 | }, 93 | { 94 | scope_name = "responsesMetadata.read" 95 | scope_description = "Users can read responses resources" 96 | }, 97 | { 98 | scope_name = "questionResponsesMetadata.read" 99 | scope_description = "Users can read responses resources" 100 | }, 101 | { 102 | scope_name = "questionChoices.read" 103 | scope_description = "Users can read question choices resources" 104 | }, 105 | ]*/ 106 | 107 | user_pool_id = "${aws_cognito_user_pool.rojopolis_user_pool.id}" 108 | } 109 | -------------------------------------------------------------------------------- /app/cognito/user_pool/outputs.tf: -------------------------------------------------------------------------------- 1 | output "rojopolis_user_pool_arn" { 2 | description = "Cognito User Pool resource" 3 | value = "${aws_cognito_user_pool.rojopolis_user_pool.arn}" 4 | } 5 | 6 | output "rojopolis_user_pool_client_id" { 7 | description = "Cognito User Pool Client ID" 8 | value = "${aws_cognito_user_pool_client.rojopolis_user_pool_client.id}" 9 | } 10 | 11 | output "rojopolis_user_pool_resource_server_identifier" { 12 | description = "Cognito User Pool Client ID" 13 | value = "${aws_cognito_resource_server.rojopolis_user_pool_resource_server.identifier}" 14 | } -------------------------------------------------------------------------------- /app/cognito/user_pool/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/app/cognito/user_pool/variables.tf -------------------------------------------------------------------------------- /app/dynamodb/README.md: -------------------------------------------------------------------------------- 1 | # DynamoDB resources 2 | 3 | DynamoDB tables are created with random suffixes. These IDs are passed to the API gateway as Stage Variables. -------------------------------------------------------------------------------- /app/dynamodb/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.12.0" 3 | backend "s3" { 4 | bucket = "rojopolis-tf" 5 | key = "rojopolis-dynamodb" 6 | region = "us-east-1" 7 | dynamodb_table = "rojopolis-terraform-lock" 8 | } 9 | } 10 | 11 | provider "aws" { 12 | version = "~> 2.7" 13 | region = "us-east-1" 14 | } 15 | 16 | locals { 17 | environment_slug = "${terraform.workspace}" 18 | } 19 | 20 | resource "aws_dynamodb_table" "agencies_table" { 21 | name = "agencies-${local.environment_slug}" 22 | read_capacity = 20 23 | write_capacity = 20 24 | hash_key = "Partition" 25 | range_key = "Sort" 26 | 27 | attribute { 28 | name = "Partition" 29 | type = "S" 30 | } 31 | 32 | attribute { 33 | name = "Sort" 34 | type = "S" 35 | } 36 | 37 | attribute { 38 | name = "LSI" 39 | type = "S" 40 | } 41 | 42 | local_secondary_index { 43 | name = "ParentIdIndex" 44 | range_key = "LSI" 45 | projection_type = "ALL" 46 | } 47 | } 48 | 49 | resource "aws_dynamodb_table" "producer_table" { 50 | name = "producer-${local.environment_slug}" 51 | read_capacity = 20 52 | write_capacity = 20 53 | hash_key = "AgencyId" 54 | range_key = "SurveyId" 55 | 56 | attribute { 57 | name = "AgencyId" 58 | type = "S" 59 | } 60 | 61 | attribute { 62 | name = "SurveyId" 63 | type = "S" 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /app/dynamodb/outputs.tf: -------------------------------------------------------------------------------- 1 | output "agencies_table_id" { 2 | description = "ID of agencies_table" 3 | value = "${aws_dynamodb_table.agencies_table.id}" 4 | } 5 | 6 | output "producer_table_id" { 7 | description = "ID of producer table" 8 | value = "${aws_dynamodb_table.producer_table.id}" 9 | } 10 | -------------------------------------------------------------------------------- /app/dynamodb/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/app/dynamodb/variables.tf -------------------------------------------------------------------------------- /app/kms/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.12.0" 3 | backend "s3" { 4 | bucket = "rojopolis-tf" 5 | key = "rojopolis-kms" 6 | region = "us-east-1" 7 | dynamodb_table = "rojopolis-terraform-lock" 8 | } 9 | } 10 | 11 | provider "aws" { 12 | version = "~> 2.7" 13 | region = "us-east-1" 14 | } 15 | 16 | locals { 17 | environment_slug = "${lower(terraform.workspace)}" 18 | } 19 | 20 | resource "aws_kms_key" "rojopolis_key" { 21 | description = "Encrypts sensitive fields in rojopolis API" 22 | } 23 | 24 | resource "aws_kms_alias" "rojopolis_key_alias" { 25 | name = "alias/rojopolis-key_${local.environment_slug}" 26 | target_key_id = "${aws_kms_key.rojopolis_key.key_id}" 27 | } 28 | -------------------------------------------------------------------------------- /app/kms/output.tf: -------------------------------------------------------------------------------- 1 | output "kms_key" { 2 | description = "KMS key" 3 | value = "${aws_kms_alias.rojopolis_key_alias.name}" 4 | } -------------------------------------------------------------------------------- /app/kms/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/app/kms/variables.tf -------------------------------------------------------------------------------- /app/lambda/README.md: -------------------------------------------------------------------------------- 1 | # Lambda function definitions. 2 | Place lambda code in subfolders of `functions` directory 3 | 4 | ## Running tests 5 | ~~~ 6 | cd lambda 7 | pip install -r test/requirements.txt 8 | python -m pytest --cov=functions/crud_handler/ test/test_crud_handler.py 9 | ~~~ 10 | 11 | ## Run locally 12 | Use sam local to run locally against cloud db. 13 | 14 | ~~~~ 15 | sam local start-api 16 | ~~~~ -------------------------------------------------------------------------------- /app/lambda/artifacts/crud_handler.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/app/lambda/artifacts/crud_handler.zip -------------------------------------------------------------------------------- /app/lambda/artifacts/producerjobs.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/app/lambda/artifacts/producerjobs.zip -------------------------------------------------------------------------------- /app/lambda/artifacts/surveyjobs.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/app/lambda/artifacts/surveyjobs.zip -------------------------------------------------------------------------------- /app/lambda/functions/crud_handler/app.py: -------------------------------------------------------------------------------- 1 | ''' 2 | CRUD Handler 3 | ''' 4 | import os 5 | import pprint 6 | import logging 7 | import json 8 | import decimal 9 | from ast import literal_eval 10 | from statistics import mean 11 | from functools import reduce 12 | from itertools import groupby 13 | from operator import itemgetter 14 | from operator import ior, iand 15 | import boto3 16 | from boto3.dynamodb.conditions import Key, Attr 17 | 18 | AGENCY_TABLE = None 19 | 20 | # Scales are currently hard-coded here and in the front end 21 | # In the future they should be moved to a shared location 22 | SCALES = { 23 | 'race': ['Asian/Pacific Islander', 'Black/African American', 24 | 'Hispanic/Latino', 'Native American', 'White/Caucasian', 25 | 'Prefer not to say', 'Other'], 26 | 'gender': ['Male', 'Female', 'Prefer not to say', 'Other'], 27 | 'age': ['Under 18', '18-24', '25-34', '35-44', '45-54', 'Over 54'], 28 | 'sentiment': ['MIXED', 'NEGATIVE', 'NEUTRAL', 'POSITIVE'] } 29 | 30 | 31 | 32 | 33 | # Fields expected to be retured with responses 34 | DEFAULT_RESPONSE_FIELDS = 'Choice,LSI,#d,Sort,#p, #t, Topic,Gender,Longitude,RespondentId,Age,Latitude,Origin,Race,IncidentCode,IncidentId,Sentiment,rojopolisEncounterScore,rojopolisGeneralScore,QuestionChoicesId,OpenResponse' 35 | DEFAULT_AGENCY_FIELDS = 'rojopolisEncounterScore,CityRacePercent,CityGenderPercent,#p,rojopolisGeneralScore,CityPopulation,Sort,LSI,ZoneofInterest,CityAgePercent,#n' 36 | DEFAULT_QUESTION_FIELDS = 'QuestionChoicesId,Sort,Category,#p,#t' 37 | 38 | def get_table(): 39 | '''Manages lazy global table instantiation''' 40 | global AGENCY_TABLE # pylint: disable=global-statement 41 | if not AGENCY_TABLE: 42 | dynamodb = boto3.resource('dynamodb') 43 | agency_table_id = os.environ['AGENCY_TABLE_ID'] 44 | AGENCY_TABLE = dynamodb.Table(agency_table_id) 45 | LOG.debug(f"AGENCY TABLE: {AGENCY_TABLE}") 46 | return AGENCY_TABLE 47 | 48 | 49 | def _logger(): 50 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') 51 | log_level = os.environ.get('LOGLEVEL', 'INFO') 52 | 53 | log = logging.getLogger('CRUD-API-Lambda') 54 | log.setLevel(log_level) 55 | 56 | stream_handler = logging.StreamHandler() 57 | stream_handler.setLevel(log_level) 58 | stream_handler.setFormatter(formatter) 59 | 60 | log.addHandler(stream_handler) 61 | return log 62 | 63 | LOG = _logger() 64 | 65 | class DecimalEncoder(json.JSONEncoder): 66 | ''' 67 | Helper class to convert a DynamoDB item to JSON. 68 | From: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GettingStarted.Python.04.html 69 | ''' 70 | def default(self, o): 71 | if isinstance(o, decimal.Decimal): 72 | if o % 1 > 0: 73 | return float(o) 74 | else: 75 | return int(o) 76 | return super(DecimalEncoder, self).default(o) 77 | 78 | 79 | # Make readable output, only for development purposes 80 | class PrettyLog(): 81 | def __init__(self, obj): 82 | self.obj = obj 83 | def __repr__(self): 84 | return pprint.pformat(self.obj) 85 | 86 | # Handler through which all calls flow 87 | def entrypoint(event, context): 88 | LOG.info(f"event: {event}") 89 | 90 | # We only support GET 91 | if event['httpMethod'] != 'GET': 92 | raise NotImplementedError(f"httpMethod not supported: {event['httpMethod']}") 93 | 94 | route_base = [x for x in event['path'].split('/') if x != ''][0] 95 | 96 | if 'aId' not in event['pathParameters']: 97 | raise TypeError('aId required') 98 | aId = event['pathParameters']['aId'] 99 | 100 | LOG.debug(route_base) 101 | queryStringParameters = event['queryStringParameters'] or {} 102 | 103 | if route_base == 'agency': 104 | response = agency(aId) 105 | elif route_base == 'questionResponsesMetadata': 106 | response = questionResponsesMetadata(aId, **queryStringParameters) 107 | elif route_base == 'responsesSentimentMetadata': 108 | response = responsesSentimentMetadata(aId, **queryStringParameters) 109 | elif route_base == 'responsesMetadata': 110 | response = responsesMetadata(aId, **queryStringParameters) 111 | elif route_base == 'questions': 112 | response = questions(aId, **queryStringParameters) 113 | elif route_base == 'topics': 114 | response = topics(aId,**queryStringParameters) 115 | elif route_base == 'responses': 116 | response = responses(aId, **queryStringParameters) 117 | elif route_base == 'questionChoices': 118 | response = questionChoices(aId, **queryStringParameters) 119 | else: 120 | raise NotImplementedError(f"Path: {event['path']!r}") 121 | 122 | try: 123 | return { "statusCode": 200, 124 | "headers": { 125 | "Access-Control-Allow-Origin": "*", 126 | "Access-Control-Allow-Method": "*", 127 | "Access-Control-Allow-Headers" : "Content-Type,X-Amz-Date,Authorization,X-Api-Key" 128 | }, 129 | "body": json.dumps(response)} 130 | except: 131 | LOG.exception(response) 132 | raise 133 | 134 | 135 | def agency(aId): 136 | # Enforce key convention 137 | if not aId.startswith('AID-'): 138 | raise TypeError("Improper agency key prefix") 139 | 140 | LOG.info(f"AID: '{aId}'") 141 | response = get_table().query( 142 | KeyConditionExpression=Key('Partition').eq(aId) & Key('Sort').eq('DeptData'), 143 | ProjectionExpression=DEFAULT_AGENCY_FIELDS, 144 | ExpressionAttributeNames={"#p":"Partition", "#n":"Name"} 145 | ) 146 | response['Items'] = _cast_ints(response['Items']) 147 | return response 148 | 149 | 150 | def questionResponsesMetadata(aId, 151 | question, 152 | startDate=None, 153 | endDate=None, 154 | age=None, 155 | gender=None, 156 | race=None, 157 | sentiment=None, 158 | origin=None, 159 | geo=None, 160 | topic=None): 161 | response = _responses(aId, 162 | question=question, 163 | startDate=startDate, 164 | endDate=endDate, 165 | age=age, 166 | gender=gender, 167 | race=race, 168 | sentiment=sentiment, 169 | origin=origin, 170 | geo=geo, 171 | topic=topic) 172 | response['Items'] = count_by_scale(response['Items']) 173 | return response 174 | 175 | 176 | def responsesSentimentMetadata(aId, 177 | question=None, 178 | startDate=None, 179 | endDate=None, 180 | age=None, 181 | gender=None, 182 | race=None, 183 | sentiment=None, 184 | origin=None, 185 | geo=None, 186 | topic=None): 187 | response = _responses(aId, 188 | question=question, 189 | startDate=startDate, 190 | endDate=endDate, 191 | age=age, 192 | gender=gender, 193 | race=race, 194 | sentiment=sentiment, 195 | origin=origin, 196 | geo=geo, 197 | topic=topic) 198 | response['Items'] = count_by_scale(response['Items'], group_field='Sentiment') 199 | return response 200 | 201 | 202 | def count_by_scale(data, group_field='Choice'): 203 | ''' 204 | Aggregate response data grouped by scale values 205 | and it's quetions possible scale 206 | 207 | group_field: top level field for grouping, 'Choice' and 'Sentiment' are 208 | only supported fields. 209 | ''' 210 | if not data: 211 | return {'age':[], 'race': [], 'gender': [], 'sentiment': [], 'dayCount': []} 212 | 213 | scales = SCALES 214 | 215 | if group_field == 'Choice': 216 | # All responses are for same question => same question scale 217 | indices = _get_question_choices_count(data[0]) 218 | elif group_field == 'Sentiment': 219 | indices = range(len(scales['sentiment'])) 220 | 221 | metadata = { 222 | # Age 223 | 'age': field_count_by_scale('Age', data, indices, keys=scales['age'], group_field=group_field), 224 | # Race 225 | 'race': field_count_by_scale('Race', data, indices, keys=scales['race'], group_field=group_field), 226 | # Gender 227 | 'gender': field_count_by_scale('Gender', data, indices, keys=scales['gender'], group_field=group_field), 228 | # Sentiment 229 | 'sentiment': field_count_by_scale('Sentiment', data, indices, keys=scales['sentiment'], group_field=group_field), 230 | # DayCount 231 | 'dayCount': field_count_by_scale('Date', data, indices, group_field=group_field) } 232 | return metadata 233 | 234 | 235 | def _get_question_choices_count(item): 236 | aid = item['Partition'] 237 | questionChoiceId = item['QuestionChoicesId'] 238 | response = questionChoices(aid, questionChoiceId) 239 | return tuple(range(len(response['Items'][questionChoiceId]))) 240 | 241 | 242 | def _groupby(field_name, data): 243 | # skip items lacking the field 244 | filtered = [x for x in data if field_name in x] 245 | 246 | # do same as cytoolz groupby but using standard lib 247 | filtered.sort(key=itemgetter(field_name)) 248 | grouped = {i:list(j) for i,j in groupby(filtered, itemgetter(field_name))} 249 | return grouped 250 | 251 | def field_count_by_scale(field_name, data, indices, group_field='Choice', keys=None): 252 | responses = [] 253 | if group_field == 'Sentiment': 254 | # Sentiment is sparely populated, so replace non-response with 255 | # value that won't be reported 256 | for row in data: 257 | if 'Sentiment' not in row: 258 | row['Sentiment'] = len(indices) 259 | 260 | grouped = _groupby(field_name, data) 261 | 262 | # All fields except Date have a set number of possible responses 263 | keys = range(len(keys)) if keys else sorted(grouped.keys()) 264 | 265 | for i in keys: 266 | group = grouped.get(i, []) 267 | response = [len(_groupby(group_field, group).get(x, [])) for x in indices] 268 | responses.append(response) 269 | 270 | return responses 271 | 272 | def responsesMetadata(aId, 273 | startDate=None, 274 | endDate=None, 275 | age=None, 276 | gender=None, 277 | race=None, 278 | sentiment=None, 279 | origin=None, 280 | geo=None, 281 | topic=None): 282 | response = _responses(aId, 283 | startDate=startDate, 284 | endDate=endDate, 285 | age=age, 286 | gender=gender, 287 | race=race, 288 | sentiment=sentiment, 289 | origin=origin, 290 | geo=geo, 291 | topic=topic) 292 | response['Items'] = count_and_mean(response['Items']) 293 | return response 294 | 295 | 296 | def count_and_mean(data): 297 | ''' 298 | Aggregate data grouped by scale values, count it, and 299 | calculate mean for rojopolis score(s). 300 | ''' 301 | scales = SCALES 302 | 303 | metadata = { 304 | # Age 305 | 'age': field_count_score_avg('Age', data, scales['age']), 306 | # Race 307 | 'race': field_count_score_avg('Race', data, scales['race']), 308 | # Gender 309 | 'gender': field_count_score_avg('Gender', data, scales['gender']), 310 | # Sentiment 311 | 'sentiment': field_count_score_avg('Sentiment', data, scales['sentiment']), 312 | # DayCount 313 | 'dayCount': field_count_score_avg('Date', data) } 314 | return metadata 315 | 316 | def field_count_score_avg(field_name, data, keys=None): 317 | responses = [] 318 | grouped = _groupby(field_name, data) 319 | 320 | # All fields except Date have a set number of possible responses 321 | keys = range(len(keys)) if keys else sorted(grouped.keys()) 322 | 323 | if len(grouped.keys()) > len(keys): 324 | raise ValueError(f"More groups found than keys for field '{field_name}'") 325 | 326 | for i in keys: 327 | group = grouped.get(i, []) 328 | rojopolisGeneralScoreAvgs = [x['rojopolisGeneralScore'] for x in group if 'rojopolisGeneralScore' in x] or [0] 329 | rojopolisEncounterScores = [x['rojopolisEncounterScore'] for x in group if 'rojopolisEncounterScore' in x] or [0] 330 | response = { 331 | 'count': len(group), 332 | 'rojopolisGeneralScoreAvg': mean(rojopolisGeneralScoreAvgs) if group else 0, 333 | 'rojopolisEncounterScoreAvg': mean(rojopolisEncounterScores) if group else 0 334 | } 335 | responses.append(response) 336 | return responses 337 | 338 | def questions(aId, limit=None, exclusiveStartKey=None): 339 | params = { 'KeyConditionExpression':Key('Partition').eq(aId) & Key('Sort').begins_with('QID'), 340 | 'ProjectionExpression':DEFAULT_QUESTION_FIELDS, 341 | 'ExpressionAttributeNames':{"#p":"Partition", "#t":"Text"} } 342 | 343 | if limit is not None: 344 | params['Limit'] = int(limit) 345 | 346 | if exclusiveStartKey is not None: 347 | params['ExclusiveStartKey'] = literal_eval(exclusiveStartKey) 348 | 349 | response = get_table().query(**params) 350 | return response 351 | 352 | 353 | def topics(aId): 354 | response = get_table().query( 355 | KeyConditionExpression=Key('Partition').eq(aId) & Key('Sort').begins_with('TID') 356 | ) 357 | return response 358 | 359 | 360 | 361 | def questionChoices(aId, qcid=None, limit=None, exclusiveStartKey=None): 362 | params = {} 363 | 364 | if qcid: 365 | params['KeyConditionExpression']=Key('Partition').eq(aId) & Key('Sort').eq(qcid) 366 | else: 367 | params['KeyConditionExpression']=Key('Partition').eq(aId) & Key('Sort').begins_with('QCID') 368 | 369 | if limit is not None: 370 | params['Limit'] = int(limit) 371 | 372 | if exclusiveStartKey is not None: 373 | params['ExclusiveStartKey'] = literal_eval(exclusiveStartKey) 374 | 375 | response = get_table().query(**params) 376 | 377 | # Convert Choices strings to tuples 378 | response['Items'] = _convert_to_map(response['Items']) 379 | return response 380 | 381 | 382 | def _convert_to_map(items): 383 | return {x['Sort']:literal_eval(x['Choices']) for x in items} 384 | 385 | 386 | def responses(aId, 387 | startDate=None, 388 | endDate=None, 389 | age=None, 390 | gender=None, 391 | race=None, 392 | sentiment=None, 393 | origin=None, 394 | geo=None, 395 | topic=None, 396 | exclusiveStartKey=None, 397 | limit=None): 398 | response = _responses(aId, 399 | startDate=startDate, 400 | endDate=endDate, 401 | age=age, 402 | gender=gender, 403 | race=race, 404 | sentiment=sentiment, 405 | origin=origin, 406 | geo=geo, 407 | topic=topic, 408 | exclusiveStartKey=exclusiveStartKey, 409 | limit=limit) 410 | return response 411 | 412 | 413 | def _responses(aId, 414 | question=None, 415 | startDate=None, 416 | endDate=None, 417 | age=None, 418 | gender=None, 419 | race=None, 420 | sentiment=None, 421 | origin=None, 422 | geo=None, 423 | topic=None, 424 | projectionExpression=None, 425 | exclusiveStartKey=None, 426 | limit=None): 427 | ProjectionExpression = projectionExpression or DEFAULT_RESPONSE_FIELDS 428 | filters = [] 429 | if question: 430 | filters.append(Attr('LSI').eq(question)) 431 | if startDate: 432 | filters.append(Attr('Date').gte(str(startDate))) 433 | if endDate: 434 | filters.append(Attr('Date').lte(str(endDate))) 435 | if age: 436 | age_filter = reduce(ior, [Attr('Age').eq(str(x)) for x in age]) 437 | filters.append(age_filter) 438 | if gender: 439 | gender_filter = reduce(ior, [Attr('Gender').eq(str(x)) for x in gender]) 440 | filters.append(gender_filter) 441 | if race: 442 | race_filter = reduce(ior, [Attr('Race').eq(str(x)) for x in race]) 443 | filters.append(race_filter) 444 | if sentiment: 445 | sentiment = sentiment.split(',') if isinstance(sentiment, str) else sentiment 446 | sentiment_filter = reduce(ior, [Attr('Sentiment').eq(str(x)) for x in sentiment]) 447 | filters.append(sentiment_filter) 448 | if origin: 449 | origin_filter = reduce(ior, [Attr('Origin').eq(str(x)) for x in origin]) 450 | filters.append(origin_filter) 451 | if geo: 452 | # [bottom left coordinates, upper right coordinates] 453 | # 27.449790329784214%2C-142.55859375000003%2C53.592504809039376%2C-32.69531250000001 454 | # Argument comes in as single string 455 | # Need to convert to use offset fields 456 | geo = geo.split(',') 457 | filters.append(Attr('LatitudeOffset').gte(_convert_latitude(geo[0]))) 458 | filters.append(Attr('LongitudeOffset').gte(_convert_longitude(geo[1]))) 459 | filters.append(Attr('LatitudeOffset').lte( _convert_latitude(geo[2]))) 460 | filters.append(Attr('LongitudeOffset').lte(_convert_longitude(geo[3]))) 461 | if topic: 462 | filters.append(Attr('Topic').eq(str(topic))) 463 | 464 | params = {'KeyConditionExpression':Key('Partition').eq(aId) & Key('Sort').begins_with('RID'), 465 | 'ProjectionExpression':ProjectionExpression, 466 | 'ExpressionAttributeNames':{"#p":"Partition", "#d": "Date", "#t": "Text"}} 467 | 468 | if filters: 469 | params['FilterExpression'] = reduce(iand, filters) 470 | 471 | if limit is not None: 472 | params['Limit'] = int(limit) 473 | 474 | if exclusiveStartKey is not None: 475 | params['ExclusiveStartKey'] = literal_eval(exclusiveStartKey) 476 | 477 | response = get_table().query(**params) 478 | LOG.debug(f"Response before casting ints: {response}") 479 | 480 | response['Items'] = _cast_ints(response['Items']) 481 | return response 482 | 483 | 484 | def _convert_latitude(value): 485 | return f"{float(value):019.15F}" 486 | 487 | 488 | def _convert_longitude(value): 489 | return f"{(float(value) + 200):019.15F}" 490 | 491 | 492 | def _cast_ints(items): 493 | ''' 494 | Temp method to cast int fields until we find a better way. 495 | ''' 496 | new_items = [] 497 | for item in items: 498 | new_item = {} 499 | for key, value in item.items(): 500 | if isinstance(value, list): 501 | new_item[key] = [_cast_num(x) for x in value] 502 | else: 503 | new_item[key] = _cast_num(value) 504 | new_items.append(new_item) 505 | 506 | return new_items 507 | 508 | 509 | def _cast_num(value): 510 | if hasattr(value, 'isdigit') and value.isdigit(): 511 | return int(value) 512 | else: 513 | return _cast_float(value) 514 | 515 | 516 | def _cast_float(value): 517 | try: 518 | return float(value) 519 | except ValueError: 520 | return value 521 | 522 | -------------------------------------------------------------------------------- /app/lambda/functions/crud_handler/requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | -------------------------------------------------------------------------------- /app/lambda/functions/crud_handler/template.yml: -------------------------------------------------------------------------------- 1 | iAWSTemplateFormatVersion: '2010-09-09' 2 | Transform: AWS::Serverless-2016-10-31 3 | Resources: 4 | 5 | CrudHandler: 6 | Type: AWS::Serverless::Function 7 | Properties: 8 | Runtime: python3.6 9 | Handler: app.entrypoint 10 | Events: 11 | Getagency: 12 | Type: Api 13 | Properties: 14 | Path: /agency/{aId} 15 | Method: get 16 | GetquestionResponses: 17 | Type: Api 18 | Properties: 19 | Path: /questionResponses/{aId} 20 | Method: get 21 | Getquestions: 22 | Type: Api 23 | Properties: 24 | Path: /questions/{aId} 25 | Method: get 26 | Gettopics: 27 | Type: Api 28 | Properties: 29 | Path: /topics/{aId} 30 | Method: get 31 | Getresponses: 32 | Type: Api 33 | Properties: 34 | Path: /responses/{aId} 35 | Method: get 36 | GetresponsesMetadata: 37 | Type: Api 38 | Properties: 39 | Path: /responsesMetadata/{aId} 40 | Method: get 41 | 42 | Environment: 43 | Variables: 44 | AGENCY_TABLE_ID: 45 | LOGLEVEL: 46 | -------------------------------------------------------------------------------- /app/lambda/functions/producerjobs/dyno2sqs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Dynamo to SQS 3 | """ 4 | 5 | import click 6 | import boto3 7 | import json 8 | import sys 9 | import os 10 | 11 | DYNAMODB = boto3.resource('dynamodb') 12 | SQS = boto3.client("sqs") 13 | 14 | #SETUP LOGGING 15 | import logging 16 | from pythonjsonlogger import jsonlogger 17 | 18 | LOG = logging.getLogger() 19 | LOG.setLevel(logging.INFO) 20 | logHandler = logging.StreamHandler() 21 | formatter = jsonlogger.JsonFormatter() 22 | logHandler.setFormatter(formatter) 23 | LOG.addHandler(logHandler) 24 | 25 | def scan_table(table): 26 | """Scans table and return results""" 27 | 28 | LOG.info(f"Scanning Table {table}") 29 | producer_table = DYNAMODB.Table(table) 30 | response = producer_table.scan() 31 | items = response['Items'] 32 | LOG.info(f"Found {len(items)} surveys") 33 | return items 34 | 35 | def send_sqs_msg(msg, queue_name, delay=0): 36 | """Send SQS Message 37 | 38 | Expects an SQS queue_name and msg in a dictionary format. 39 | Returns a response dictionary. 40 | """ 41 | 42 | queue_url = SQS.get_queue_url(QueueName=queue_name)["QueueUrl"] 43 | queue_send_log_msg = "Send message to queue url: %s, with body: %s" %\ 44 | (queue_url, msg) 45 | LOG.info(queue_send_log_msg) 46 | json_msg = json.dumps(msg) 47 | response = SQS.send_message( 48 | QueueUrl=queue_url, 49 | MessageBody=json_msg, 50 | DelaySeconds=delay) 51 | queue_send_log_msg_resp = "Message Response: %s for queue url: %s" %\ 52 | (response, queue_url) 53 | LOG.info(queue_send_log_msg_resp) 54 | return response 55 | 56 | def send_emissions(table, queue_name): 57 | """Send Emissions""" 58 | 59 | surveys = scan_table(table=table) 60 | for survey in surveys: 61 | LOG.info(f"Sending survey {survey} to queue: {queue_name}") 62 | response = send_sqs_msg(survey, queue_name=queue_name) 63 | LOG.debug(response) 64 | 65 | def entrypoint(event, context): 66 | ''' 67 | Lambda entrypoint 68 | ''' 69 | LOG.info(f"event {event}, context {context}", extra=os.environ) 70 | cli.main(args=['emit', 71 | '--table', os.environ.get('PRODUCER_JOB_TABLE'), 72 | '--queue', os.environ.get('PRODUCER_JOB_QUEUE') 73 | ] 74 | ) 75 | 76 | 77 | @click.group() 78 | def cli(): 79 | pass 80 | 81 | @cli.command() 82 | @click.option("--table", envvar="PRODUCER_JOB_TABLE", 83 | help="Dynamo Table") 84 | @click.option("--queue", envvar="PRODUCER_JOB_QUEUE", 85 | help="SQS") 86 | def emit(table, queue): 87 | """Emit Surveys from DynamoDB into SQS 88 | 89 | To run with environmental variables 90 | export PRODUCER_JOB_TABLE="foo";\ 91 | export PRODUCER_JOB_QUEUE="bar";\ 92 | python dyno2sqs.py emit 93 | 94 | """ 95 | 96 | LOG.info(f"Running Click emit with table: {table}, queue: {queue}") 97 | try: 98 | send_emissions(table=table, queue_name=queue) 99 | except AttributeError: 100 | LOG.exception(f"Error, check passed in values: table: {table}, queue: {queue}") 101 | sys.exit(1) 102 | 103 | 104 | if __name__ == "__main__": 105 | cli() -------------------------------------------------------------------------------- /app/lambda/functions/producerjobs/requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | click 3 | python-json-logger -------------------------------------------------------------------------------- /app/lambda/functions/surveyjobs/SV_cGXWxvADgIihxrf-Example Qualtrics Output.csv: -------------------------------------------------------------------------------- 1 | StartDate,EndDate,Status,IPAddress,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,RecipientLastName,RecipientFirstName,RecipientEmail,ExternalReference,LocationLatitude,LocationLongitude,DistributionChannel,UserLanguage,Q1,Q2,Q3,Q5,Q4,Q6,Q7,Q9,Q_RecipientPhoneNumber,RespondentID,Agency,Partition,Sort,LSI,PhoneNumber,Code,PoliceReportedRace,PoliceReportedAge,PoliceReportedGender,Latitude,Longitude,Create New Field or Choose From Dropdown...,Category,Scale,Origin 2 | Start Date,End Date,Response Type,IP Address,Progress,Duration (in seconds),Finished,Recorded Date,Response ID,Recipient Last Name,Recipient First Name,Recipient Email,External Data Reference,Location Latitude,Location Longitude,Distribution Channel,User Language,"Hi, this is rojopolis. We're to help make public safety improvement and guide you to other services you may need. Your feedback is ANONYMOUS, and it will go directly to your police chief. We will not disclose who you are. (Text ""STOP"" to stop receiving messages, standard messaging rates apply).","On a scale of 0-100, did the responding officers",What public safety issue needs to be addressed for you to feel safer in your neighborhood?,"Ok, can you please explain?","We're so sorry that you're having to go through all of this. We can put you directly in touch with services and people that can provide assistance if you need it. If you want us to do this, please select from the following list, and we'll have someone follow up with you within 48 hours:","Thanks for your feedback. You can view what others in your city and neighborhood are saying here: www.textrojopolis.com/community. If you have anything else to say, please text it now.","Ok, someone will be in touch. Thanks for your feedback. You can view what others in your city and neighborhood are saying here: www.textrojopolis.com/communiy. If you have anything else to say, please text it now.",Click to write the question text,Q_RecipientPhoneNumber,RespondentID,Agency,Partition,Sort,LSI,PhoneNumber,Code,PoliceReportedRace,PoliceReportedAge,PoliceReportedGender,Latitude,Longitude,Create New Field or Choose From Dropdown...,Category,Scale,Origin 3 | "{""ImportId"":""startDate"",""timeZone"":""Z""}","{""ImportId"":""endDate"",""timeZone"":""Z""}","{""ImportId"":""status""}","{""ImportId"":""ipAddress""}","{""ImportId"":""progress""}","{""ImportId"":""duration""}","{""ImportId"":""finished""}","{""ImportId"":""recordedDate"",""timeZone"":""Z""}","{""ImportId"":""_recordId""}","{""ImportId"":""recipientLastName""}","{""ImportId"":""recipientFirstName""}","{""ImportId"":""recipientEmail""}","{""ImportId"":""externalDataReference""}","{""ImportId"":""locationLatitude""}","{""ImportId"":""locationLongitude""}","{""ImportId"":""distributionChannel""}","{""ImportId"":""userLanguage""}","{""ImportId"":""QID1_TEXT""}","{""ImportId"":""QID2_TEXT""}","{""ImportId"":""QID3""}","{""ImportId"":""QID5_TEXT""}","{""ImportId"":""QID4""}","{""ImportId"":""QID6_TEXT""}","{""ImportId"":""QID7_TEXT""}","{""ImportId"":""QID9""}","{""ImportId"":""Q_RecipientPhoneNumber""}","{""ImportId"":""RespondentID""}","{""ImportId"":""Agency""}","{""ImportId"":""Partition""}","{""ImportId"":""Sort""}","{""ImportId"":""LSI""}","{""ImportId"":""PhoneNumber""}","{""ImportId"":""Code""}","{""ImportId"":""PoliceReportedRace""}","{""ImportId"":""PoliceReportedAge""}","{""ImportId"":""PoliceReportedGender""}","{""ImportId"":""Latitude""}","{""ImportId"":""Longitude""}","{""ImportId"":""Create New Field or Choose From Dropdown...""}","{""ImportId"":""Category""}","{""ImportId"":""Scale""}","{""ImportId"":""Origin""}" 4 | 2018-11-15 21:39:46,2018-11-15 21:39:46,2,,100,0,1,2018-11-15 21:39:46,R_9vosXWIAao1sL1b,,,,,47.6605987548828125,-122.291900634765625,test,,Iaculis potenti? Vel vestibulum ultricies laoreet placerat vel wisi duis.,7,1,,4,,Ultricies maecenas velit dolor ullamcorper temporibus. Integer sagittis lectus facilisi a.,,,340714413,,,,,,,,,,,,,Encounter,1,1 5 | 2018-11-15 21:39:46,2018-11-15 21:39:46,2,,100,0,1,2018-11-15 21:39:46,R_bpfsF1Fhkwk7RCB,,,,,47.6605987548828125,-122.291900634765625,test,,"Euismod? Justo fringilla auctor egestas velit egestas ut dapibus, atque integer.",10,2,,2,,Neque consequat enim massa! Fusce nulla sagittis. Pharetra. Ante suspendisse.,,,655201607,,,,,,,,,,,,,Encounter,1,1 6 | 2018-11-15 21:39:46,2018-11-15 21:39:46,2,,100,0,1,2018-11-15 21:39:46,R_0wGEfwlIw8SWnfD,,,,,47.6605987548828125,-122.291900634765625,test,,"Nec integer atque lorem interdum! A. Integer suscipit lorem lectus consequat, sodales mi lorem.",10,1,,3,,Eros porta mauris primis. Tellus. Pretium. Orci vestibulum quis blandit viverra.,,,849070583,,,,,,,,,,,,,Encounter,1,1 7 | 2018-11-15 21:39:46,2018-11-15 21:39:46,2,,100,0,1,2018-11-15 21:39:46,R_e5xuzXEDA27ATpX,,,,,47.6605987548828125,-122.291900634765625,test,,"Turpis ultricies proin odio morbi velit orci in diam, ut culpa.",4,5,,1,,"Neque vulputate molestie donec quam, viverra blandit egestas tempus diam. Interdum per.",,,413977300,,,,,,,,,,,,,Encounter,1,1 8 | 2018-11-15 21:39:46,2018-11-15 21:39:46,2,,100,0,1,2018-11-15 21:39:46,R_6EyCOrJPPZzFgNf,,,,,47.6605987548828125,-122.291900634765625,test,,Vestibulum! Etiam nibh arcu aliquet? Praesent integer eleifend at? Eu accusamus.,8,4,"Curabitur vestibulum maecenas ut tellus phasellus natoque ipsum, ridiculus elit.",4,,Lorem cursus augue nullam elit morbi massa per! Bibendum rutrum duis per vulputate dui. Ante.,,,60938510,,,,,,,,,,,,,Encounter,1,1 9 | 2018-11-15 21:39:46,2018-11-15 21:39:46,2,,100,0,1,2018-11-15 21:39:46,R_eeNwoSrWKyvqixf,,,,,47.6605987548828125,-122.291900634765625,test,,Ac odio laoreet? Justo facilisis! Nunc rutrum morbi etiam dapibus mi venenatis! Laoreet primis odio.,5,5,,1,,"Maecenas rutrum et nonummy dapibus elementum eleifend rutrum ultrices aenean sodales, est metus sapien nibh.",,,94991820,,,,,,,,,,,,,Encounter,1,1 10 | 2018-11-15 21:39:46,2018-11-15 21:39:46,2,,100,0,1,2018-11-15 21:39:46,R_3rz4NyWcjc8t6rX,,,,,47.6605987548828125,-122.291900634765625,test,,"Pharetra aliquet primis accusamus risus, integer sollicitudin nulla nullam? Sagittis. Fermentum primis ante facilisis! Praesent.",3,2,,4,,Venenatis temporibus placerat. In wisi aenean cursus non montes urna turpis molestie purus.,,,953554244,,,,,,,,,,,,,Encounter,1,1 11 | 2018-11-15 21:39:46,2018-11-15 21:39:46,2,,100,0,1,2018-11-15 21:39:47,R_eDQEBd7UJY5fO9D,,,,,47.6605987548828125,-122.291900634765625,test,,Mauris accusamus cursus montes sit suscipit. Erat dui velit urna arcu. Ut atque.,3,3,,4,,Ultrices elementum temporibus platea orci elementum sagittis euismod velit imperdiet consectetuer imperdiet! Vel. Leo fringilla.,,,976514056,,,,,,,,,,,,,Encounter,1,1 12 | 2018-11-15 21:39:47,2018-11-15 21:39:47,2,,100,0,1,2018-11-15 21:39:47,R_0uimE6vCcFMUDFb,,,,,47.6605987548828125,-122.291900634765625,test,,Nibh suscipit nulla venenatis fringilla enim. Elementum potenti cursus venenatis et cras.,8,3,,5,,Faucibus magna ridiculus. Rutrum? Cursus accusamus suspendisse vehicula laoreet eget tempor gravida eget sed ullamcorper.,,,230347384,,,,,,,,,,,,,Encounter,1,1 13 | 2018-11-15 21:39:47,2018-11-15 21:39:47,2,,100,0,1,2018-11-15 21:39:47,R_bpjuYopYD7vjR2J,,,,,47.6605987548828125,-122.291900634765625,test,,"Maecenas, commodo sed quis mauris temporibus venenatis pharetra. Ullamcorper ridiculus sapien potenti aliquet ultrices.",9,5,,3,,Rutrum pharetra amet eros dapibus scelerisque donec ac imperdiet sapien cursus felis? Elit fringilla.,,,227517295,,,,,,,,,,,,,Encounter,1,1 14 | 2018-11-15 21:39:47,2018-11-15 21:39:47,2,,100,0,1,2018-11-15 21:39:47,R_3BFORvyDysbCiTr,,,,,47.6605987548828125,-122.291900634765625,test,,Vulputate. Orci etiam mattis culpa faucibus posuere rutrum leo porta natoque imperdiet vel.,9,2,,5,,Mi ultrices faucibus! Vivamus nibh facilisis tellus magna aenean! Morbi? Velit sed.,,,270067540,,,,,,,,,,,,,Encounter,1,1 15 | 2018-11-15 21:39:47,2018-11-15 21:39:47,2,,100,0,1,2018-11-15 21:39:47,R_4GBE5dCOcQwk6fr,,,,,47.6605987548828125,-122.291900634765625,test,,Potenti sagittis facilisis imperdiet cursus nunc natoque ab ridiculus ipsum. Fringilla viverra.,2,1,,4,,Consequat platea lectus? Id ultricies non lorem ullamcorper cursus eget dictumst neque.,,,15228536,,,,,,,,,,,,,Encounter,1,1 16 | 2018-11-15 21:39:47,2018-11-15 21:39:47,2,,100,0,1,2018-11-15 21:39:47,R_4VCB5SLc8bI2x3D,,,,,47.6605987548828125,-122.291900634765625,test,,"Pellentesque fringilla metus lorem in! Mauris wisi odio culpa, tortor, vestibulum mauris dolor massa scelerisque.",8,5,,4,,Sit consequat aliquet. Eros urna. Mattis proin? Quam commodo aliquet egestas.,,,628151625,,,,,,,,,,,,,Encounter,1,1 17 | 2018-11-15 21:39:47,2018-11-15 21:39:47,2,,100,0,1,2018-11-15 21:39:47,R_6FK5jImFRWH7J09,,,,,47.6605987548828125,-122.291900634765625,test,,Faucibus luctus erat et pede? Donec? Primis. Elit convallis aliquet wisi.,0,4,Nunc consequat imperdiet cras rutrum ante consequat. Nunc id consectetuer pretium.,6,Potenti malesuada dignissim tempus. Amet vulputate? Sapien sollicitudin commodo tempor sapien. Tortor. Eleifend.,,,,119183016,,,,,,,,,,,,,Encounter,1,1 18 | 2018-11-15 21:39:47,2018-11-15 21:39:47,2,,100,0,1,2018-11-15 21:39:47,R_3sk9tjUYIwhi98x,,,,,47.6605987548828125,-122.291900634765625,test,,Molestie et. Natoque accusamus augue ultrices viverra! Auctor aliquet pharetra nibh primis blandit aenean.,3,4,A placerat tempor odio pede dictumst leo consectetuer sagittis malesuada porta.,2,,Nulla viverra eleifend magnis orci magnis ultricies. Arcu molestie diam culpa aenean in.,,,553920885,,,,,,,,,,,,,Encounter,1,1 19 | 2018-11-15 21:39:48,2018-11-15 21:39:48,2,,100,0,1,2018-11-15 21:39:48,R_3f7H7yXtQ7w3fVz,,,,,47.6605987548828125,-122.291900634765625,test,,Lectus pede atque egestas primis pellentesque ante quam. Vehicula at non? Et? Lorem urna.,10,5,,3,,"Nullam porttitor etiam? Quis, per sit tempus? Quis? Tempor tempus tortor mauris.",,,266677916,,,,,,,,,,,,,Encounter,1,1 20 | 2018-11-15 21:39:48,2018-11-15 21:39:48,2,,100,0,1,2018-11-15 21:39:48,R_9XJ0qjepQcxKCah,,,,,47.6605987548828125,-122.291900634765625,test,,"Ridiculus elit urna elementum magnis imperdiet orci augue turpis, suscipit commodo.",3,1,,3,,"Felis volutpat dolor magnis mauris mauris? A magnis! Rhoncus luctus magnis, dolorem commodo.",,,907964059,,,,,,,,,,,,,Encounter,1,1 21 | 2018-11-15 21:39:48,2018-11-15 21:39:48,2,,100,0,1,2018-11-15 21:39:48,R_aUZ05wQfGeeZ5Pf,,,,,47.6605987548828125,-122.291900634765625,test,,Lectus sed mi consectetuer sollicitudin ab nonummy sapien sem turpis.,7,2,,4,,Tempora facilisi ipsum gravida non maecenas! Eleifend. In diam dapibus porta.,,,841911459,,,,,,,,,,,,,Encounter,1,1 22 | 2018-11-15 21:39:48,2018-11-15 21:39:48,2,,100,0,1,2018-11-15 21:39:48,R_80ustZ7tmzQWQbH,,,,,47.6605987548828125,-122.291900634765625,test,,"Temporibus vehicula nullam aliquam, felis. Curabitur, magnis ab tellus dolorem at velit elit nibh imperdiet.",3,4,"In velit, suspendisse et? Maecenas temporibus viverra! Purus malesuada? Felis donec volutpat aliquam.",6,"Eu eros. Scelerisque justo suspendisse id! Orci suscipit rhoncus, lorem porta scelerisque nunc, cursus mi.",,,,249925281,,,,,,,,,,,,,Encounter,1,1 23 | 2018-11-15 21:39:48,2018-11-15 21:39:48,2,,100,0,1,2018-11-15 21:39:48,R_emVrhQA8RjhJKa9,,,,,47.6605987548828125,-122.291900634765625,test,,Consequat nulla. Per et commodo tempus? Non pede purus lacus dapibus. Eros.,3,4,"Sem in magnis. Accumsan, urna fermentum! Tellus? Lectus ultricies id natoque.",2,,"Pellentesque culpa elementum, etiam? Nulla praesent erat aliquam nec eget porttitor donec vitae dui est.",,,62298341,,,,,,,,,,,,,Encounter,1,1 24 | 2018-11-15 21:39:48,2018-11-15 21:39:48,2,,100,0,1,2018-11-15 21:39:48,R_9mLLhKZrYWWWz8F,,,,,47.6605987548828125,-122.291900634765625,test,,Platea bibendum! Phasellus mauris volutpat. Justo eros quam potenti eget.,6,4,Malesuada aliquet convallis molestie velit blandit blandit facilisis dolor diam.,3,,"Gravida dignissim felis quam in sed non, cursus malesuada per nonummy risus.",,,307205138,,,,,,,,,,,,,Encounter,1,1 25 | 2018-11-15 21:39:48,2018-11-15 21:39:48,2,,100,0,1,2018-11-15 21:39:48,R_6X61YWIUi7RRQnb,,,,,47.6605987548828125,-122.291900634765625,test,,Rhoncus venenatis diam gravida. Justo eget iaculis ante viverra quis mauris velit per sapien.,1,2,,6,A. Interdum? Tincidunt! Leo sapien suscipit commodo dictumst lorem volutpat sed felis.,,,,247883528,,,,,,,,,,,,,Encounter,1,1 26 | 2018-11-15 21:39:48,2018-11-15 21:39:48,2,,100,0,1,2018-11-15 21:39:48,R_2fT4ZMHnNunEKwd,,,,,47.6605987548828125,-122.291900634765625,test,,Accumsan vehicula consectetuer ridiculus dui. Consectetuer iaculis? Tortor faucibus tincidunt vel posuere. Pharetra ultricies. Eget.,7,5,,5,,"Nulla platea, nulla sed, scelerisque pretium! Accusamus donec eget? Interdum nullam commodo cras porta.",,,983353330,,,,,,,,,,,,,Encounter,1,1 27 | 2018-11-15 21:39:48,2018-11-15 21:39:48,2,,100,0,1,2018-11-15 21:39:49,R_eni5QGTE19RrzLL,,,,,47.6605987548828125,-122.291900634765625,test,,"Sapien lectus rhoncus aliquet eu velit leo, velit, interdum ab ridiculus placerat gravida ultricies.",4,1,,1,,Nec auctor lacus nullam. Laoreet ultricies accusamus sapien sodales in.,,,587202580,,,,,,,,,,,,,Encounter,1,1 28 | 2018-11-15 21:39:49,2018-11-15 21:39:49,2,,100,0,1,2018-11-15 21:39:49,R_1X15XmHZJ2epvHT,,,,,47.6605987548828125,-122.291900634765625,test,,"Non augue nibh! Ac. Fermentum integer euismod facilisi? Scelerisque, potenti? Nec proin facilisi.",4,4,"Duis est dignissim tincidunt cursus! Felis quam dignissim. Posuere vel dolorem, convallis.",4,,"Fringilla culpa facilisis sed iaculis commodo sapien turpis non, temporibus molestie? Tortor vitae et porttitor.",,,952094618,,,,,,,,,,,,,Encounter,1,1 29 | 2018-11-15 21:39:49,2018-11-15 21:39:49,2,,100,0,1,2018-11-15 21:39:49,R_8vlwNBjtcjka0jb,,,,,47.6605987548828125,-122.291900634765625,test,,Egestas. Eget nunc vulputate nullam! Non. Morbi at gravida enim curabitur pede. Maecenas fusce. Morbi.,4,3,,4,,Imperdiet. Facilisis dapibus nibh aenean venenatis facilisis. Diam laoreet mi mattis morbi pretium! Sagittis.,,,893759253,,,,,,,,,,,,,Encounter,1,1 30 | 2018-11-15 21:39:49,2018-11-15 21:39:49,2,,100,0,1,2018-11-15 21:39:49,R_8C9eKS1UHSMRw3z,,,,,47.6605987548828125,-122.291900634765625,test,,"Interdum ullamcorper metus diam! Proin, consequat in sollicitudin tempor sapien? Bibendum phasellus ac! Posuere sodales.",3,5,,6,"Orci facilisi faucibus. Scelerisque lorem aenean elit aenean. Rutrum, sit commodo.",,,,190287383,,,,,,,,,,,,,Encounter,1,1 31 | 2018-11-15 21:39:49,2018-11-15 21:39:49,2,,100,0,1,2018-11-15 21:39:49,R_9tMDyf4HGLgjjsp,,,,,47.6605987548828125,-122.291900634765625,test,,Felis atque phasellus aliquet quis eget. Amet! Ligula sit proin quis.,8,4,"Rhoncus pharetra. Curabitur felis sit tempus commodo vehicula egestas, blandit auctor fringilla.",6,Ligula pellentesque porta. Natoque. Elit vivamus turpis eget? Potenti ridiculus metus id erat.,,,,152284133,,,,,,,,,,,,,Encounter,1,1 32 | 2018-11-15 21:39:49,2018-11-15 21:39:49,2,,100,0,1,2018-11-15 21:39:49,R_9BJ2cOkkoeV6Tlz,,,,,47.6605987548828125,-122.291900634765625,test,,"Porta! Sem in sed. Montes non, ligula. Venenatis cras cras.",6,2,,3,,"Blandit curabitur cras sodales, atque porttitor eu facilisi viverra elementum dolor sagittis. Rutrum. Phasellus. Sodales.",,,142157358,,,,,,,,,,,,,Encounter,1,1 33 | 2018-11-15 21:39:49,2018-11-15 21:39:49,2,,100,0,1,2018-11-15 21:39:49,R_b30tGIt27aCvzpP,,,,,47.6605987548828125,-122.291900634765625,test,,Faucibus? Porttitor metus mauris dignissim vivamus. Ac consequat cursus magnis nonummy.,7,4,"Elementum! Pretium viverra nibh interdum, sodales dignissim faucibus? Enim erat duis nulla.",6,Potenti? Phasellus et faucibus? Accumsan mi interdum consectetuer pellentesque. Malesuada.,,,,535738544,,,,,,,,,,,,,Encounter,1,1 34 | 2018-11-15 21:39:49,2018-11-15 21:39:49,2,,100,0,1,2018-11-15 21:39:49,R_39o74oBEMzmRK2F,,,,,47.6605987548828125,-122.291900634765625,test,,Vehicula? Metus. Convallis pharetra diam platea dapibus elementum arcu odio.,4,2,,3,,Phasellus ipsum primis? Phasellus urna interdum ipsum! Montes pharetra iaculis rutrum vivamus. Accumsan.,,,85732259,,,,,,,,,,,,,Encounter,1,1 35 | 2018-11-15 21:39:49,2018-11-15 21:39:49,2,,100,0,1,2018-11-15 21:39:49,R_bwOFjCboTt0WEjH,,,,,47.6605987548828125,-122.291900634765625,test,,Suscipit ultricies euismod! Dui quam volutpat iaculis risus accusamus faucibus cras imperdiet rutrum tincidunt curabitur.,8,4,Accumsan cras primis. Consequat. Ridiculus turpis natoque leo consectetuer a.,2,,Aliquam nulla erat tincidunt enim culpa ultrices diam dictumst. Curabitur a.,,,525990495,,,,,,,,,,,,,Encounter,1,1 36 | 2018-11-15 21:39:49,2018-11-15 21:39:49,2,,100,0,1,2018-11-15 21:39:49,R_00pbJJkTM5cKJwh,,,,,47.6605987548828125,-122.291900634765625,test,,Nibh potenti aliquam euismod. Enim aliquet sollicitudin nulla ante eu.,7,5,,3,,Sodales potenti erat pede eu metus cras temporibus vitae aliquet? Ac potenti.,,,129043473,,,,,,,,,,,,,Encounter,1,1 37 | 2018-11-15 21:39:50,2018-11-15 21:39:50,2,,100,0,1,2018-11-15 21:39:50,R_6QIHKrcr8fUdBid,,,,,47.6605987548828125,-122.291900634765625,test,,Dignissim platea posuere lorem magnis fringilla platea iaculis fringilla ante dictumst a lacus dolor lorem.,2,3,,6,Phasellus? Sit. Et curabitur tellus per dui phasellus morbi eget facilisis.,,,,47108258,,,,,,,,,,,,,Encounter,1,1 38 | 2018-11-15 21:39:50,2018-11-15 21:39:50,2,,100,0,1,2018-11-15 21:39:50,R_6xlfHU7S2D0jj5H,,,,,47.6605987548828125,-122.291900634765625,test,,Facilisis. Facilisi ipsum dolorem sapien neque felis platea fringilla. Egestas curabitur.,2,2,,5,,Ac felis potenti lacus temporibus vestibulum vel nec potenti nibh tellus praesent.,,,367450706,,,,,,,,,,,,,Encounter,1,1 39 | 2018-11-15 21:39:50,2018-11-15 21:39:50,2,,100,0,1,2018-11-15 21:39:50,R_cxc5Vez2broSotv,,,,,47.6605987548828125,-122.291900634765625,test,,Purus potenti. Potenti montes turpis mi elementum viverra urna leo porta natoque fusce duis accumsan.,7,1,,1,,"Ullamcorper nibh, accusamus! Ipsum dolorem suspendisse vehicula? Vulputate sed placerat tempus lectus vulputate. Blandit.",,,20012461,,,,,,,,,,,,,Encounter,1,1 40 | 2018-11-15 21:39:50,2018-11-15 21:39:50,2,,100,0,1,2018-11-15 21:39:50,R_9ZvOw5qtNhtTaMR,,,,,47.6605987548828125,-122.291900634765625,test,,"Bibendum magnis vestibulum platea, ipsum dictumst amet egestas. Posuere maecenas atque dapibus? Malesuada! Dignissim dictumst.",2,1,,2,,Vulputate porttitor tempus volutpat sapien sem. Rutrum velit sapien volutpat natoque auctor est.,,,388450130,,,,,,,,,,,,,Encounter,1,1 41 | 2018-11-15 21:39:50,2018-11-15 21:39:50,2,,100,0,1,2018-11-15 21:39:50,R_bg9nI0hohTQE21f,,,,,47.6605987548828125,-122.291900634765625,test,,"Sed lectus vel dolor integer fringilla volutpat nullam rutrum metus fusce, vel nunc.",9,5,,6,"Faucibus? Nunc dolorem facilisi iaculis tellus, placerat pede, eros eu, massa? Diam.",,,,994984071,,,,,,,,,,,,,Encounter,1,1 42 | 2018-11-15 21:39:50,2018-11-15 21:39:50,2,,100,0,1,2018-11-15 21:39:50,R_cSJjqLXk9XjmMDz,,,,,47.6605987548828125,-122.291900634765625,test,,"Enim. Integer sagittis mi tempora montes et, sem! Arcu. Natoque eleifend.",8,3,,6,Primis risus sem vulputate iaculis neque commodo placerat tempus at mattis cras.,,,,618409926,,,,,,,,,,,,,Encounter,1,1 43 | 2018-11-15 21:39:50,2018-11-15 21:39:50,2,,100,0,1,2018-11-15 21:39:50,R_7O0BAhv51tkAkcZ,,,,,47.6605987548828125,-122.291900634765625,test,,Proin temporibus. Dictumst erat ullamcorper! Sed vel aliquet nibh! Eu aenean.,1,4,Blandit risus lacus consequat sagittis! Wisi dolor lacus phasellus donec.,4,,"Bibendum fringilla tempora elit eleifend cras blandit felis consequat auctor, culpa in.",,,159263034,,,,,,,,,,,,,Encounter,1,1 44 | 2018-11-15 21:39:50,2018-11-15 21:39:50,2,,100,0,1,2018-11-15 21:39:51,R_0JoXZGoongJDlRP,,,,,47.6605987548828125,-122.291900634765625,test,,Cursus pretium lectus faucibus posuere venenatis iaculis eu imperdiet rhoncus ultricies porta odio.,10,4,"Mauris. Tempora pretium sollicitudin, sed tortor bibendum, est nibh temporibus. Dictumst vehicula amet.",3,,"Proin aliquet. Nulla! Pede orci, dolor sagittis. Etiam et. Integer. Ab.",,,4775038,,,,,,,,,,,,,Encounter,1,1 45 | 2018-11-15 21:39:51,2018-11-15 21:39:51,2,,100,0,1,2018-11-15 21:39:51,R_e4FBRycAxsG8vNX,,,,,47.6605987548828125,-122.291900634765625,test,,"Purus eu, proin mauris magnis praesent. Accusamus ipsum eget aenean.",2,2,,1,,"Suspendisse. Vel. Pede dictumst, arcu mi ullamcorper in. Vel amet, magnis sodales blandit pede.",,,598930349,,,,,,,,,,,,,Encounter,1,1 46 | 2018-11-15 21:39:51,2018-11-15 21:39:51,2,,100,0,1,2018-11-15 21:39:51,R_b3FspdPD3niApwN,,,,,47.6605987548828125,-122.291900634765625,test,,Tempora magnis purus lectus? Accumsan augue ullamcorper morbi mi morbi sed pellentesque platea.,8,2,,3,,Vehicula tempor convallis neque ultrices justo nulla per lorem vel.,,,212726813,,,,,,,,,,,,,Encounter,1,1 47 | 2018-11-15 21:39:51,2018-11-15 21:39:51,2,,100,0,1,2018-11-15 21:39:51,R_0cAVyh2sq7yYSNv,,,,,47.6605987548828125,-122.291900634765625,test,,Eget curabitur facilisis aliquam a? Rutrum atque duis nunc gravida integer dolor pede! Gravida.,1,3,,3,,Nunc interdum ultricies. Leo pharetra lacus accumsan vulputate! Pharetra id lectus.,,,206964375,,,,,,,,,,,,,Encounter,1,1 48 | 2018-11-15 21:39:51,2018-11-15 21:39:51,2,,100,0,1,2018-11-15 21:39:51,R_3fQ47YCvKFYo9vL,,,,,47.6605987548828125,-122.291900634765625,test,,Culpa blandit egestas sollicitudin pede mattis egestas augue. Montes. Malesuada. Sodales montes bibendum mattis ante.,8,5,,1,,"Imperdiet dolorem eros. Magna justo faucibus tempora placerat est, arcu mattis accusamus. Curabitur montes.",,,367753185,,,,,,,,,,,,,Encounter,1,1 49 | 2018-11-15 21:39:51,2018-11-15 21:39:51,2,,100,0,1,2018-11-15 21:39:51,R_dgQHEMjcxe5XDFP,,,,,47.6605987548828125,-122.291900634765625,test,,Ultricies. Potenti purus tempus enim ab ut suscipit eu tempor lectus. Aliquet? Per.,1,3,,1,,Per. Egestas? Facilisis vitae felis suspendisse ultrices gravida sit tempora suscipit leo maecenas nibh.,,,779602520,,,,,,,,,,,,,Encounter,1,1 50 | 2018-11-15 21:39:51,2018-11-15 21:39:51,2,,100,0,1,2018-11-15 21:39:51,R_eW00fYiEbXmpZnT,,,,,47.6605987548828125,-122.291900634765625,test,,"Tincidunt vestibulum. Ligula ridiculus tempora! Auctor arcu morbi, sollicitudin mauris.",7,1,,3,,Vulputate? Erat tempor ligula. Dui fusce tortor sapien. Consectetuer? Tempor tempora accusamus lectus ante mauris.,,,578783125,,,,,,,,,,,,,Encounter,1,1 51 | 2018-11-15 21:39:51,2018-11-15 21:39:51,2,,100,0,1,2018-11-15 21:39:51,R_1zvEv3jMFIrBRn7,,,,,47.6605987548828125,-122.291900634765625,test,,"Nulla sagittis, praesent sollicitudin duis. Imperdiet placerat dictumst! Vulputate turpis.",10,4,Leo auctor felis sed risus. Vehicula culpa nunc magna. Metus ullamcorper.,6,"Arcu consequat. Risus, rutrum nullam dictumst turpis pede? Eget fusce, leo! Wisi bibendum.",,,,919489348,,,,,,,,,,,,,Encounter,1,1 52 | 2018-11-15 21:39:51,2018-11-15 21:39:51,2,,100,0,1,2018-11-15 21:39:51,R_aVFlLsEhnIBMXhr,,,,,47.6605987548828125,-122.291900634765625,test,,"Suspendisse. At montes! Urna turpis vulputate, auctor ridiculus enim primis eleifend. Consectetuer tortor, nec euismod.",6,2,,3,,Tempora vivamus! Elementum. At vitae scelerisque felis elementum vivamus egestas.,,,446306870,,,,,,,,,,,,,Encounter,1,1 53 | 2018-11-15 21:39:51,2018-11-15 21:39:51,2,,100,0,1,2018-11-15 21:39:52,R_6qZq1b6VXD9GR13,,,,,47.6605987548828125,-122.291900634765625,test,,"Dapibus aliquet scelerisque pharetra id sed posuere nulla accumsan per. Suscipit et in mauris, facilisis.",8,1,,4,,Pharetra diam. Tellus accusamus! Risus pretium. Odio facilisis ante. Ligula aliquam.,,,230632438,,,,,,,,,,,,,Encounter,1,1 54 | 2018-11-15 21:39:52,2018-11-15 21:39:52,2,,100,0,1,2018-11-15 21:39:52,R_8BORzMBlFJYPzZr,,,,,47.6605987548828125,-122.291900634765625,test,,Et posuere culpa dolorem. Tincidunt culpa. Massa pretium neque tellus odio potenti luctus.,3,4,"Nec integer? Natoque at quis! Eleifend tempora! Per, in tempus! Viverra mi consequat.",6,"Volutpat, dui, quam proin ac ac dui odio nunc euismod sed.",,,,458018844,,,,,,,,,,,,,Encounter,1,1 55 | 2018-11-15 21:39:52,2018-11-15 21:39:52,2,,100,0,1,2018-11-15 21:39:52,R_4VDGxBJqRY73wS9,,,,,47.6605987548828125,-122.291900634765625,test,,Id vel montes tellus. Purus? Convallis? Magnis sit dapibus laoreet temporibus.,8,3,,2,,"Odio erat. Imperdiet. Primis neque? Malesuada, wisi bibendum ac placerat porta. Donec.",,,455739155,,,,,,,,,,,,,Encounter,1,1 56 | 2018-11-15 21:39:52,2018-11-15 21:39:52,2,,100,0,1,2018-11-15 21:39:52,R_e2nrR83KCD81f5H,,,,,47.6605987548828125,-122.291900634765625,test,,Praesent culpa? Atque elit eu pede elit accumsan eu? Orci dui.,10,4,Augue culpa. Eu arcu! Magnis? Lorem integer sed suspendisse dictumst et.,4,,"Nunc? Augue donec faucibus euismod, quis laoreet cursus dolorem? Ultrices sed.",,,439059805,,,,,,,,,,,,,Encounter,1,1 57 | 2018-11-15 21:39:52,2018-11-15 21:39:52,2,,100,0,1,2018-11-15 21:39:52,R_cNk7y4j4YZlyIux,,,,,47.6605987548828125,-122.291900634765625,test,,Metus est ultricies primis pede convallis. Primis. Vulputate. Vivamus tempora quam.,0,4,"Aliquam volutpat erat? Est luctus cursus luctus proin, elit accumsan commodo praesent eu.",6,Sapien sem vestibulum. Magnis ab proin tempora. Massa posuere fringilla massa ultrices. Laoreet.,,,,411391648,,,,,,,,,,,,,Encounter,1,1 58 | 2018-11-15 21:39:52,2018-11-15 21:39:52,2,,100,0,1,2018-11-15 21:39:52,R_efEBywhppQKWuAR,,,,,47.6605987548828125,-122.291900634765625,test,,Turpis posuere suscipit facilisis porttitor venenatis fusce atque wisi suscipit? Culpa.,4,1,,5,,"Facilisis accusamus gravida. Molestie sed proin odio dolor ab justo, euismod.",,,713037975,,,,,,,,,,,,,Encounter,1,1 59 | 2018-11-15 21:39:52,2018-11-15 21:39:52,2,,100,0,1,2018-11-15 21:39:52,R_6mOa9pyy6cmNMxv,,,,,47.6605987548828125,-122.291900634765625,test,,Ridiculus suspendisse elit odio sapien tempor cursus. Mauris dolor phasellus quis.,7,4,Tempus? Metus a temporibus convallis vehicula! Egestas sodales aliquet accusamus.,2,,"Tempus, suspendisse sed integer. Pharetra at leo? Luctus? Sit venenatis dolor phasellus tincidunt.",,,83879850,,,,,,,,,,,,,Encounter,1,1 60 | 2018-11-15 21:39:52,2018-11-15 21:39:52,2,,100,0,1,2018-11-15 21:39:52,R_8tVrxmajOhzjA2N,,,,,47.6605987548828125,-122.291900634765625,test,,Sapien vestibulum molestie integer tempora neque. Fermentum pellentesque justo nonummy.,5,1,,5,,Suspendisse tempora tempor. Faucibus culpa nonummy ipsum diam ultricies montes fusce quam. Odio amet.,,,249255135,,,,,,,,,,,,,Encounter,1,1 61 | 2018-11-15 21:39:52,2018-11-15 21:39:52,2,,100,0,1,2018-11-15 21:39:53,R_265ahxjMybNDHFz,,,,,47.6605987548828125,-122.291900634765625,test,,Pede eros rhoncus et facilisis lacus suspendisse quis accusamus! Porta ultrices natoque.,5,3,,2,,"Velit? Dapibus nibh at lacus dictumst platea natoque ligula quis, augue turpis quam.",,,562196121,,,,,,,,,,,,,Encounter,1,1 62 | 2018-11-15 21:39:53,2018-11-15 21:39:53,2,,100,0,1,2018-11-15 21:39:53,R_6EvYJ8EeZJb2eKp,,,,,47.6605987548828125,-122.291900634765625,test,,"Tempor, lacus vitae euismod dolorem fermentum tincidunt tellus dignissim wisi.",3,1,,5,,Laoreet euismod suspendisse auctor facilisis a etiam! Vestibulum vulputate iaculis laoreet.,,,179507281,,,,,,,,,,,,,Encounter,1,1 63 | 2018-11-15 21:39:53,2018-11-15 21:39:53,2,,100,0,1,2018-11-15 21:39:53,R_d5N1KKYIEdFRWDj,,,,,47.6605987548828125,-122.291900634765625,test,,Malesuada montes wisi imperdiet accusamus mi ullamcorper vehicula. Morbi curabitur.,5,5,,1,,"Vulputate egestas ante. Turpis gravida elit! Maecenas lorem dolorem! Metus urna metus, suspendisse! Vitae.",,,120564999,,,,,,,,,,,,,Encounter,1,1 64 | 2018-11-15 21:39:53,2018-11-15 21:39:53,2,,100,0,1,2018-11-15 21:39:53,R_8rcbluHdBJ6b06p,,,,,47.6605987548828125,-122.291900634765625,test,,Turpis dictumst. Vel tempus in dapibus nulla tincidunt iaculis nibh nonummy. Quam! Nunc culpa tempora.,6,3,,3,,"Eget phasellus vulputate per cras fringilla eros, orci suspendisse dui montes eleifend.",,,663661253,,,,,,,,,,,,,Encounter,1,1 65 | 2018-11-15 21:39:53,2018-11-15 21:39:53,2,,100,0,1,2018-11-15 21:39:53,R_0j6KqgWlWwL78k5,,,,,47.6605987548828125,-122.291900634765625,test,,Lacus. Mattis lorem sem faucibus commodo? Proin convallis rutrum dui! Cras.,6,3,,3,,"Ridiculus, tortor atque suscipit. Massa tempor montes, fusce sollicitudin arcu consequat. Ipsum porta.",,,734804136,,,,,,,,,,,,,Encounter,1,1 66 | 2018-11-15 21:39:53,2018-11-15 21:39:53,2,,100,0,1,2018-11-15 21:39:53,R_a461stlcmYJe39r,,,,,47.6605987548828125,-122.291900634765625,test,,Dolor tellus dictumst enim consequat dignissim! Commodo metus volutpat magnis orci luctus fermentum cras.,6,3,,5,,Molestie rhoncus curabitur suscipit blandit bibendum quis. Cras odio. Ut lorem.,,,883909760,,,,,,,,,,,,,Encounter,1,1 67 | 2018-11-15 21:39:53,2018-11-15 21:39:53,2,,100,0,1,2018-11-15 21:39:53,R_4IWhbAWa9C60ti5,,,,,47.6605987548828125,-122.291900634765625,test,,Pharetra accumsan luctus? Justo natoque rhoncus risus molestie erat euismod dignissim facilisi.,9,3,,4,,Consequat duis et interdum sollicitudin accumsan neque sagittis leo fusce commodo lorem fusce accumsan ultrices.,,,46678572,,,,,,,,,,,,,Encounter,1,1 68 | 2018-11-15 21:39:53,2018-11-15 21:39:53,2,,100,0,1,2018-11-15 21:39:53,R_5jVQt9CmcTnlpGZ,,,,,47.6605987548828125,-122.291900634765625,test,,Eleifend ullamcorper. Nec consequat laoreet etiam! Euismod bibendum nec sollicitudin vehicula imperdiet.,3,3,,1,,"Neque dignissim fermentum est aenean. Eget, ac mattis faucibus, malesuada.",,,471481366,,,,,,,,,,,,,Encounter,1,1 69 | 2018-11-15 21:39:53,2018-11-15 21:39:54,2,,100,1,1,2018-11-15 21:39:54,R_cAPhAMYAtqOQFLL,,,,,47.6605987548828125,-122.291900634765625,test,,Amet ultrices! Curabitur duis risus curabitur tempora! Vel? Porta montes curabitur porttitor in laoreet.,5,4,"Morbi tincidunt dolorem, vestibulum eu vivamus ante, sed magna maecenas. Blandit primis morbi nullam etiam.",1,,Neque gravida est velit augue metus duis amet tellus. Gravida.,,,292811440,,,,,,,,,,,,,Encounter,1,1 70 | 2018-11-15 21:39:54,2018-11-15 21:39:54,2,,100,0,1,2018-11-15 21:39:54,R_cTqLJsP7E84TuOp,,,,,47.6605987548828125,-122.291900634765625,test,,"Volutpat tempora vulputate tincidunt praesent, porta bibendum quis ultricies ante.",8,4,Lorem. Magnis? Arcu viverra viverra ultricies consectetuer integer massa pellentesque faucibus.,1,,Maecenas laoreet venenatis? Massa faucibus pharetra enim gravida. Magna facilisi eu. Wisi! Dapibus wisi tortor.,,,366601101,,,,,,,,,,,,,Encounter,1,1 71 | 2018-11-15 21:39:54,2018-11-15 21:39:54,2,,100,0,1,2018-11-15 21:39:54,R_cPlVXbCjxE3YAFD,,,,,47.6605987548828125,-122.291900634765625,test,,"Purus mi phasellus ullamcorper. Sollicitudin auctor etiam, blandit tempus! Aliquet id.",4,4,"Sed cursus, malesuada? Pharetra dolor vivamus lorem per. Ab lorem diam nonummy lorem.",6,Tempor natoque in! Rutrum. Ut arcu! Dapibus tempus enim. Viverra.,,,,486465942,,,,,,,,,,,,,Encounter,1,1 72 | 2018-11-15 21:39:54,2018-11-15 21:39:54,2,,100,0,1,2018-11-15 21:39:54,R_0keC1TyqCRqQHK5,,,,,47.6605987548828125,-122.291900634765625,test,,"Imperdiet. Morbi id odio atque. Montes! Atque pellentesque? Scelerisque, mattis consequat amet metus vestibulum! Morbi.",4,4,Morbi curabitur pretium nibh sed sem augue. Vulputate magnis? Convallis! Vivamus.,4,,Et sem. At duis faucibus. Quis. Viverra eros accusamus pede interdum mi! Quam.,,,860140055,,,,,,,,,,,,,Encounter,1,1 73 | 2018-11-15 21:39:54,2018-11-15 21:39:54,2,,100,0,1,2018-11-15 21:39:54,R_a5cKoARuIZTXWwl,,,,,47.6605987548828125,-122.291900634765625,test,,"Diam tortor sodales, consequat morbi turpis justo vehicula nullam ipsum facilisi urna.",5,3,,5,,Accumsan eu malesuada fusce at tempus eget molestie tincidunt suscipit. Vulputate.,,,713683390,,,,,,,,,,,,,Encounter,1,1 74 | 2018-11-15 21:39:54,2018-11-15 21:39:54,2,,100,0,1,2018-11-15 21:39:54,R_0rlcuf59RZ3UVhz,,,,,47.6605987548828125,-122.291900634765625,test,,"Pharetra, ridiculus? Duis faucibus praesent vivamus? Ligula volutpat fusce arcu.",10,1,,1,,"Neque! Placerat ullamcorper atque erat, ab purus ipsum? Per laoreet.",,,985178732,,,,,,,,,,,,,Encounter,1,1 75 | 2018-11-15 21:39:54,2018-11-15 21:39:54,2,,100,0,1,2018-11-15 21:39:54,R_1zBNYSa7FEoENZH,,,,,47.6605987548828125,-122.291900634765625,test,,Dolor sed sodales porttitor vel molestie? Lacus leo primis fringilla est venenatis eu elit mauris.,0,4,Suspendisse maecenas facilisi arcu pede? Eleifend magna platea imperdiet dapibus gravida.,1,,Duis a facilisi ut massa pede. Quam convallis quis nibh.,,,260164434,,,,,,,,,,,,,Encounter,1,1 76 | 2018-11-15 21:39:54,2018-11-15 21:39:54,2,,100,0,1,2018-11-15 21:39:54,R_cZNe3Uzm4lcGU8l,,,,,47.6605987548828125,-122.291900634765625,test,,A fringilla porta suspendisse. Justo nec ac nonummy fermentum consequat faucibus donec.,8,2,,4,,"Dictumst aenean leo. Placerat per, convallis id! Gravida. Interdum gravida! Aenean. Viverra ullamcorper.",,,49956054,,,,,,,,,,,,,Encounter,1,1 77 | 2018-11-15 21:39:54,2018-11-15 21:39:54,2,,100,0,1,2018-11-15 21:39:54,R_cVm3XmOtM738HqJ,,,,,47.6605987548828125,-122.291900634765625,test,,"Fusce vitae? Dolor in dignissim wisi accusamus arcu mattis, nec? Viverra tortor! Commodo consequat primis.",1,2,,2,,"Egestas gravida est ab metus metus, at nullam vehicula? Etiam erat interdum odio integer vehicula.",,,739018057,,,,,,,,,,,,,Encounter,1,1 78 | 2018-11-15 21:39:55,2018-11-15 21:39:55,2,,100,0,1,2018-11-15 21:39:55,R_d1oL79FeBmFPJqZ,,,,,47.6605987548828125,-122.291900634765625,test,,Dolorem egestas lacus? Malesuada dictumst dictumst facilisi tempora scelerisque! Platea interdum.,0,3,,6,Potenti non suscipit sapien pretium morbi! Fringilla fringilla accusamus! Atque! Imperdiet.,,,,232342597,,,,,,,,,,,,,Encounter,1,1 79 | 2018-11-15 21:39:55,2018-11-15 21:39:55,2,,100,0,1,2018-11-15 21:39:55,R_dgO2rZFKiOwHTAV,,,,,47.6605987548828125,-122.291900634765625,test,,Egestas ullamcorper maecenas egestas. Pede nec malesuada ligula. Aliquet sed.,10,4,Eget mauris consectetuer. Ante cras temporibus in ut atque dolor bibendum nunc purus ridiculus wisi.,3,,Dictumst? Nulla facilisis consectetuer? Volutpat mi quis? Malesuada ac suscipit.,,,931823964,,,,,,,,,,,,,Encounter,1,1 80 | 2018-11-15 21:39:55,2018-11-15 21:39:55,2,,100,0,1,2018-11-15 21:39:55,R_0HEJkFopBGhOL89,,,,,47.6605987548828125,-122.291900634765625,test,,Ultrices sit viverra et enim et integer. Orci volutpat temporibus ultricies quis.,8,3,,1,,Tempus? Donec arcu eget fusce. Curabitur dapibus lacus curabitur interdum.,,,802491445,,,,,,,,,,,,,Encounter,1,1 81 | 2018-11-15 21:39:55,2018-11-15 21:39:55,2,,100,0,1,2018-11-15 21:39:55,R_erp4C0x1ODVCH2d,,,,,47.6605987548828125,-122.291900634765625,test,,Sapien magna ullamcorper et? Ullamcorper mauris dictumst porta velit dolor.,8,5,,3,,"Sapien tincidunt, rhoncus at, mi mattis commodo curabitur! Praesent placerat.",,,274265795,,,,,,,,,,,,,Encounter,1,1 82 | 2018-11-15 21:39:55,2018-11-15 21:39:55,2,,100,0,1,2018-11-15 21:39:55,R_1XoTbEuo6eJo7nT,,,,,47.6605987548828125,-122.291900634765625,test,,"Mauris, tempor metus porttitor tincidunt dictumst? Risus donec atque sed culpa! Lacus? Magnis, cursus.",4,2,,1,,Mauris justo non lorem tempus accusamus urna ligula facilisi dapibus. Atque et felis pharetra.,,,7252971,,,,,,,,,,,,,Encounter,1,1 83 | 2018-11-15 21:39:55,2018-11-15 21:39:55,2,,100,0,1,2018-11-15 21:39:55,R_6fDFedI4QLdrqOp,,,,,47.6605987548828125,-122.291900634765625,test,,"Suscipit facilisi ab turpis augue faucibus mauris enim nulla pharetra laoreet, magna dui cursus ullamcorper.",4,4,Fermentum sagittis ante gravida nonummy iaculis aenean amet? Fringilla? Nulla.,4,,"Lacus commodo fringilla velit? Elementum facilisi vehicula tellus pretium, cursus. Laoreet! Commodo placerat donec.",,,148575762,,,,,,,,,,,,,Encounter,1,1 84 | 2018-11-15 21:39:55,2018-11-15 21:39:55,2,,100,0,1,2018-11-15 21:39:55,R_8nOC27P4PY8ChVj,,,,,47.6605987548828125,-122.291900634765625,test,,"Dolor potenti praesent nonummy, velit viverra vehicula accusamus interdum a dictumst dui non magna.",5,5,,2,,Primis ultrices massa ipsum suscipit duis sapien ut viverra vestibulum.,,,99931127,,,,,,,,,,,,,Encounter,1,1 85 | 2018-11-15 21:39:55,2018-11-15 21:39:55,2,,100,0,1,2018-11-15 21:39:56,R_ei0TKzQP4teLcy1,,,,,47.6605987548828125,-122.291900634765625,test,,"Lectus atque quis tempus. Vel, mattis a elementum nec mattis. Metus tempora venenatis.",8,2,,6,Consectetuer accumsan. Mauris! Atque. Scelerisque. Mattis malesuada viverra. Diam. Diam pretium.,,,,239480628,,,,,,,,,,,,,Encounter,1,1 86 | 2018-11-15 21:39:56,2018-11-15 21:39:56,2,,100,0,1,2018-11-15 21:39:56,R_6KJ0lBdP0f1EsfP,,,,,47.6605987548828125,-122.291900634765625,test,,Sodales urna orci primis dignissim erat. Consectetuer. Mi phasellus tortor.,1,5,,3,,Primis nonummy sodales a consequat. Pharetra laoreet venenatis. Suspendisse nulla elit suscipit proin.,,,987133940,,,,,,,,,,,,,Encounter,1,1 87 | 2018-11-15 21:39:56,2018-11-15 21:39:56,2,,100,0,1,2018-11-15 21:39:56,R_77BOrJX2sirwatn,,,,,47.6605987548828125,-122.291900634765625,test,,"Viverra non accumsan, sit lectus vestibulum ridiculus eros! Risus. Nibh purus! Magna volutpat porttitor arcu.",1,4,"Lorem felis sed imperdiet scelerisque fringilla, at magna praesent! Pellentesque placerat facilisis volutpat culpa convallis.",3,,"Iaculis nullam? Fringilla nibh sem faucibus sed, non magnis! Tempor! Et.",,,510032126,,,,,,,,,,,,,Encounter,1,1 88 | 2018-11-15 21:39:56,2018-11-15 21:39:56,2,,100,0,1,2018-11-15 21:39:56,R_cx29seoVvCRhHA9,,,,,47.6605987548828125,-122.291900634765625,test,,Dolor egestas interdum leo mi tempora? Natoque ridiculus iaculis eu.,6,1,,1,,Nulla sem pharetra laoreet commodo nonummy ridiculus mattis aenean metus elementum viverra cras culpa.,,,263693775,,,,,,,,,,,,,Encounter,1,1 89 | 2018-11-15 21:39:56,2018-11-15 21:39:56,2,,100,0,1,2018-11-15 21:39:56,R_9SJSxTaIgJfRwLH,,,,,47.6605987548828125,-122.291900634765625,test,,Quis. Risus interdum nibh arcu nibh orci platea. Culpa. Consequat suscipit eleifend pede.,1,5,,4,,"Sed commodo gravida elit tempus, aliquet natoque suspendisse non auctor.",,,175000735,,,,,,,,,,,,,Encounter,1,1 90 | 2018-11-15 21:39:56,2018-11-15 21:39:56,2,,100,0,1,2018-11-15 21:39:56,R_1A1piIRDKLLvvJX,,,,,47.6605987548828125,-122.291900634765625,test,,Pellentesque commodo viverra! Et viverra! Rutrum neque ultrices est accumsan vitae mauris.,2,1,,4,,Porttitor venenatis sollicitudin ac! Phasellus culpa. Pede massa ridiculus integer! Nullam. Nunc ullamcorper elementum donec.,,,962341685,,,,,,,,,,,,,Encounter,1,1 91 | 2018-11-15 21:39:56,2018-11-15 21:39:56,2,,100,0,1,2018-11-15 21:39:56,R_6xnWcMIMqD17N1X,,,,,47.6605987548828125,-122.291900634765625,test,,Velit. Posuere? Mattis elit ipsum! Blandit atque facilisi. Tempora donec.,1,4,Scelerisque mattis. Odio lorem. Velit nullam gravida pellentesque viverra velit quam nunc? Sem dolorem.,5,,Pede sapien arcu id ultricies sapien. Suscipit phasellus id aenean bibendum ullamcorper turpis est.,,,230989136,,,,,,,,,,,,,Encounter,1,1 92 | 2018-11-15 21:39:56,2018-11-15 21:39:56,2,,100,0,1,2018-11-15 21:39:56,R_7QmGbXGN6BVde85,,,,,47.6605987548828125,-122.291900634765625,test,,Fermentum turpis mi tellus dignissim montes dolorem justo laoreet wisi id.,10,4,"Consequat, quam convallis wisi, lacus erat rutrum urna pede. Mauris lorem mauris mattis.",3,,Dui ultrices? Eros primis ultrices? Et. Lorem! Dignissim gravida magna vivamus imperdiet.,,,523168013,,,,,,,,,,,,,Encounter,1,1 93 | 2018-11-15 21:39:56,2018-11-15 21:39:56,2,,100,0,1,2018-11-15 21:39:56,R_9YTLLwDwlTySwqp,,,,,47.6605987548828125,-122.291900634765625,test,,Accusamus mattis a quam amet neque! Tincidunt? Culpa sagittis odio. Turpis tellus ullamcorper.,6,4,"Auctor potenti nunc ut, vehicula, pretium etiam nibh scelerisque velit scelerisque sed! Nibh.",3,,Donec. Wisi porttitor lectus. Quis integer. Ac interdum aliquet molestie. Erat purus pharetra.,,,373721306,,,,,,,,,,,,,Encounter,1,1 94 | 2018-11-15 21:39:56,2018-11-15 21:39:56,2,,100,0,1,2018-11-15 21:39:57,R_7TEiDjeZtlWzGXr,,,,,47.6605987548828125,-122.291900634765625,test,,Lectus. Bibendum quam vestibulum. Pretium mauris viverra quam platea morbi etiam? Duis tempor.,2,3,,6,Malesuada. Nunc consequat at laoreet molestie imperdiet. Euismod orci velit vulputate ridiculus orci.,,,,948653967,,,,,,,,,,,,,Encounter,1,1 95 | 2018-11-15 21:39:57,2018-11-15 21:39:57,2,,100,0,1,2018-11-15 21:39:57,R_1ZgGhsi6lXwVMVf,,,,,47.6605987548828125,-122.291900634765625,test,,Ultrices sollicitudin a lacus vitae velit laoreet leo dignissim suspendisse eget quam molestie.,9,3,,5,,Elit felis eros sit vivamus nibh magna! Est imperdiet curabitur.,,,839081111,,,,,,,,,,,,,Encounter,1,1 96 | 2018-11-15 21:39:57,2018-11-15 21:39:57,2,,100,0,1,2018-11-15 21:39:57,R_5sFvzEOu7D2u4Dz,,,,,47.6605987548828125,-122.291900634765625,test,,Orci gravida! Dolor vel platea leo vitae rutrum pharetra culpa iaculis.,9,5,,3,,"Facilisi ullamcorper tellus laoreet justo porttitor. Lacus, nulla justo justo eros dolor! Non fermentum. Vitae.",,,60681413,,,,,,,,,,,,,Encounter,1,1 97 | 2018-11-15 21:39:57,2018-11-15 21:39:57,2,,100,0,1,2018-11-15 21:39:57,R_1N4cH2oYmSutMSF,,,,,47.6605987548828125,-122.291900634765625,test,,"Est nec facilisis quis, non cursus vitae montes sodales pretium! Arcu.",0,3,,4,,Pede aliquet auctor! Diam cras maecenas integer tincidunt tellus magna laoreet tempor venenatis diam.,,,906519105,,,,,,,,,,,,,Encounter,1,1 98 | 2018-11-15 21:39:57,2018-11-15 21:39:57,2,,100,0,1,2018-11-15 21:39:57,R_1YT5eQ7CiFV5c33,,,,,47.6605987548828125,-122.291900634765625,test,,Euismod dui augue ridiculus gravida tempor phasellus interdum? Purus sit. Augue iaculis mi vel duis.,7,1,,4,,In luctus? Eleifend facilisi diam mauris tempus diam sagittis! Tellus fringilla nonummy ligula blandit. Ultrices.,,,945759361,,,,,,,,,,,,,Encounter,1,1 99 | 2018-11-15 21:39:57,2018-11-15 21:39:57,2,,100,0,1,2018-11-15 21:39:57,R_6LtVsRv3M6NtqyF,,,,,47.6605987548828125,-122.291900634765625,test,,Eget tincidunt? Tellus! Tempus lectus magna dolor nunc aliquam cras faucibus nulla gravida.,9,1,,1,,Mauris magna! Viverra porttitor pede scelerisque at commodo. Ultrices iaculis accusamus.,,,685366039,,,,,,,,,,,,,Encounter,1,1 100 | 2018-11-15 21:39:57,2018-11-15 21:39:57,2,,100,0,1,2018-11-15 21:39:57,R_bdfC8CYFWtmXTF3,,,,,47.6605987548828125,-122.291900634765625,test,,"Duis mattis urna mauris dui, magnis. Pellentesque suspendisse eget nulla.",7,5,,3,,Viverra et tempus diam eros cras massa tempora etiam rutrum in sed.,,,4289746,,,,,,,,,,,,,Encounter,1,1 101 | 2018-11-15 21:39:57,2018-11-15 21:39:57,2,,100,0,1,2018-11-15 21:39:57,R_24s8YfEEaUgCPU9,,,,,47.6605987548828125,-122.291900634765625,test,,"Atque justo tempora, ac elementum elementum cras lacus metus enim et dapibus non consequat.",1,4,"Nibh in, mi purus potenti tortor sit magnis imperdiet vel! Etiam at.",5,,Dolor. Facilisi aliquam molestie a magna lacus quam integer molestie etiam viverra natoque enim enim.,,,165790543,,,,,,,,,,,,,Encounter,1,1 102 | 2018-11-15 21:39:57,2018-11-15 21:39:57,2,,100,0,1,2018-11-15 21:39:58,R_6ET7zC0vh7Zxfwh,,,,,47.6605987548828125,-122.291900634765625,test,,"Molestie eleifend blandit. Curabitur massa porttitor dolorem. In vestibulum, vestibulum nunc non temporibus.",4,2,,6,Dolorem. Augue orci a commodo facilisis iaculis odio neque vivamus risus non tincidunt sollicitudin scelerisque.,,,,267790571,,,,,,,,,,,,,Encounter,1,1 103 | 2018-11-15 21:39:58,2018-11-15 21:39:58,2,,100,0,1,2018-11-15 21:39:58,R_cT3OGSICXKA3Dgh,,,,,47.6605987548828125,-122.291900634765625,test,,Dapibus auctor sit per. Facilisis eleifend! Tempus vehicula turpis. Dignissim a. Pede imperdiet consequat imperdiet.,0,2,,3,,Porttitor atque ab! Venenatis tortor tellus at fermentum odio. Pellentesque justo! Accumsan platea.,,,101205434,,,,,,,,,,,,,Encounter,1,1 104 | -------------------------------------------------------------------------------- /app/lambda/functions/surveyjobs/qualtrics.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Export variable: 4 | 5 | export X_API_TOKEN=xxxx 6 | 7 | Sync to Dynamo like this: 8 | 9 | ./qualtrics.py sync-db 10 | 11 | """ 12 | import base64 13 | import zipfile 14 | import io 15 | import os 16 | import sys 17 | import json 18 | import urllib.parse 19 | import time 20 | import dateutil.parser 21 | 22 | 23 | import boto3 24 | import botocore 25 | import requests 26 | import pandas as pd 27 | import click 28 | 29 | from hashlib import md5 30 | 31 | #SETUP LOGGING 32 | import logging 33 | from pythonjsonlogger import jsonlogger 34 | 35 | LOG = logging.getLogger() 36 | LOG.setLevel(logging.DEBUG) 37 | logHandler = logging.StreamHandler() 38 | formatter = jsonlogger.JsonFormatter() 39 | logHandler.setFormatter(formatter) 40 | LOG.addHandler(logHandler) 41 | 42 | #S3 BUCKET 43 | REGION = "us-east-1" 44 | DYNAMODB = boto3.resource('dynamodb') 45 | TABLE = None 46 | 47 | def setup_environment(): 48 | ### Qualtrics ### 49 | try: 50 | api_token = os.environ['X_API_TOKEN'] 51 | except KeyError: 52 | LOG.error("ERROR!: set environment variable X_API_TOKEN") 53 | sys.exit(2) 54 | LOG.info(f"Using 'X_API_TOKEN': {api_token}") 55 | 56 | ### DynamoDB 57 | try: 58 | AGENCIES_TABLE_ID = os.environ['AGENCIES_TABLE_ID'] 59 | table = DYNAMODB.Table(AGENCIES_TABLE_ID) 60 | except KeyError: 61 | LOG.error("ERROR!: set environment variable AGENCIES_TABLE_ID [comes from Jet Steps: i.e. policeDepartments-6a2557316621d95d]") 62 | sys.exit(2) 63 | 64 | LOG.info(f"Using 'AGENCIES_TABLE_ID': {AGENCIES_TABLE_ID}") 65 | LOG.info(f"Using DYNAMODB TABLE: {TABLE}") 66 | return api_token, table 67 | 68 | 69 | ### KMS Utils### 70 | 71 | def encrypt(secret, extra=None): 72 | if secret == "": 73 | LOG.info('Encrypting empty string', extra=extra) 74 | return "" 75 | client = boto3.client('kms') 76 | key_alias = os.environ.get('KMS_KEY') 77 | ciphertext = client.encrypt( 78 | KeyId=key_alias, 79 | Plaintext=secret, 80 | ) 81 | LOG.info(f'Encrypted value: {ciphertext}') 82 | return base64.b64encode(ciphertext['CiphertextBlob']) 83 | 84 | 85 | def decrypt(secret, extra=None): 86 | client = boto3.client('kms') 87 | LOG.info(f'Decrypting: {secret}', extra) 88 | plaintext = client.decrypt( 89 | CiphertextBlob=base64.b64decode(secret) 90 | ) 91 | return plaintext['Plaintext'] 92 | 93 | 94 | ### SQS Utils### 95 | 96 | def sqs_queue_resource(queue_name): 97 | """Returns an SQS queue resource connection 98 | 99 | Usage example: 100 | In [2]: queue = sqs_queue_resource("dev-job-24910") 101 | In [4]: queue.attributes 102 | Out[4]: 103 | {'ApproximateNumberOfMessages': '0', 104 | 'ApproximateNumberOfMessagesDelayed': '0', 105 | 'ApproximateNumberOfMessagesNotVisible': '0', 106 | 'CreatedTimestamp': '1476240132', 107 | 'DelaySeconds': '0', 108 | 'LastModifiedTimestamp': '1476240132', 109 | 'MaximumMessageSize': '262144', 110 | 'MessageRetentionPeriod': '345600', 111 | 'QueueArn': 'arn:aws:sqs:us-west-2:414930948375:dev-job-24910', 112 | 'ReceiveMessageWaitTimeSeconds': '0', 113 | 'VisibilityTimeout': '120'} 114 | 115 | """ 116 | 117 | sqs_resource = boto3.resource('sqs', region_name=REGION) 118 | log_sqs_resource_msg = "Creating SQS resource conn with qname: [%s] in region: [%s]" %\ 119 | (queue_name, REGION) 120 | LOG.info(log_sqs_resource_msg) 121 | queue = sqs_resource.get_queue_by_name(QueueName=queue_name) 122 | return queue 123 | 124 | def sqs_connection(): 125 | """Creates an SQS Connection which defaults to global var REGION""" 126 | 127 | sqs_client = boto3.client("sqs", region_name=REGION) 128 | log_sqs_client_msg = "Creating SQS connection in Region: [%s]" % REGION 129 | LOG.info(log_sqs_client_msg) 130 | return sqs_client 131 | 132 | def sqs_approximate_count(queue_name): 133 | """Return an approximate count of messages left in queue""" 134 | 135 | queue = sqs_queue_resource(queue_name) 136 | attr = queue.attributes 137 | num_message = int(attr['ApproximateNumberOfMessages']) 138 | num_message_not_visible = int(attr['ApproximateNumberOfMessagesNotVisible']) 139 | queue_value = sum([num_message, num_message_not_visible]) 140 | sum_msg = """'ApproximateNumberOfMessages' and 'ApproximateNumberOfMessagesNotVisible' = *** [%s] *** for QUEUE NAME: [%s]""" %\ 141 | (queue_value, queue_name) 142 | LOG.info(sum_msg) 143 | return queue_value 144 | 145 | def delete_sqs_msg(queue_name, receipt_handle): 146 | 147 | sqs_client = sqs_connection() 148 | try: 149 | queue_url = sqs_client.get_queue_url(QueueName=queue_name)["QueueUrl"] 150 | delete_log_msg = "Deleting msg with ReceiptHandle %s" % receipt_handle 151 | LOG.info(delete_log_msg) 152 | response = sqs_client.delete_message(QueueUrl=queue_url, ReceiptHandle=receipt_handle) 153 | except botocore.exceptions.ClientError as error: 154 | exception_msg = "FAILURE TO DELETE SQS MSG: Queue Name [%s] with error: [%s]" %\ 155 | (queue_name, error) 156 | LOG.exception(exception_msg) 157 | return None 158 | 159 | delete_log_msg_resp = "Response from delete from queue: %s" % response 160 | LOG.info(delete_log_msg_resp) 161 | return response 162 | 163 | 164 | ### S3 ### 165 | def s3_resource(): 166 | """Create S3 Resource""" 167 | 168 | resource = boto3.resource('s3', region_name=REGION) 169 | LOG.info("s3 RESOURCE connection initiated") 170 | return resource 171 | 172 | def write_s3(source_file, file_to_write, bucket): 173 | """Write S3 Bucket""" 174 | 175 | # Boto 3 176 | s3 = s3_resource() 177 | path = f'{source_file}' 178 | res = s3.Object(bucket, file_to_write).\ 179 | put(Body=open(path, 'rb')) 180 | LOG.info(f"result of write {file_to_write} | {bucket} with:\n {res}") 181 | s3_payload = (bucket, file_to_write) 182 | return s3_payload 183 | 184 | def df_read_csv(file_to_read, bucket): 185 | """Uses pandas to read s3 csv and return DataFrame 186 | 187 | Ref: 188 | https://stackoverflow.com/questions/37703634/\ 189 | how-to-import-a-text-file-on-aws-s3-into-pandas\ 190 | -without-writing-to-disk 191 | 192 | output looks like: 193 | 194 | In [12]: df.columns 195 | Out[12]: 196 | Index(['StartDate', 'EndDate', 'Status', 'IPAddress', 'Progress', 197 | 'Duration (in seconds)', 'Finished', 'RecordedDate', 'ResponseId', 198 | 'RecipientLastName', 'RecipientFirstName', 'RecipientEmail', 199 | 'ExternalReference', 'LocationLatitude', 'LocationLongitude', 200 | 'DistributionChannel', 'UserLanguage', 'Q1', 'Q2', 201 | 'Q_RecipientPhoneNumber'], 202 | dtype='object') 203 | 204 | """ 205 | 206 | s3 = boto3.client('s3') 207 | obj = s3.get_object(Bucket=bucket, Key=file_to_read) 208 | LOG.info(f"reading s3:{bucket}/{file_to_read}") 209 | df = pd.read_csv(io.BytesIO(obj['Body'].read())) 210 | return df 211 | 212 | def list_qualtrics_bucket_content(bucket): 213 | """Lists content of qualtrics bucket 214 | 215 | Ref: 216 | https://stackoverflow.com/questions/30249069/\ 217 | listing-contents-of-a-bucket-with-boto3 218 | """ 219 | 220 | s3 = boto3.resource('s3') 221 | my_bucket = s3.Bucket(bucket) 222 | for found_key in my_bucket.objects.all(): 223 | print(f"Bucket: {bucket} | key: {found_key}") 224 | 225 | def size_of_zip(file_handle): 226 | """finds zip size""" 227 | 228 | size = sum([zinfo.file_size for zinfo in file_handle.filelist]) 229 | return size 230 | 231 | def download_csv_survey(survey_id="SV_1G2GmpaXrcPAenr", 232 | api_token=None, data_center="co1", file_format="csv", temp_location="/tmp"): 233 | """Download Survey""" 234 | 235 | extra_logging = {"survey_id": survey_id, "temp_location": temp_location} 236 | # Setting static parameters 237 | requestCheckProgress = 0.0 238 | progressStatus = "inProgress" 239 | baseUrl = "https://{0}.qualtrics.com/API/v3/surveys/{1}/export-responses/".format(data_center, survey_id) 240 | headers = { 241 | "content-type": "application/json", 242 | "x-api-token": api_token, 243 | } 244 | 245 | # Step 1: Creating Data Export 246 | downloadRequestUrl = baseUrl 247 | downloadRequestPayload = '{"format":"' + file_format + '"}' 248 | downloadRequestResponse = requests.request("POST", 249 | downloadRequestUrl, data=downloadRequestPayload, headers=headers) 250 | progressId = downloadRequestResponse.json()["result"]["progressId"] 251 | LOG.info(downloadRequestResponse.text, extra=extra_logging) 252 | 253 | # Step 2: Checking on Data Export Progress and waiting until export is ready 254 | while progressStatus != "complete" and progressStatus != "failed": 255 | LOG.info(f"progressStatus {progressStatus}", extra=extra_logging) 256 | requestCheckUrl = baseUrl + progressId 257 | requestCheckResponse = requests.request("GET", requestCheckUrl, headers=headers) 258 | requestCheckProgress = requestCheckResponse.json()["result"]["percentComplete"] 259 | LOG.info(f"Download is {str(requestCheckProgress)} complete", extra=extra_logging) 260 | progressStatus = requestCheckResponse.json()["result"]["status"] 261 | 262 | #step 2.1: Check for error 263 | if progressStatus is "failed": 264 | LOG.error("export failed", extra=extra_logging) 265 | raise Exception("export failed") 266 | 267 | fileId = requestCheckResponse.json()["result"]["fileId"] 268 | 269 | # Step 3: Downloading file 270 | requestDownloadUrl = baseUrl + fileId + '/file' 271 | requestDownload = requests.request("GET", requestDownloadUrl, headers=headers, stream=True) 272 | 273 | # Step 4: Unzipping the file 274 | zip_temp = io.BytesIO(requestDownload.content) 275 | zp = zipfile.ZipFile(zip_temp) 276 | size = size_of_zip(zp) #returns size and logs it 277 | filename = zp.namelist()[0] 278 | LOG.info(f"Zip Size is: {size} with filename: {filename}", extra=extra_logging) 279 | output_filename = f"{temp_location}/{survey_id}.csv" 280 | LOG.info(f"Writing ZIP CONTENTS to output_filename: {output_filename}", extra=extra_logging) 281 | with open(output_filename, "wb") as output_file: 282 | LOG.info(f"Reading zipfile with name: {filename}", extra=extra_logging) 283 | output_file.write(zp.read(filename)) 284 | 285 | LOG.info(f"Zip Extraction Complete. Returning filename: {output_filename}", extra=extra_logging) 286 | return output_filename 287 | 288 | def collect_survey_endpoint(url="https://co1.qualtrics.com/API/v3/surveys/", 289 | survey_id="SV_4JFLLqWZwHJGi3z",extra=None, api_token=None): 290 | 291 | """Grabs the information about a single survey 292 | 293 | Example url created 294 | https://co1.qualtrics.com/API/v3/surveys/SV_4JFLLqWZwHJGi3z 295 | """ 296 | 297 | endpoint = urllib.parse.urljoin(url,survey_id) 298 | LOG.info(f"Creating endpoint: {endpoint} from url: {url} and survey_id: {survey_id}", extra=extra) 299 | headers = { 300 | "content-type": "application/json", 301 | "x-api-token": api_token, 302 | } 303 | result = requests.get(endpoint, headers=headers) 304 | json_response = result.json() 305 | LOG.info(f"JSON result: {json_response} of endpoint {endpoint}", extra=extra) 306 | return json_response 307 | 308 | ##Pandas Mapping############################## 309 | ############################################## 310 | def get_question_metadata(df, extra): 311 | """Accepts original DataFrame and grabs Qualtrics MetaData 312 | 313 | 'Q0': '{"ImportId":"QID24"}', 314 | 'Q1': '{"ImportId":"QID135400005_TEXT"}', 315 | """ 316 | 317 | metadata_recs = {} 318 | for key, value in df.iloc[1].items(): 319 | metadata_recs[key]=value 320 | LOG.info(f"Metadata recs: {metadata_recs}", extra=extra) 321 | return metadata_recs 322 | 323 | def make_sort(question, responseid, extra=None): 324 | """Makes Sort""" 325 | 326 | sort_value = f"RID-{question}-{responseid}" 327 | LOG.info(f"Creating sort_value: {sort_value}", extra=extra) 328 | return sort_value 329 | 330 | 331 | 332 | def sentiment_mapper(sentiment): 333 | """Maps a sentiment to a numerical value""" 334 | 335 | sentiment_map = { 336 | "NEGATIVE": '1', 337 | "MIXED": '0', 338 | "NEUTRAL": '2', 339 | "POSITIVE": '3', 340 | } 341 | result = sentiment_map[sentiment] 342 | LOG.info(f"Sentiment found {sentiment} with result {result}") 343 | return result 344 | 345 | def create_sentiment(row, extra=None): 346 | """Uses AWS Comprehend to Create Sentiment 347 | 348 | Return Categorical Sentiment: Positive, Neutral, Negative: 349 | 350 | Example payload from comprehend: 351 | {'Sentiment': 'NEUTRAL', 352 | 'SentimentScore': {'Positive': 0.05472605302929878, 353 | 'Negative': 0.011656931601464748, 354 | 'Neutral': 0.9297710061073303, 355 | 'Mixed': 0.003845960134640336}, 356 | 357 | """ 358 | 359 | LOG.info(f"CREATE SENTIMENT with raw value: {row}", extra=extra) 360 | comprehend = boto3.client(service_name='comprehend') 361 | payload = comprehend.detect_sentiment(Text=row, LanguageCode='en') 362 | LOG.info(f"Found Sentiment: {payload}", extra=extra) 363 | sentiment_category = payload['Sentiment'] 364 | LOG.info(f"Sentiment Category: {sentiment_category}", extra=extra) 365 | sentiment_score = payload['SentimentScore'] 366 | LOG.info(f"Sentiment Score: {sentiment_score}", extra=extra) 367 | map_sentiment_result = sentiment_mapper(sentiment=sentiment_category) 368 | LOG.info(f"Created Sentiment Score: {map_sentiment_result}") 369 | return map_sentiment_result 370 | 371 | def make_record(iloc, extra=None, questions_choices=None): 372 | """Makes DynamoDB Record From DataFrame 373 | 374 | response = { 375 | ## More information about these fields 376 | #https://docs.google.com/document/d/1A4UQvA8uNC6boQ3OjO-G0v9EdVrypKUS4KpCqmulPTM/edit 377 | 378 | """ 379 | 380 | LOG.info(f"make_record: Making Record for dyanamodb", extra=extra) 381 | recs = [] 382 | partition = iloc.get("Partition") 383 | responseid = iloc.get("_recordId") 384 | question_index = [col for col in iloc.keys() if (col.startswith("QID") or col == 'Text')] 385 | for question in question_index: 386 | question_value = iloc[question] 387 | if not question_value: 388 | LOG.info(f"Empty Question: {question}", extra=extra) 389 | continue 390 | else: 391 | LOG.info(f"Processing Question:{question}", extra=extra) 392 | new_rec = {} 393 | 394 | if question == "Text": 395 | LOG.info(f"Found Text question: {question} with value {question_value}", extra=extra) 396 | # Parse response, it contains the question id 397 | # response format for these: //ChoiceTextEntryValue} 398 | 399 | text, question, response_type = question_value.split('/') 400 | 401 | if text and question.startswith('QID'): 402 | #Since there is Text, we can also create sentiment 403 | new_rec["Text"] = text 404 | new_rec["Sentiment"] = create_sentiment(row=new_rec["Text"],extra=extra) 405 | # We want to match the question id of the question in it's own column 406 | # so we don't make duplicate rows 407 | question = f"{question}_TEXT" 408 | else: 409 | LOG.warning(f"Unable to process response: {question_value}", extra=extra) 410 | continue 411 | elif question.endswith("_TEXT"): 412 | LOG.info(f"Found open response question {question} with value {question_value}", extra=extra) 413 | new_rec["OpenResponse"] = question_value 414 | else: 415 | LOG.info(f"Found CHOICE question: {question} with value {question_value}", extra=extra) 416 | new_rec["Choice"] = question_value 417 | 418 | try: 419 | new_rec["Sort"] = make_sort(question, responseid, extra=extra) 420 | new_rec["Partition"] = partition 421 | new_rec["LSI"] = question 422 | new_rec["Origin"] = iloc.get("Origin") 423 | new_rec["Race"] = iloc.get("Race") 424 | new_rec["Age"] = iloc.get("Age") 425 | new_rec["Gender"] = iloc.get("Gender") 426 | 427 | 428 | #handle empty latitude 429 | latitude = iloc.get('Latitude') or 0.0 430 | new_rec["Latitude"] = latitude 431 | new_rec["LatitudeOffset"] = f"{float(latitude):019.15F}" 432 | 433 | #handle empty longitude 434 | longitude = iloc.get('Longitude') or 0.0 435 | new_rec["Longitude"] = longitude 436 | new_rec["LongitudeOffset"] = f"{(float(longitude) + 200):019.15F}" 437 | 438 | new_rec["Date"] = str(time.mktime(dateutil.parser.parse(iloc.get("Date")).timetuple())) 439 | new_rec["IncidentId"] = iloc.get("IncidentId") 440 | new_rec["rojopolisEncounterScore"] = iloc.get("rojopolisEncounterScore") 441 | new_rec["rojopolisGeneralScore"] = iloc.get("rojopolisGeneralScore") 442 | 443 | #handle empty phone set to: 00000000000 444 | phone_number = iloc.get("PhoneNumber") or "00000000000" 445 | new_rec["PhoneNumber"] = encrypt(phone_number) 446 | 447 | except Exception as error: 448 | LOG.exception(f"Problem making record", extra=extra) 449 | raise error 450 | if question in questions_choices: 451 | new_rec["QuestionChoicesId"] = questions_choices[question] 452 | LOG.info(f"new_rec **BEFORE** None filter: {new_rec}", extra=extra) 453 | new_rec = {x:y for x,y in new_rec.items() if y != ""} 454 | LOG.info(f"new_rec **AFTER** None filter: {new_rec}", extra=extra) 455 | recs.append(new_rec) 456 | LOG.info(f"Created Records: {recs}", extra=extra) 457 | return recs 458 | 459 | def get_question_columns(df, extra): 460 | """takes a DataFrame and returns Question Key/Value Pairs """ 461 | 462 | cols = [col for col in list(df.columns) if (col.startswith("QID") or col == 'Text')] 463 | LOG.info(f"Found Question Columns: {cols}", extra=extra) 464 | return cols 465 | 466 | def fill_empty_values(df=None, fill_value="", extra=None): 467 | """Fills empty values with fill value, defaults to empty string""" 468 | 469 | LOG.info(f"Filling DataFrame Empty Values with fill value: {fill_value}", extra=extra) 470 | df_filled = df.fillna(fill_value) 471 | return df_filled 472 | 473 | def rename_df_colnames_cleanup(df,extra): 474 | """Rename the columns, drop first two rows and clean""" 475 | 476 | #rename columns 477 | import ast 478 | vals = df.iloc[1].tolist() 479 | columns = [ast.literal_eval(x)['ImportId'] for x in vals] 480 | df.columns = columns 481 | LOG.info(f"Set Columns via AST Rename: {columns}") 482 | 483 | #go back to renaming 484 | new_columns = {} 485 | recs = get_question_metadata(df, extra) 486 | LOG.info(f"rename_df_colnames_cleanup: METADATA {recs}", extra=extra) 487 | cols = get_question_columns(df,extra) 488 | new_recs = dict((k, recs[k]) for k in cols) 489 | LOG.info(f"rename_df_colnames_cleanup: NEW_COLUMNS: {new_recs}", extra=extra) 490 | for key,value in new_recs.items(): 491 | values = eval(value) 492 | LOG.info(f"rename_df_colnames_cleanup: VALUES: {values}", extra=extra) 493 | new_columns[key]= list(values.values())[0] 494 | LOG.info(f"Created new DataFrame Column Names: NEW_COLUMNS {new_columns}", extra=extra) 495 | df = df.rename(columns=new_columns) 496 | #drop row 497 | df = df.iloc[1:] 498 | LOG.info(f"Dropped first row: {df.head(2)}", extra=extra) 499 | df = fill_empty_values(df=df, extra=extra) 500 | return df 501 | 502 | def populate_dynamodb(rec, extra=None): 503 | """Creates a DynamoDB Record""" 504 | 505 | LOG.info(f"Populating DynamoDB with rec: {rec}", extra=extra) 506 | try: 507 | res = TABLE.put_item(Item=rec) 508 | except Exception: # pylint:disable=broad-except 509 | LOG.exception(f"FATAL--ERROR--WRITING--TO--DYNAMO for rec: {rec}", extra=extra) 510 | return None 511 | LOG.info(f"SUCCESS**WRITE**RECORD**DYNAMO for rec {rec} with response: {res}", extra=extra) 512 | 513 | def pd_table_populate(df=None, extra=None, survey_id=None, api_token=None, agency_id=None): 514 | """Populate DynamoDB with contents of survey dataframe""" 515 | 516 | #collect survey metadata 517 | response = collect_survey_endpoint(survey_id=survey_id, 518 | extra=extra, api_token=api_token) 519 | questions, choices = process_questions_from_survey(aid=agency_id, 520 | survey_data=response['result'], extra=extra) 521 | LOG.info(f"Creating questions {questions} and choices {choices}", extra=extra) 522 | 523 | #Process Questions 524 | for question in questions: 525 | LOG.info(f"Processing question: {question}", extra=extra) 526 | populate_dynamodb(question, extra=extra) 527 | 528 | #Process Choices 529 | for choice in choices: 530 | LOG.info(f"Processing choice: {choice}", extra=extra) 531 | populate_dynamodb(choice, extra=extra) 532 | 533 | #Create Question Choices 534 | questions_choices = {x['Sort']: x['QuestionChoicesId'] for x in questions if 'QuestionChoicesId' in x} 535 | LOG.info(f"Create Question Choices: {questions_choices}", extra=extra) 536 | 537 | #Rename columns and process records 538 | df = rename_df_colnames_cleanup(df, extra) 539 | LOG.info(f"Created DataFrame: {df.iloc[1]}", extra=extra) 540 | rows,_ = df.shape 541 | LOG.info(f"Found number of rows: {rows}", extra=extra) 542 | for index in range(1,rows): 543 | LOG.info(f"Processing DataFrame Row {index}", extra=extra) 544 | recs = make_record(df.iloc[index], extra=extra, questions_choices=questions_choices) 545 | for rec in recs: 546 | LOG.info(f"Processing a rec: {rec}", extra=extra) 547 | populate_dynamodb(rec, extra=extra) 548 | LOG.info(f"FINISHED: Processing DataFrame Rows", extra=extra) 549 | return df 550 | 551 | 552 | def process_questions_from_survey(aid, survey_data, extra=None): 553 | ''' Take suvey data and determine quesion choices and questions to make 554 | returns two lists of dicts, one representing questions and one 555 | representing question choices. 556 | 557 | Params: 558 | aid: string id of the agency 559 | survey_data: 'result' section from qualtrics survey endpoint 560 | ''' 561 | questions_items = [] 562 | choices_items = [] 563 | if survey_data: 564 | LOG.info(f"survey_data {survey_data}, aid {aid}", extra=extra) 565 | for question_key, question_data in survey_data['questions'].items(): 566 | question_item = {'Partition': aid, 567 | 'Sort': question_key, 568 | 'Text': question_data['questionText']} 569 | 570 | if 'choices' in question_data: 571 | # The choices should be ordered by the 'recode' field 572 | choices_texts = [y['choiceText'] for y in sorted(question_data['choices'].values(), key=lambda x:x['recode'])] 573 | h = md5() 574 | h.update(str(choices_texts).encode()) 575 | qcid = f"QCID-{h.hexdigest()}" 576 | choices_items.append({'Partition': aid, 577 | 'Sort': qcid, 578 | 'Choices':str(choices_texts)}) 579 | 580 | question_item['QuestionChoicesId'] = qcid 581 | 582 | questions_items.append(question_item) 583 | 584 | return questions_items, choices_items 585 | 586 | 587 | def entrypoint(event, context): 588 | ''' 589 | Lambda entrypoint 590 | ''' 591 | 592 | LOG.info(f"SURVEYJOB LAMBDA, event {event}, context {context}", extra=os.environ) 593 | receipt_handle = event['Records'][0]['receiptHandle'] #sqs message 594 | #'eventSourceARN': 'arn:aws:sqs:us-east-1:698112575222:etl-queue-etl-resources' 595 | event_source_arn = event['Records'][0]['eventSourceARN'] 596 | for record in event['Records']: 597 | body = json.loads(record['body']) 598 | survey_id = body['SurveyId'] 599 | agency_id = body['AgencyId'] 600 | api_token = os.environ.get('X_API_TOKEN') 601 | table_id = os.environ.get('AGENCIES_TABLE_ID') 602 | global TABLE # pylint:disable=W0603 603 | TABLE = DYNAMODB.Table(table_id)# pylint:disable=W0621 604 | bucket = os.environ.get('S3_BUCKET') 605 | extra_logging = {"body": body, "survey_id": survey_id, "lambda role": "SURVEYJOB", 606 | "agency_id":agency_id, api_token: "api_token", "bucket": bucket, "table": {TABLE}} 607 | LOG.info(f"SURVEYJOB LAMBDA, splitting sqs arn with value: {event_source_arn}",extra=extra_logging) 608 | qname = event_source_arn.split(":")[-1] 609 | extra_logging["queue"] = qname 610 | LOG.info(f"Calling click run function: Will download qualtrics data and write to s3", extra=extra_logging) 611 | written_bucket, downloaded_csv_file = cli.main( 612 | args=[ 613 | 'run', 614 | '--surveyid', survey_id, 615 | '--apitoken', api_token, 616 | '--bucket', bucket, 617 | '--queue', qname 618 | ], 619 | standalone_mode=False 620 | ) 621 | LOG.info(f"this is downloaded_csv_file path: {downloaded_csv_file} from bucket: {written_bucket}",extra=extra_logging) 622 | extra_logging["csvfile"] = downloaded_csv_file 623 | extra_logging["written_bucket"] = written_bucket 624 | LOG.info(f"Running sync-db click function: will read from s3 and map csv data to dynamodb",extra=extra_logging) 625 | cli.main( 626 | args=[ 627 | 'sync-db', 628 | '--surveyid', survey_id, 629 | '--apitoken', api_token, 630 | '--bucket', written_bucket, 631 | '--agencyid', agency_id, 632 | '--csvfile', downloaded_csv_file, 633 | '--queue', qname 634 | ], 635 | standalone_mode=False 636 | ) 637 | LOG.info(f"Attemping Deleting SQS receiptHandle {receipt_handle} with queue_name {qname}", extra=extra_logging) 638 | res = delete_sqs_msg(queue_name=qname, receipt_handle=receipt_handle) 639 | LOG.info(f"Deleted SQS receipt_handle {receipt_handle} with res {res}", extra=extra_logging) 640 | 641 | @click.group() 642 | def cli(): 643 | pass 644 | 645 | @click.option("--qurl", 646 | default="etl-queue-etl-resources", 647 | help="Finds out number of messages in a AWS queue") 648 | @cli.command() 649 | def qcount(qurl): 650 | """Util for Queue count""" 651 | 652 | LOG.info(f"Using queue name {qurl}") 653 | click.echo(sqs_approximate_count(queue_name=qurl)) 654 | 655 | @cli.command() 656 | @click.option("--surveyid", envvar="SURVEY_TABLE", 657 | default="SV_1G2GmpaXrcPAenr", help="qualtrics survey id") 658 | @click.option("--apitoken", envvar="X_API_TOKEN", help="apitoken") 659 | @click.option("--bucket", envvar="SURVEY_BUCKET", help="S3 bucket to write survey csv") 660 | @click.option("--queue", default=None) 661 | def run(surveyid, apitoken, bucket, queue): 662 | """Run export via cli and write to s3""" 663 | 664 | extra_logging = {"surveyid":surveyid, "apitoken":apitoken, "bucket":bucket, "queue":queue} 665 | LOG.info(f"Running Click run with surveyid", extra=extra_logging) 666 | downloaded_csv_file = download_csv_survey(api_token=apitoken, survey_id=surveyid) 667 | file_name = os.path.split(downloaded_csv_file)[-1] 668 | LOG.info(f"Found filename {file_name}", extra=extra_logging) 669 | s3_name_to_create = f"{surveyid}-{file_name}" 670 | LOG.info(f"Writing qualtrics download with name: {s3_name_to_create} to S3", extra=extra_logging) 671 | s3_file_handle = write_s3(source_file=downloaded_csv_file, 672 | file_to_write=s3_name_to_create, bucket=bucket) 673 | LOG.info(f"Running export with downloaded_csv_file {downloaded_csv_file}", extra=extra_logging) 674 | LOG.info(f"Boto S3 file handle: {s3_file_handle}", extra=extra_logging) 675 | return s3_file_handle 676 | 677 | @cli.command() 678 | @click.option("--csvfile", envvar="CSV_FILE", 679 | help="s3 based csv file") 680 | @click.option("--bucket", envvar="SURVEY_BUCKET", 681 | help="s3 bucket") 682 | @click.option("--agencyid", 683 | help="Agency ID") 684 | @click.option("--queue", default=None) 685 | @click.option("--surveyid", envvar="SURVEY_TABLE", 686 | default="SV_1G2GmpaXrcPAenr", help="qualtrics survey id") 687 | @click.option("--apitoken", envvar="X_API_TOKEN", help="apitoken") 688 | def sync_db(csvfile, bucket, agencyid, queue, surveyid, apitoken): 689 | """Sync CSV to DynamoDB 690 | 691 | 692 | To test locally: 693 | 694 | python qualtrics.py sync-db --bucket rojopolis-survey-us-east-1-698112575222 \ 695 | --csvfile "SV_cGXWxvADgIihxrf-Example Qualtrics Output.csv" 696 | 697 | """ 698 | saved_args = locals() 699 | LOG.info(f'locals in sync_db: {saved_args}') 700 | extra_logging = { 701 | "csvfile": csvfile, 702 | "bucket": bucket, 703 | "agencyid": agencyid, 704 | "queue": queue, 705 | "function_name" :"sync_db", 706 | "locals": saved_args 707 | } 708 | LOG.info(f"Running Click syncdb with csvfile", extra=extra_logging) 709 | df = df_read_csv( 710 | file_to_read=csvfile, 711 | bucket=bucket 712 | ) 713 | LOG.info(f"Contents of initial dataframe: {df.to_dict()}", extra=extra_logging) 714 | #LOG.info(f"Found Survey Metadata: {}") 715 | LOG.info(f"Found DataFrame Columns: {df.columns}", extra=extra_logging);df.head() 716 | LOG.info(f"START SYNCDB:", extra=extra_logging) 717 | pd_table_populate(df,extra=extra_logging, survey_id=surveyid, 718 | api_token=apitoken, agency_id=agencyid) 719 | LOG.info(f"FINISH SYNCDB: ", extra=extra_logging) 720 | 721 | if __name__ == "__main__": 722 | API_TOKEN, TABLE = setup_environment() 723 | cli() 724 | -------------------------------------------------------------------------------- /app/lambda/functions/surveyjobs/requirements.txt: -------------------------------------------------------------------------------- 1 | requests -------------------------------------------------------------------------------- /app/lambda/functions/surveyjobs/sentiment.py: -------------------------------------------------------------------------------- 1 | """Sentiments Tool""" 2 | 3 | #SETUP LOGGING 4 | import logging 5 | from pythonjsonlogger import jsonlogger 6 | 7 | LOG = logging.getLogger() 8 | LOG.setLevel(logging.DEBUG) 9 | logHandler = logging.StreamHandler() 10 | formatter = jsonlogger.JsonFormatter() 11 | logHandler.setFormatter(formatter) 12 | LOG.addHandler(logHandler) 13 | 14 | import click 15 | import boto3 16 | import pandas as pd 17 | 18 | TEST_DF = pd.DataFrame( 19 | {"SentimentRaw": ["I am very Angry", 20 | "We are very Happy", 21 | "It is raining in Seattle"]} 22 | ) 23 | 24 | def create_sentiment(row): 25 | """Uses AWS Comprehend to Create Sentiments on a DataFrame""" 26 | 27 | LOG.info(f"Processing {row}") 28 | comprehend = boto3.client(service_name='comprehend') 29 | payload = comprehend.detect_sentiment(Text=row, LanguageCode='en') 30 | LOG.debug(f"Found Sentiment: {payload}") 31 | sentiment = payload['Sentiment'] 32 | return sentiment 33 | 34 | def apply_sentiment(df, column="SentimentRaw"): 35 | """Uses Pandas Apply to Create Sentiment Analysis""" 36 | 37 | df['Sentiment'] = df[column].apply(create_sentiment) 38 | return df 39 | 40 | @click.group() 41 | def cli(): 42 | pass 43 | 44 | @cli.command() 45 | def dataframe_sentiments(df=TEST_DF): 46 | """Processes DataFrame and adds Sentiment 47 | 48 | To run: 49 | python sentiment.py dataframe-sentiments 50 | """ 51 | 52 | df_incoming = apply_sentiment(df) 53 | click.echo(df_incoming) 54 | 55 | 56 | 57 | if __name__ == "__main__": 58 | cli() -------------------------------------------------------------------------------- /app/lambda/functions/surveyjobs/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/app/lambda/functions/surveyjobs/tests/__init__.py -------------------------------------------------------------------------------- /app/lambda/functions/surveyjobs/tests/surveys_endpoint_response.json: -------------------------------------------------------------------------------- 1 | { 2 | "result": { 3 | "id": "SV_4JFLLqWZwHJGi3z", 4 | "name": "QA / Metropolis PD / Call for Service / Police Reported Demographics", 5 | "ownerId": "UR_73s2JubmBGFaLvT", 6 | "organizationId": "rojopolis", 7 | "isActive": true, 8 | "creationDate": "2018-12-07T03:11:25Z", 9 | "lastModifiedDate": "2018-12-07T05:12:43Z", 10 | "expiration": { 11 | "startDate": null, 12 | "endDate": null 13 | }, 14 | "questions": { 15 | "QID24": { 16 | "questionType": { 17 | "type": "MC", 18 | "selector": "SAVR", 19 | "subSelector": "TX" 20 | }, 21 | "questionText": "Hi, this is rojopolis. We're an independent group that improves public safety with community feedback. We will share your thoughts directly to your police chief, but you will be ANONYMOUS. First, we need your consent. This will be a short conversation! Do you want to continue?", 22 | "questionLabel": null, 23 | "validation": { 24 | "doesForceResponse": false 25 | }, 26 | "questionName": "Q0", 27 | "choices": { 28 | "1": { 29 | "recode": "1", 30 | "description": "English", 31 | "choiceText": "Yes, continue in English", 32 | "imageDescription": null, 33 | "variableName": "English", 34 | "analyze": true 35 | }, 36 | "2": { 37 | "recode": "2", 38 | "description": "Spanish", 39 | "choiceText": "Sí, continuar en Español", 40 | "imageDescription": null, 41 | "variableName": "Spanish", 42 | "analyze": true 43 | }, 44 | "3": { 45 | "recode": "3", 46 | "description": "No", 47 | "choiceText": "No", 48 | "imageDescription": null, 49 | "variableName": "No", 50 | "analyze": true 51 | } 52 | } 53 | }, 54 | "QID135400005": { 55 | "questionType": { 56 | "type": "TE", 57 | "selector": "SL", 58 | "subSelector": null 59 | }, 60 | "questionText": "Confirmed! Reply STOP end anytime; standard messaging rates apply. On a scale of 0-100, how would you rate the overall performance of the responding officer(s)?", 61 | "questionLabel": null, 62 | "validation": { 63 | "doesForceResponse": false, 64 | "type": "ValidNumber", 65 | "settings": { 66 | "maxDecimals": 0, 67 | "maximum": 100, 68 | "minimum": 0 69 | } 70 | }, 71 | "questionName": "Q1" 72 | }, 73 | "QID25": { 74 | "questionType": { 75 | "type": "MC", 76 | "selector": "SAVR", 77 | "subSelector": "TX" 78 | }, 79 | "questionText": "What is the #1 public safety issue that needs to be addressed for you to feel safer in your neighborhood?", 80 | "questionLabel": null, 81 | "validation": { 82 | "doesForceResponse": false 83 | }, 84 | "questionName": "Q2", 85 | "choices": { 86 | "1": { 87 | "recode": "1", 88 | "description": "Property crime", 89 | "choiceText": "Property crime", 90 | "imageDescription": null, 91 | "variableName": "Property crime", 92 | "analyze": true 93 | }, 94 | "2": { 95 | "recode": "2", 96 | "description": "Violent crime", 97 | "choiceText": "Violent crime", 98 | "imageDescription": null, 99 | "variableName": "Violent crime", 100 | "analyze": true 101 | }, 102 | "3": { 103 | "recode": "3", 104 | "description": "Environmental issues (i.e. lighting, stop signs, etc.)", 105 | "choiceText": "Environmental issues (i.e. lighting, stop signs, etc.)", 106 | "imageDescription": null, 107 | "variableName": "Environmental issues (i.e. lighting, stop signs, etc.)", 108 | "analyze": true 109 | }, 110 | "4": { 111 | "recode": "4", 112 | "description": "Officer behavior and trust in police", 113 | "choiceText": "Officer behavior and trust in police", 114 | "imageDescription": null, 115 | "variableName": "Officer behavior and trust in police", 116 | "analyze": true 117 | }, 118 | "5": { 119 | "recode": "5", 120 | "description": "Other", 121 | "choiceText": "Other", 122 | "imageDescription": null, 123 | "variableName": "Other", 124 | "analyze": true 125 | } 126 | } 127 | }, 128 | "QID26": { 129 | "questionType": { 130 | "type": "TE", 131 | "selector": "SL", 132 | "subSelector": null 133 | }, 134 | "questionText": "Ok, can you please explain?", 135 | "questionLabel": null, 136 | "validation": { 137 | "doesForceResponse": false 138 | }, 139 | "questionName": "Q3" 140 | }, 141 | "QID27": { 142 | "questionType": { 143 | "type": "MC", 144 | "selector": "SAVR", 145 | "subSelector": "TX" 146 | }, 147 | "questionText": "Got it. Thanks. We're so sorry that you're having to go through all of this. We can put you directly in touch with services and people that can provide assistance if you need it. If you want us to do this, please select from the following list, and we'll have someone follow up with you:", 148 | "questionLabel": null, 149 | "validation": { 150 | "doesForceResponse": false 151 | }, 152 | "questionName": "Q4", 153 | "choices": { 154 | "1": { 155 | "recode": "1", 156 | "description": "Community organizations for neighborhood issues", 157 | "choiceText": "Community organizations for neighborhood issues", 158 | "imageDescription": null, 159 | "variableName": "Community organizations for neighborhood issues", 160 | "analyze": true 161 | }, 162 | "2": { 163 | "recode": "2", 164 | "description": "Services that help people after experiencing a crime", 165 | "choiceText": "Services that help people after experiencing a crime", 166 | "imageDescription": null, 167 | "variableName": "Services that help people after experiencing a crime", 168 | "analyze": true 169 | }, 170 | "3": { 171 | "recode": "3", 172 | "description": "Organizations that help with legal issues relating to a crimes, tickets, etc.", 173 | "choiceText": "Organizations that help with legal issues relating to a crimes, tickets, etc.", 174 | "imageDescription": null, 175 | "variableName": "Organizations that help with legal issues relating to a crimes, tickets, etc.", 176 | "analyze": true 177 | }, 178 | "4": { 179 | "recode": "4", 180 | "description": "Groups that provide insurance, safety, or security around my home", 181 | "choiceText": "Groups that provide insurance, safety, or security around my home", 182 | "imageDescription": null, 183 | "variableName": "Groups that provide insurance, safety, or security around my home", 184 | "analyze": true 185 | }, 186 | "5": { 187 | "recode": "5", 188 | "description": "My local government representatives office", 189 | "choiceText": "My local government representatives office", 190 | "imageDescription": null, 191 | "variableName": "My local government representatives office", 192 | "analyze": true 193 | }, 194 | "6": { 195 | "recode": "6", 196 | "description": "No, I just need to get back to my normal routine", 197 | "choiceText": "No, I just need to get back to my normal routine", 198 | "imageDescription": null, 199 | "variableName": "No, I just need to get back to my normal routine", 200 | "analyze": true 201 | } 202 | } 203 | }, 204 | "QID28": { 205 | "questionType": { 206 | "type": "TE", 207 | "selector": "SL", 208 | "subSelector": null 209 | }, 210 | "questionText": "Thanks for your feedback. You can view what others in your city and neighborhood are saying here: www.textrojopolis.com/community. If you have anything else to say, please text it now.", 211 | "questionLabel": null, 212 | "validation": { 213 | "doesForceResponse": false 214 | }, 215 | "questionName": "Q5" 216 | }, 217 | "QID29": { 218 | "questionType": { 219 | "type": "TE", 220 | "selector": "SL", 221 | "subSelector": null 222 | }, 223 | "questionText": "Ok, someone will be in touch. Thanks for your feedback. You can view what others in your city and neighborhood are saying here: www.textrojopolis.com/community. If you have anything else to say, please text it now.", 224 | "questionLabel": null, 225 | "validation": { 226 | "doesForceResponse": false 227 | }, 228 | "questionName": "Q6" 229 | }, 230 | "QID17": { 231 | "questionType": { 232 | "type": "TE", 233 | "selector": "SL", 234 | "subSelector": null 235 | }, 236 | "questionText": "Confirmed! Reply STOP end anytime; standard messaging rates apply. On a scale of 0-100, how would you rate the overall performance of the responding officer(s)?", 237 | "questionLabel": null, 238 | "validation": { 239 | "doesForceResponse": false, 240 | "type": "CustomValidation" 241 | }, 242 | "questionName": "Q1S" 243 | }, 244 | "QID18": { 245 | "questionType": { 246 | "type": "MC", 247 | "selector": "SAVR", 248 | "subSelector": "TX" 249 | }, 250 | "questionText": "What is the #1 public safety issue that needs to be addressed for you to feel safer in your neighborhood?", 251 | "questionLabel": null, 252 | "validation": { 253 | "doesForceResponse": false 254 | }, 255 | "questionName": "Q2S", 256 | "choices": { 257 | "1": { 258 | "recode": "1", 259 | "description": "Property crime", 260 | "choiceText": "Property crime", 261 | "imageDescription": null, 262 | "variableName": "Property crime", 263 | "analyze": true 264 | }, 265 | "2": { 266 | "recode": "2", 267 | "description": "Violent crime", 268 | "choiceText": "Violent crime", 269 | "imageDescription": null, 270 | "variableName": "Violent crime", 271 | "analyze": true 272 | }, 273 | "3": { 274 | "recode": "3", 275 | "description": "Environmental issues (i.e. lighting, stop signs, etc.)", 276 | "choiceText": "Environmental issues (i.e. lighting, stop signs, etc.)", 277 | "imageDescription": null, 278 | "variableName": "Environmental issues (i.e. lighting, stop signs, etc.)", 279 | "analyze": true 280 | }, 281 | "4": { 282 | "recode": "4", 283 | "description": "Officer behavior and trust in police", 284 | "choiceText": "Officer behavior and trust in police", 285 | "imageDescription": null, 286 | "variableName": "Officer behavior and trust in police", 287 | "analyze": true 288 | }, 289 | "5": { 290 | "recode": "5", 291 | "description": "Other", 292 | "choiceText": "Other", 293 | "imageDescription": null, 294 | "variableName": "Other", 295 | "analyze": true 296 | } 297 | } 298 | }, 299 | "QID19": { 300 | "questionType": { 301 | "type": "TE", 302 | "selector": "SL", 303 | "subSelector": null 304 | }, 305 | "questionText": "Ok, can you please explain?", 306 | "questionLabel": null, 307 | "validation": { 308 | "doesForceResponse": false 309 | }, 310 | "questionName": "Q3S" 311 | }, 312 | "QID30": { 313 | "questionType": { 314 | "type": "MC", 315 | "selector": "SAVR", 316 | "subSelector": "TX" 317 | }, 318 | "questionText": "Got it. Thanks. We're so sorry that you're having to go through all of this. We can put you directly in touch with services and people that can provide assistance if you need it. If you want us to do this, please select from the following list, and we'll have someone follow up with you:", 319 | "questionLabel": null, 320 | "validation": { 321 | "doesForceResponse": false 322 | }, 323 | "questionName": "Q4S", 324 | "choices": { 325 | "1": { 326 | "recode": "1", 327 | "description": "Community organizations for neighborhood issues", 328 | "choiceText": "Community organizations for neighborhood issues", 329 | "imageDescription": null, 330 | "variableName": "Community organizations for neighborhood issues", 331 | "analyze": true 332 | }, 333 | "2": { 334 | "recode": "2", 335 | "description": "Services that help people after experiencing a crime", 336 | "choiceText": "Services that help people after experiencing a crime", 337 | "imageDescription": null, 338 | "variableName": "Services that help people after experiencing a crime", 339 | "analyze": true 340 | }, 341 | "3": { 342 | "recode": "3", 343 | "description": "Organizations that help with legal issues relating to a crimes, tickets, etc.", 344 | "choiceText": "Organizations that help with legal issues relating to a crimes, tickets, etc.", 345 | "imageDescription": null, 346 | "variableName": "Organizations that help with legal issues relating to a crimes, tickets, etc.", 347 | "analyze": true 348 | }, 349 | "4": { 350 | "recode": "4", 351 | "description": "Groups that provide insurance, safety, or security around my home", 352 | "choiceText": "Groups that provide insurance, safety, or security around my home", 353 | "imageDescription": null, 354 | "variableName": "Groups that provide insurance, safety, or security around my home", 355 | "analyze": true 356 | }, 357 | "5": { 358 | "recode": "5", 359 | "description": "My local government representatives office", 360 | "choiceText": "My local government representatives office", 361 | "imageDescription": null, 362 | "variableName": "My local government representatives office", 363 | "analyze": true 364 | }, 365 | "6": { 366 | "recode": "6", 367 | "description": "No, I just need to get back to my normal routine", 368 | "choiceText": "No, I just need to get back to my normal routine", 369 | "imageDescription": null, 370 | "variableName": "No, I just need to get back to my normal routine", 371 | "analyze": true 372 | } 373 | } 374 | }, 375 | "QID31": { 376 | "questionType": { 377 | "type": "TE", 378 | "selector": "SL", 379 | "subSelector": null 380 | }, 381 | "questionText": "Thanks for your feedback. You can view what others in your city and neighborhood are saying here: www.textrojopolis.com/community. If you have anything else to say, please text it now.", 382 | "questionLabel": null, 383 | "validation": { 384 | "doesForceResponse": false 385 | }, 386 | "questionName": "Q5S" 387 | }, 388 | "QID32": { 389 | "questionType": { 390 | "type": "TE", 391 | "selector": "SL", 392 | "subSelector": null 393 | }, 394 | "questionText": "Ok, someone will be in touch. Thanks for your feedback. You can view what others in your city and neighborhood are saying here: www.textrojopolis.com/community. If you have anything else to say, please text it now.", 395 | "questionLabel": null, 396 | "validation": { 397 | "doesForceResponse": false 398 | }, 399 | "questionName": "Q6S" 400 | } 401 | }, 402 | "exportColumnMap": { 403 | "Q0": { 404 | "question": "QID24" 405 | }, 406 | "Q1": { 407 | "question": "QID135400005" 408 | }, 409 | "Q2": { 410 | "question": "QID25" 411 | }, 412 | "Q3": { 413 | "question": "QID26" 414 | }, 415 | "Q4": { 416 | "question": "QID27" 417 | }, 418 | "Q5": { 419 | "question": "QID28" 420 | }, 421 | "Q6": { 422 | "question": "QID29" 423 | }, 424 | "Q1S": { 425 | "question": "QID17" 426 | }, 427 | "Q2S": { 428 | "question": "QID18" 429 | }, 430 | "Q3S": { 431 | "question": "QID19" 432 | }, 433 | "Q4S": { 434 | "question": "QID30" 435 | }, 436 | "Q5S": { 437 | "question": "QID31" 438 | }, 439 | "Q6S": { 440 | "question": "QID32" 441 | } 442 | }, 443 | "blocks": { 444 | "BL_eVPUOqjcCrs0l2l": { 445 | "description": "Consent / Language Selection / Block", 446 | "elements": [ 447 | { 448 | "type": "Question", 449 | "questionId": "QID24" 450 | } 451 | ] 452 | }, 453 | "BL_cYjEINrxGUsY4jX": { 454 | "description": "rojopolis Score / Encounter / English", 455 | "elements": [ 456 | { 457 | "type": "Question", 458 | "questionId": "QID135400005" 459 | } 460 | ] 461 | }, 462 | "BL_38ESkhXxpNJ5qgR": { 463 | "description": "Public Safety Priority / English", 464 | "elements": [ 465 | { 466 | "type": "Question", 467 | "questionId": "QID25" 468 | }, 469 | { 470 | "type": "Question", 471 | "questionId": "QID26" 472 | } 473 | ] 474 | }, 475 | "BL_6Fr1TaFUk9DuV25": { 476 | "description": "Resources / English", 477 | "elements": [ 478 | { 479 | "type": "Question", 480 | "questionId": "QID27" 481 | } 482 | ] 483 | }, 484 | "BL_etwVTHVezjxpYln": { 485 | "description": "Resources Follow Up / English", 486 | "elements": [ 487 | { 488 | "type": "Question", 489 | "questionId": "QID28" 490 | }, 491 | { 492 | "type": "Question", 493 | "questionId": "QID29" 494 | } 495 | ] 496 | }, 497 | "BL_8bT6RE4sg2hUB0x": { 498 | "description": "rojopolis Score / Spanish", 499 | "elements": [ 500 | { 501 | "type": "Question", 502 | "questionId": "QID17" 503 | } 504 | ] 505 | }, 506 | "BL_1UkWf620nO474pv": { 507 | "description": "Public Safety Priority / Spanish", 508 | "elements": [ 509 | { 510 | "type": "Question", 511 | "questionId": "QID18" 512 | }, 513 | { 514 | "type": "Question", 515 | "questionId": "QID19" 516 | } 517 | ] 518 | }, 519 | "BL_4MBmVryNzOXnVsx": { 520 | "description": "Resources / Spanish", 521 | "elements": [ 522 | { 523 | "type": "Question", 524 | "questionId": "QID30" 525 | } 526 | ] 527 | }, 528 | "BL_78VAPu4fVL8Wznn": { 529 | "description": "Resources Follow Up / Spanish", 530 | "elements": [ 531 | { 532 | "type": "Question", 533 | "questionId": "QID31" 534 | }, 535 | { 536 | "type": "Question", 537 | "questionId": "QID32" 538 | } 539 | ] 540 | } 541 | }, 542 | "flow": [ 543 | { 544 | "id": "BL_eVPUOqjcCrs0l2l", 545 | "type": "Block" 546 | }, 547 | { 548 | "type": "EmbeddedData" 549 | }, 550 | { 551 | "type": "Branch", 552 | "flow": [ 553 | { 554 | "type": "EndSurvey" 555 | } 556 | ] 557 | }, 558 | { 559 | "type": "Branch", 560 | "flow": [ 561 | { 562 | "id": "BL_cYjEINrxGUsY4jX", 563 | "type": "Block" 564 | }, 565 | { 566 | "type": "EmbeddedData" 567 | }, 568 | { 569 | "id": "BL_38ESkhXxpNJ5qgR", 570 | "type": "Block" 571 | }, 572 | { 573 | "type": "EmbeddedData" 574 | }, 575 | { 576 | "type": "Branch", 577 | "flow": [ 578 | { 579 | "type": "EmbeddedData" 580 | } 581 | ] 582 | }, 583 | { 584 | "id": "BL_6Fr1TaFUk9DuV25", 585 | "type": "Block" 586 | }, 587 | { 588 | "type": "EmbeddedData" 589 | }, 590 | { 591 | "id": "BL_etwVTHVezjxpYln", 592 | "type": "Block" 593 | }, 594 | { 595 | "type": "Branch", 596 | "flow": [ 597 | { 598 | "type": "EmbeddedData" 599 | } 600 | ] 601 | }, 602 | { 603 | "type": "Branch", 604 | "flow": [ 605 | { 606 | "type": "EmbeddedData" 607 | } 608 | ] 609 | }, 610 | { 611 | "type": "EndSurvey" 612 | } 613 | ] 614 | }, 615 | { 616 | "type": "Branch", 617 | "flow": [ 618 | { 619 | "id": "BL_8bT6RE4sg2hUB0x", 620 | "type": "Block" 621 | }, 622 | { 623 | "type": "EmbeddedData" 624 | }, 625 | { 626 | "id": "BL_1UkWf620nO474pv", 627 | "type": "Block" 628 | }, 629 | { 630 | "type": "EmbeddedData" 631 | }, 632 | { 633 | "type": "Branch", 634 | "flow": [ 635 | { 636 | "type": "EmbeddedData" 637 | } 638 | ] 639 | }, 640 | { 641 | "id": "BL_4MBmVryNzOXnVsx", 642 | "type": "Block" 643 | }, 644 | { 645 | "type": "EmbeddedData" 646 | }, 647 | { 648 | "id": "BL_78VAPu4fVL8Wznn", 649 | "type": "Block" 650 | }, 651 | { 652 | "type": "Branch", 653 | "flow": [ 654 | { 655 | "type": "EmbeddedData" 656 | } 657 | ] 658 | }, 659 | { 660 | "type": "Branch", 661 | "flow": [ 662 | { 663 | "type": "EmbeddedData" 664 | } 665 | ] 666 | }, 667 | { 668 | "type": "EndSurvey" 669 | } 670 | ] 671 | } 672 | ], 673 | "embeddedData": [ 674 | { 675 | "name": "Partition" 676 | }, 677 | { 678 | "name": "Sort" 679 | }, 680 | { 681 | "name": "LSI", 682 | "defaultValue": "QID24" 683 | }, 684 | { 685 | "name": "Language", 686 | "defaultValue": "${q://QID24/ChoiceGroup/SelectedChoices}" 687 | }, 688 | { 689 | "name": "Scale" 690 | }, 691 | { 692 | "name": "Category", 693 | "defaultValue": "Demographic" 694 | }, 695 | { 696 | "name": "Origin", 697 | "defaultValue": "2" 698 | }, 699 | { 700 | "name": "Text", 701 | "defaultValue": "${q://QID24/SelectedChoicesRecode}" 702 | }, 703 | { 704 | "name": "Sentiment" 705 | }, 706 | { 707 | "name": "Race", 708 | "defaultValue": "${q://QID135400011/SelectedChoicesRecode}" 709 | }, 710 | { 711 | "name": "Age", 712 | "defaultValue": "${q://QID135400012/SelectedChoicesRecode}" 713 | }, 714 | { 715 | "name": "Gender", 716 | "defaultValue": "${q://QID135400013/SelectedChoicesRecode}" 717 | }, 718 | { 719 | "name": "Latitude" 720 | }, 721 | { 722 | "name": "Longitude" 723 | }, 724 | { 725 | "name": "Date", 726 | "defaultValue": "${date://CurrentDate/m%2Fd%2FY}" 727 | }, 728 | { 729 | "name": "PhoneNumber" 730 | }, 731 | { 732 | "name": "IncidentId" 733 | }, 734 | { 735 | "name": "rojopolisEncounterScore", 736 | "defaultValue": "${q://QID135400005/ChoiceTextEntryValue}" 737 | }, 738 | { 739 | "name": "rojopolisGeneralScore" 740 | } 741 | ], 742 | "comments": {}, 743 | "loopAndMerge": {}, 744 | "responseCounts": { 745 | "auditable": 3, 746 | "generated": 1, 747 | "deleted": 0 748 | } 749 | }, 750 | "meta": { 751 | "httpStatus": "200 - OK", 752 | "requestId": "67c9bcaf-baeb-473c-906b-62f2295b86f1" 753 | } 754 | } 755 | -------------------------------------------------------------------------------- /app/lambda/functions/surveyjobs/tests/test_qualtrics.py: -------------------------------------------------------------------------------- 1 | import sys;sys.path.append("..") 2 | import pytest 3 | from decimal import Decimal 4 | from qualtrics import decider,QUALTRICS_MAP 5 | import math 6 | 7 | @pytest.fixture 8 | def setup_qualtrics_stub(): 9 | rec = {"rojopolisGeneralScore": 13, "Text": math.nan} 10 | return QUALTRICS_MAP, rec 11 | 12 | def test_decider(setup_qualtrics_stub): 13 | 14 | qualtrics_map, rec = setup_qualtrics_stub 15 | res = decider(rec, qualtrics_map) 16 | assert res == {"rojopolisGeneralScore": 13} 17 | -------------------------------------------------------------------------------- /app/lambda/functions/surveyjobs/tests/test_question_choices.py: -------------------------------------------------------------------------------- 1 | import sys;sys.path.append("..") 2 | import pytest 3 | from os import path 4 | from qualtrics import process_questions_from_survey 5 | import json 6 | 7 | def _file_path(file_name): 8 | basepath = path.dirname(__file__) 9 | return path.abspath(path.join(basepath, file_name)) 10 | 11 | def test_process_questions_from_survey(): 12 | aid = 'AID-BOS-dd3244' 13 | file_path = _file_path('./surveys_endpoint_response.json') 14 | with open(file_path, 'r') as opened_file: 15 | survey_data = json.load(opened_file) 16 | questions, choices = process_questions_from_survey(aid, 17 | survey_data['result']) 18 | 19 | expected_questions = [ 20 | {'Partition': aid, 21 | 'Sort': 'QID24' , 22 | 'QuestionChoicesId': 'QCID-678bc4f5c1664d4c9ed1bdf3387521f6' , 23 | 'Category': '', 24 | "Text": "Hi, this is rojopolis. We're an independent group that improves public safety with community feedback. We will share your thoughts directly to your police chief, but you will be ANONYMOUS. First, we need your consent. This will be a short conversation! Do you want to continue?"}, 25 | {'Partition': aid, 26 | 'Sort' : 'QID135400005', 27 | 'Category': '', 28 | 'Text' : 'Confirmed! Reply STOP end anytime; standard messaging rates apply. On a scale of 0-100, how would you rate the overall performance of the responding officer(s)?'}, 29 | {'Partition': aid, 30 | 'Sort' : 'QID25', 31 | 'QuestionChoicesId': 'QCID-3bf5635ba2f559823257c2e62ab3379d', 32 | 'Category': '', 33 | 'Text' : 'What is the #1 public safety issue that needs to be addressed for you to feel safer in your neighborhood?'}, 34 | {'Partition': aid, 35 | 'Sort' : 'QID26', 36 | 'Category': '', 37 | 'Text' : 'Ok, can you please explain?'}, 38 | {'Partition': aid, 39 | 'Sort' : 'QID27', 40 | 'QuestionChoicesId': 'QCID-baf58d5ea2c4c4dd8b4bb2ece082d7bd', 41 | 'Category': '', 42 | 'Text' : "Got it. Thanks. We're so sorry that you're having to go through all of this. We can put you directly in touch with services and people that can provide assistance if you need it. If you want us to do this, please select from the following list, and we'll have someone follow up with you:"}, 43 | {'Partition': aid, 44 | 'Sort' : 'QID28', 45 | 'Category': '', 46 | 'Text' : "Thanks for your feedback. You can view what others in your city and neighborhood are saying here: www.textrojopolis.com/community. If you have anything else to say, please text it now."}, 47 | {'Partition': aid, 48 | 'Sort' : 'QID29', 49 | 'Category': '', 50 | 'Text' : "Ok, someone will be in touch. Thanks for your feedback. You can view what others in your city and neighborhood are saying here: www.textrojopolis.com/community. If you have anything else to say, please text it now."}, 51 | {'Partition': aid, 52 | 'Sort' : 'QID17', 53 | 'Category': '', 54 | 'Text' : "Confirmed! Reply STOP end anytime; standard messaging rates apply. On a scale of 0-100, how would you rate the overall performance of the responding officer(s)?"}, 55 | {'Partition': aid, 56 | 'Sort' : 'QID18', 57 | 'QuestionChoicesId': 'QCID-3bf5635ba2f559823257c2e62ab3379d', 58 | 'Category': '', 59 | 'Text' : "What is the #1 public safety issue that needs to be addressed for you to feel safer in your neighborhood?"}, 60 | {'Partition': aid, 61 | 'Sort' : 'QID19', 62 | 'Category': '', 63 | 'Text' : "Ok, can you please explain?"}, 64 | {'Partition': aid, 65 | 'Sort' : 'QID30', 66 | 'QuestionChoicesId': 'QCID-baf58d5ea2c4c4dd8b4bb2ece082d7bd', 67 | 'Category': '', 68 | 'Text' : "Got it. Thanks. We're so sorry that you're having to go through all of this. We can put you directly in touch with services and people that can provide assistance if you need it. If you want us to do this, please select from the following list, and we'll have someone follow up with you:"}, 69 | {'Partition': aid, 70 | 'Sort' : 'QID31', 71 | 'Category': '', 72 | 'Text' : "Thanks for your feedback. You can view what others in your city and neighborhood are saying here: www.textrojopolis.com/community. If you have anything else to say, please text it now."}, 73 | {'Partition': aid, 74 | 'Sort' : 'QID32', 75 | 'Category': '', 76 | 'Text' : "Ok, someone will be in touch. Thanks for your feedback. You can view what others in your city and neighborhood are saying here: www.textrojopolis.com/community. If you have anything else to say, please text it now."}] 77 | expected_choices = [ 78 | { 'Sort' : 'QCID-678bc4f5c1664d4c9ed1bdf3387521f6', 79 | 'Partition': aid, 80 | 'Choices' : ('Yes, continue in English', 'Sí, continuar en Español', 'No') }, 81 | { 'Sort' : 'QCID-3bf5635ba2f559823257c2e62ab3379d', 82 | 'Partition': aid, 83 | 'Choices' : ('Property crime', 'Violent crime', 'Environmental issues (i.e. lighting, stop signs, etc.)', 'Officer behavior and trust in police', 84 | 'Other') }, 85 | { 'Sort' : 'QCID-baf58d5ea2c4c4dd8b4bb2ece082d7bd', 86 | 'Partition': aid, 87 | 'Choices' : ('Community organizations for neighborhood issues', 'Services that help people after experiencing a crime', 'Organizations that help with legal issues relating to a crimes, tickets, etc.', 'Groups that provide insurance, safety, or security around my home', 'My local government representatives office', 'No, I just need to get back to my normal routine') }, 88 | { 'Sort' : 'QCID-3bf5635ba2f559823257c2e62ab3379d', 89 | 'Partition': aid, 90 | 'Choices' : ('Property crime', 'Violent crime', 'Environmental issues (i.e. lighting, stop signs, etc.)', 'Officer behavior and trust in police', 'Other') }, 91 | { 'Sort' : 'QCID-baf58d5ea2c4c4dd8b4bb2ece082d7bd', 92 | 'Partition': aid, 93 | 'Choices' : ('Community organizations for neighborhood issues', 'Services that help people after experiencing a crime', 'Organizations that help with legal issues relating to a crimes, tickets, etc.', 'Groups that provide insurance, safety, or security around my home', 'My local government representatives office', 'No, I just need to get back to my normal routine') }, 94 | ] 95 | assert expected_questions == questions 96 | assert expected_choices == choices 97 | -------------------------------------------------------------------------------- /app/lambda/lambda_role_policy.tpl: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [{ 4 | "Effect": "Allow", 5 | "Action": [ 6 | "dynamodb:BatchGetItem", 7 | "dynamodb:GetItem", 8 | "dynamodb:Query", 9 | "dynamodb:Scan", 10 | "dynamodb:BatchWriteItem", 11 | "dynamodb:PutItem", 12 | "dynamodb:UpdateItem" 13 | ], 14 | "Resource": "arn:aws:dynamodb:${aws_region}:${aws_account_id}:*" 15 | }, 16 | { 17 | "Effect": "Allow", 18 | "Action": [ 19 | "logs:CreateLogStream", 20 | "logs:PutLogEvents" 21 | ], 22 | "Resource": "arn:aws:logs:${aws_region}:${aws_account_id}:*" 23 | }, 24 | { 25 | "Effect": "Allow", 26 | "Action": "logs:CreateLogGroup", 27 | "Resource": "*" 28 | }, 29 | { 30 | "Effect": "Allow", 31 | "Action": [ 32 | "sqs:ReceiveMessage", 33 | "sqs:DeleteMessage", 34 | "sqs:GetQueueAttributes", 35 | "sqs:GetQueueUrl", 36 | "sqs:SendMessage", 37 | "sqs:SendMessageBatch" 38 | ], 39 | "Resource": "arn:aws:sqs:${aws_region}:${aws_account_id}:*" 40 | }, 41 | { 42 | "Effect": "Allow", 43 | "Action": [ 44 | "s3:*" 45 | ], 46 | "Resource": "arn:aws:s3:::*" 47 | }, 48 | { 49 | "Effect": "Allow", 50 | 51 | "Action": [ 52 | "kms:*" 53 | ], 54 | "Resource": "*" 55 | }, 56 | { 57 | "Effect": "Allow", 58 | "Action": [ 59 | "comprehend:*" 60 | ], 61 | "Resource": "*" 62 | } 63 | ] 64 | } -------------------------------------------------------------------------------- /app/lambda/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.12.0" 3 | backend "s3" { 4 | bucket = "rojopolis-tf" 5 | key = "rojopolis-lambda" 6 | region = "us-east-1" 7 | dynamodb_table = "rojopolis-terraform-lock" 8 | } 9 | } 10 | 11 | provider "aws" { 12 | version = "~> 2.7" 13 | region = "us-east-1" 14 | } 15 | 16 | data "aws_caller_identity" "current" {} 17 | data "aws_region" "current" {} 18 | 19 | locals { 20 | aws_account_id = data.aws_caller_identity.current.account_id 21 | aws_region = data.aws_region.current.name 22 | lambda_base_dir = "${path.module}/functions" 23 | environment_slug = "${lower(terraform.workspace)}" 24 | } 25 | 26 | data "terraform_remote_state" "dynamodb" { 27 | backend = "s3" 28 | workspace = local.environment_slug 29 | config = { 30 | bucket = "rojopolis-tf" 31 | key = "rojopolis-dynamodb" 32 | region = "us-east-1" 33 | dynamodb_table = "rojopolis-terraform-lock" 34 | } 35 | } 36 | 37 | data "terraform_remote_state" "sqs" { 38 | backend = "s3" 39 | workspace = local.environment_slug 40 | config = { 41 | bucket = "rojopolis-tf" 42 | key = "rojopolis-api-sqs" 43 | region = "us-east-1" 44 | dynamodb_table = "rojopolis-terraform-lock" 45 | } 46 | } 47 | 48 | data "terraform_remote_state" "kms" { 49 | backend = "s3" 50 | workspace = local.environment_slug 51 | config = { 52 | bucket = "rojopolis-tf" 53 | key = "rojopolis-kms" 54 | region = "us-east-1" 55 | dynamodb_table = "rojopolis-terraform-lock" 56 | } 57 | } 58 | 59 | resource "aws_s3_bucket" "rojopolis_lambda_bucket" { 60 | bucket = "rojopolis-lambda-${local.aws_region}-${local.aws_account_id}" 61 | acl = "private" 62 | 63 | versioning { 64 | enabled = true 65 | } 66 | } 67 | 68 | #------------------------------------------------------------------------------- 69 | #-- CRUD Handler 70 | #------------------------------------------------------------------------------- 71 | data "template_file" "lambda_role_policy_template" { 72 | template = "${file("${path.module}/lambda_role_policy.tpl")}" 73 | vars = { 74 | aws_account_id = "${local.aws_account_id}" 75 | aws_region = "${local.aws_region}" 76 | } 77 | } 78 | 79 | resource "aws_iam_role_policy" "lambda_role_policy" { 80 | name = "lambda_role_policy" 81 | role = "${aws_iam_role.crud_lambda_role.id}" 82 | 83 | policy = "${data.template_file.lambda_role_policy_template.rendered}" 84 | } 85 | 86 | resource "aws_iam_role" "crud_lambda_role" { 87 | name = "crud_lambda_role-${local.environment_slug}" 88 | assume_role_policy = <Show Output 9 | 10 | \`\`\`diff 11 | $1 12 | \`\`\` 13 | 14 | 15 | " 16 | else 17 | echo " 18 | \`\`\`diff 19 | $1 20 | \`\`\` 21 | " 22 | fi 23 | } 24 | 25 | set -e 26 | 27 | cd "${TF_ACTION_WORKING_DIR:-.}" 28 | 29 | if [[ ! -z "$TF_ACTION_TFE_TOKEN" ]]; then 30 | cat > ~/.terraformrc << EOF 31 | credentials "${TF_ACTION_TFE_HOSTNAME:-app.terraform.io}" { 32 | token = "$TF_ACTION_TFE_TOKEN" 33 | } 34 | EOF 35 | fi 36 | 37 | if [[ ! -z "$TF_ACTION_WORKSPACE" ]] && [[ "$TF_ACTION_WORKSPACE" != "default" ]]; then 38 | terraform workspace select "$TF_ACTION_WORKSPACE" 39 | fi 40 | 41 | set +e 42 | OUTPUT=$(sh -c "TF_IN_AUTOMATION=true terraform plan -no-color -input=false $*" 2>&1) 43 | SUCCESS=$? 44 | echo "$OUTPUT" 45 | set -e 46 | 47 | if [ "$TF_ACTION_COMMENT" = "1" ] || [ "$TF_ACTION_COMMENT" = "false" ]; then 48 | exit $SUCCESS 49 | fi 50 | 51 | # Build the comment we'll post to the PR. 52 | COMMENT="" 53 | if [ $SUCCESS -ne 0 ]; then 54 | OUTPUT=$(wrap "$OUTPUT") 55 | COMMENT="#### \`terraform plan\` Failed 56 | $OUTPUT 57 | 58 | *Workflow: \`$GITHUB_WORKFLOW\`, Action: \`$GITHUB_ACTION\`*" 59 | else 60 | # Remove "Refreshing state..." lines by only keeping output after the 61 | # delimiter (72 dashes) that represents the end of the refresh stage. 62 | # We do this to keep the comment output smaller. 63 | if echo "$OUTPUT" | egrep '^-{72}$'; then 64 | OUTPUT=$(echo "$OUTPUT" | sed -n -r '/-{72}/,/-{72}/{ /-{72}/d; p }') 65 | fi 66 | 67 | # Remove whitespace at the beginning of the line for added/modified/deleted 68 | # resources so the diff markdown formatting highlights those lines. 69 | OUTPUT=$(echo "$OUTPUT" | sed -r -e 's/^ \+/\+/g' | sed -r -e 's/^ ~/~/g' | sed -r -e 's/^ -/-/g') 70 | 71 | # Call wrap to optionally wrap our output in a collapsible markdown section. 72 | OUTPUT=$(wrap "$OUTPUT") 73 | 74 | COMMENT="#### \`terraform plan\` Success 75 | $OUTPUT 76 | 77 | *Workflow: \`$GITHUB_WORKFLOW\`, Action: \`$GITHUB_ACTION\`*" 78 | fi 79 | 80 | # Post the comment. 81 | PAYLOAD=$(echo '{}' | jq --arg body "$COMMENT" '.body = $body') 82 | COMMENTS_URL=$(cat /github/workflow/event.json | jq -r .pull_request.comments_url) 83 | curl -s -S -H "Authorization: token $GITHUB_TOKEN" --header "Content-Type: application/json" --data "$PAYLOAD" "$COMMENTS_URL" > /dev/null 84 | 85 | exit $SUCCESS -------------------------------------------------------------------------------- /collaboration/backend_aws.hcl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/collaboration/backend_aws.hcl -------------------------------------------------------------------------------- /collaboration/backend_remote.hcl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/collaboration/backend_remote.hcl -------------------------------------------------------------------------------- /collaboration/inputs.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/collaboration/inputs.tf -------------------------------------------------------------------------------- /collaboration/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = "~> 0.12.0" 3 | backend "remote"{ 4 | hostname = "app.terraform.io" 5 | organization = "rojopolis" 6 | workspaces { 7 | name = "collaboration" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /collaboration/outputs.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/collaboration/outputs.tf -------------------------------------------------------------------------------- /modules/terraform-google-ip-range-datasource/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/modules/terraform-google-ip-range-datasource/README.md -------------------------------------------------------------------------------- /modules/terraform-google-ip-range-datasource/examples/example_1/main.tf: -------------------------------------------------------------------------------- 1 | module "google-cloud-ip-range" { 2 | source = "../" 3 | } 4 | 5 | output "gcp-ipv4-ranges" { 6 | value = module.google-cloud-ip-range.ip_v4_cidrs 7 | } 8 | 9 | output "gcp-ipv6-ranges" { 10 | value = module.google-cloud-ip-range.ip_v6_cidrs 11 | } 12 | 13 | output "gcp-ranges" { 14 | value = module.google-cloud-ip-range.cidrs 15 | } -------------------------------------------------------------------------------- /modules/terraform-google-ip-range-datasource/main.tf: -------------------------------------------------------------------------------- 1 | data "external" "google_cloud_cidrs" { 2 | program = [ 3 | "python3", 4 | "${path.module}/scripts/datasource.py", 5 | ] 6 | } 7 | 8 | locals { 9 | ip_v4_cidrs = distinct(compact(split(" ", data.external.google_cloud_cidrs.result["ipv4Cidrs"]))) 10 | ip_v6_cidrs = distinct(compact(split(" ", data.external.google_cloud_cidrs.result["ipv6Cidrs"]))) 11 | cidrs = concat(local.ip_v4_cidrs, local.ip_v6_cidrs) 12 | } -------------------------------------------------------------------------------- /modules/terraform-google-ip-range-datasource/outputs.tf: -------------------------------------------------------------------------------- 1 | output "ip_v4_cidrs" { 2 | description = "List of CIDR blocks of IPv4 addresses." 3 | value = local.ip_v4_cidrs 4 | } 5 | 6 | output "ip_v6_cidrs" { 7 | description = "List of CIDR blocks of IPv6 addresses." 8 | value = local.ip_v6_cidrs 9 | } 10 | 11 | output "cidrs" { 12 | description = "List of CIDR blocks (IPv4 & IPv6)" 13 | value = local.cidrs 14 | } -------------------------------------------------------------------------------- /modules/terraform-google-ip-range-datasource/scripts/__pycache__/datasource.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/modules/terraform-google-ip-range-datasource/scripts/__pycache__/datasource.cpython-37.pyc -------------------------------------------------------------------------------- /modules/terraform-google-ip-range-datasource/scripts/datasource.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Generate IP Whitelist for published GCP ranges 3 | https://cloud.google.com/compute/docs/faq#find_ip_range 4 | ''' 5 | 6 | import json 7 | import logging 8 | import dns.resolver 9 | 10 | logging.basicConfig(level='ERROR') 11 | 12 | def get_netblock(record, cidrs): 13 | ''' 14 | Recurse through netblocks and append CIDRS 15 | ''' 16 | includes = [] 17 | answers = dns.resolver.query(record, 'TXT') 18 | logging.debug(f'answers={answers}') 19 | for rdata in answers: 20 | logging.debug(f'rdata={rdata}') 21 | logging.debug(f'type={type(rdata)}') 22 | includes = [x.split('include:')[1] for x in str(rdata).split(' ') if x.startswith('include:')] 23 | cidrs.extend([x for x in str(rdata).split(' ') if x.startswith('ip')]) 24 | logging.debug(f'includes={includes}') 25 | logging.debug(f'cidrs={cidrs}') 26 | for include in includes: 27 | get_netblock(include, cidrs) 28 | 29 | def stringify_cidrs(cidr_list, ip_ver='4'): 30 | ''' 31 | Terraform only supports strings for external data sources, 32 | so flatten lists inst space separated strings 33 | ''' 34 | cidr_string = '' 35 | for cidr in [x.split(f'ip{ip_ver}:')[1] for x in cidrs if x.startswith(f'ip{ip_ver}')]: 36 | cidr_string += cidr + ' ' 37 | return cidr_string 38 | 39 | if __name__ == '__main__': 40 | cidrs = [] 41 | get_netblock('_cloud-netblocks.googleusercontent.com', cidrs) 42 | print(json.dumps( 43 | { 44 | 'ipv4Cidrs': stringify_cidrs(cidrs, '4'), 45 | 'ipv6Cidrs': stringify_cidrs(cidrs, '6') 46 | } 47 | ) 48 | ) -------------------------------------------------------------------------------- /modules/terraform-google-ip-range-datasource/scripts/requirements.txt: -------------------------------------------------------------------------------- 1 | dnspython==1.16.0 -------------------------------------------------------------------------------- /modules/terraform-google-ip-range-datasource/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/modules/terraform-google-ip-range-datasource/variables.tf -------------------------------------------------------------------------------- /modules/trivial/examples/main.tf: -------------------------------------------------------------------------------- 1 | module "local-module" { 2 | source = "../" 3 | string_param = "foo" 4 | } 5 | 6 | output "module_output" { 7 | description = "The output from a module" 8 | value = module.local-module.string_output 9 | } 10 | -------------------------------------------------------------------------------- /modules/trivial/main.tf: -------------------------------------------------------------------------------- 1 | variable "string_param" { 2 | type = "string" 3 | description = "A string" 4 | default = "biz" 5 | } 6 | output "string_output" { 7 | description = "The value of string_param" 8 | value = var.string_param 9 | } 10 | -------------------------------------------------------------------------------- /modules/trivial/outputs.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/modules/trivial/outputs.tf -------------------------------------------------------------------------------- /modules/trivial/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/modules/trivial/variables.tf -------------------------------------------------------------------------------- /slides/Take Terraform To The Next Level.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rojopolis/Take-Terraform-to-the-Next-Level/cfc9d92e77e012a7af4ade8ec93434dd6243267b/slides/Take Terraform To The Next Level.pptx --------------------------------------------------------------------------------