├── .gitignore ├── .vscode └── settings.json ├── A_D_Reports ├── build.sbt ├── hmda-spark-reporting │ ├── Jenkinsfile │ ├── README.md │ ├── build.sbt │ ├── spark-pi.yaml │ └── src │ │ └── main │ │ ├── resources │ │ └── application.conf │ │ └── scala │ │ └── com │ │ └── hmda │ │ └── reports │ │ ├── AggregateReports.scala │ │ ├── DisclosureReports.scala │ │ ├── model │ │ ├── AggregateModel.scala │ │ └── DisclosureModel.scala │ │ └── processing │ │ ├── AggregateProcessing.scala │ │ ├── BaseProcessing.scala │ │ ├── DisclosureProcessing.scala │ │ ├── IncomeRaceEthnicityProcessing.scala │ │ ├── MedianAgeProcessing.scala │ │ └── RaceGenderProcessing.scala └── kubernetes │ └── hmda-spark-reporting │ ├── hmda-spark-aggregate.yaml │ ├── hmda-spark-disclosure-scheduled.yaml │ ├── hmda-spark-disclosure.yaml │ ├── project │ └── build.properties │ └── spark-pi.yaml ├── LICENSE ├── Pipfile ├── TERMS.md ├── analysis_examples ├── 1. Plotting a Time Series of HMDA Filers from 2004-2017.ipynb ├── 1. Plotting a Time Series of HMDA Filers from 2004-2017.py ├── 2. Plotting a Time Series of HMDA Filers by Category.ipynb ├── 2. Plotting a Time Series of HMDA Filers by Category.py ├── HMDA_Panel_ID_Mapping_2017_2018.ipynb ├── analysis_output │ ├── charts │ │ ├── 1_filer_count_time_series.png │ │ └── 2_filer_count_by_category.png │ └── tables │ │ ├── 1_filer_count_time_series.txt │ │ └── 2_filer_count_by_category.txt └── sql_commands │ ├── 1_filer_count_time_series.sql │ └── 2_filer_count_by_category.sql ├── census └── readme.me ├── documentation_resources ├── AboutModifiedLAR.md ├── ModifiedLarWithExcel.md ├── code_sheets │ └── modified_LAR │ │ └── modified_LAR_2018.csv ├── example_images │ └── mlar_tutorial_images │ │ ├── Step2.JPG │ │ ├── Step3.JPG │ │ ├── Step4.JPG │ │ ├── Step5.JPG │ │ ├── Step6.JPG │ │ ├── Step8.JPG │ │ ├── Step9.JPG │ │ ├── Step_7_1.JPG │ │ └── Step_7_2.JPG └── schemas │ ├── lar │ ├── lar_2004_2013.csv │ ├── lar_2004_2013.md │ ├── lar_2014_2016.csv │ ├── lar_2014_2016.md │ ├── lar_2017.csv │ ├── lar_2017.md │ ├── lar_2018_2019.csv │ └── lar_2018_2019.md │ ├── mlar │ ├── mlar_header_2017.csv │ ├── mlar_header_2018.csv │ ├── mlar_schema_2017.csv │ └── mlar_schema_2018_2019.csv │ ├── panel │ ├── panel_2004_2013.csv │ ├── panel_2004_2013.md │ ├── panel_2014_2016.csv │ ├── panel_2014_2016.md │ ├── panel_2017.csv │ ├── panel_2017.md │ ├── panel_2018_2019.csv │ └── panel_2018_2019.md │ ├── schema_source_links.md │ └── ts │ ├── ts_2014_2016.csv │ ├── ts_2014_2016.md │ ├── ts_2017.csv │ ├── ts_2017.md │ ├── ts_2018_2019.csv │ └── ts_2018_2019.md ├── download_scripts ├── download_hmda.sh ├── unzip_all.sh ├── unzip_and_rename_lar.sh ├── unzip_panel.sh └── unzip_ts.sh ├── federal_pubs.md ├── hmda_data_links.md ├── load_scripts ├── SQL │ ├── create_and_load_lar_2004.sql │ ├── create_and_load_lar_2005.sql │ ├── create_and_load_lar_2006.sql │ ├── create_and_load_lar_2007.sql │ ├── create_and_load_lar_2008.sql │ ├── create_and_load_lar_2009.sql │ ├── create_and_load_lar_2010.sql │ ├── create_and_load_lar_2011.sql │ ├── create_and_load_lar_2012.sql │ ├── create_and_load_lar_2013.sql │ ├── create_and_load_lar_2014.sql │ ├── create_and_load_lar_2015.sql │ ├── create_and_load_lar_2016.sql │ ├── create_and_load_lar_2017.sql │ ├── create_and_load_lar_2018.sql │ ├── create_and_load_lar_2019.sql │ ├── create_and_load_lar_2020.sql │ ├── create_and_load_lar_2021.sql │ ├── create_and_load_lar_2022.sql │ ├── create_and_load_modified_lar_2017.sql │ ├── create_and_load_modified_lar_2018.sql │ ├── create_and_load_modified_lar_2019.sql │ ├── create_and_load_panel_2004.sql │ ├── create_and_load_panel_2005.sql │ ├── create_and_load_panel_2006.sql │ ├── create_and_load_panel_2007.sql │ ├── create_and_load_panel_2008.sql │ ├── create_and_load_panel_2009.sql │ ├── create_and_load_panel_2010.sql │ ├── create_and_load_panel_2011.sql │ ├── create_and_load_panel_2012.sql │ ├── create_and_load_panel_2013.sql │ ├── create_and_load_panel_2014.sql │ ├── create_and_load_panel_2015.sql │ ├── create_and_load_panel_2016.sql │ ├── create_and_load_panel_2017.sql │ ├── create_and_load_panel_2018.sql │ ├── create_and_load_panel_2019.sql │ ├── create_and_load_panel_2020.sql │ ├── create_and_load_panel_2021.sql │ ├── create_and_load_panel_2022.sql │ ├── create_and_load_ts_2004.sql │ ├── create_and_load_ts_2005.sql │ ├── create_and_load_ts_2006.sql │ ├── create_and_load_ts_2007.sql │ ├── create_and_load_ts_2008.sql │ ├── create_and_load_ts_2009.sql │ ├── create_and_load_ts_2010.sql │ ├── create_and_load_ts_2011.sql │ ├── create_and_load_ts_2012.sql │ ├── create_and_load_ts_2013.sql │ ├── create_and_load_ts_2014.sql │ ├── create_and_load_ts_2015.sql │ ├── create_and_load_ts_2016.sql │ ├── create_and_load_ts_2017.sql │ ├── create_and_load_ts_2018.sql │ ├── create_and_load_ts_2019.sql │ ├── create_and_load_ts_2020.sql │ ├── create_and_load_ts_2021.sql │ ├── create_and_load_ts_2022.sql │ └── create_hmda_db.sql ├── create_and_load_hmda.sh ├── create_and_load_lar_2004_2017.sh ├── create_and_load_lar_2004_2019.sh ├── create_and_load_lar_2004_2020.sh ├── create_and_load_lar_2004_2022.sh ├── create_and_load_panel_2004_2017.sh ├── create_and_load_panel_2004_2019.sh ├── create_and_load_panel_2004_2020.sh ├── create_and_load_panel_2004_2022.sh ├── create_and_load_ts_2004_2017.sh ├── create_and_load_ts_2004_2019.sh ├── create_and_load_ts_2004_2020.sh ├── create_and_load_ts_2004_2022.sh ├── create_hmda_db.sh ├── remove_tab_ts_2016.py ├── reset_path.py ├── write_data_paths.py └── write_data_paths.sh ├── readme.md └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | documentation_resources/.DS_Store 3 | documentation_resources/schemas/.DS_Store 4 | load_scripts/.DS_Store 5 | 6 | data/ 7 | 8 | # Byte-compiled / optimized / DLL files 9 | __pycache__/ 10 | *.py[cod] 11 | *$py.class 12 | 13 | # C extensions 14 | *.so 15 | 16 | # Distribution / packaging 17 | .Python 18 | build/ 19 | develop-eggs/ 20 | dist/ 21 | downloads/ 22 | eggs/ 23 | .eggs/ 24 | lib/ 25 | lib64/ 26 | parts/ 27 | sdist/ 28 | var/ 29 | wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | MANIFEST 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .coverage 49 | .coverage.* 50 | .cache 51 | nosetests.xml 52 | coverage.xml 53 | *.cover 54 | .hypothesis/ 55 | .pytest_cache/ 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | local_settings.py 64 | db.sqlite3 65 | 66 | # Flask stuff: 67 | instance/ 68 | .webassets-cache 69 | 70 | # Scrapy stuff: 71 | .scrapy 72 | 73 | # Sphinx documentation 74 | docs/_build/ 75 | 76 | # PyBuilder 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # celery beat schedule file 86 | celerybeat-schedule 87 | 88 | # SageMath parsed files 89 | *.sage.py 90 | 91 | # Environments 92 | .env 93 | .venv 94 | env/ 95 | venv/ 96 | ENV/ 97 | env.bak/ 98 | venv.bak/ 99 | 100 | # Spyder project settings 101 | .spyderproject 102 | .spyproject 103 | 104 | # Rope project settings 105 | .ropeproject 106 | 107 | # mkdocs documentation 108 | /site 109 | 110 | # mypy 111 | .mypy_cache/ 112 | 113 | 114 | #Python checkpoints 115 | .ipynb_checkpoints/ 116 | python/DS_Store 117 | 118 | #ipbny checkpoints 119 | .ipynb_checkpoints 120 | Python/.ipynb_checkpoints 121 | .ipynb 122 | 123 | Untitled.ipynb 124 | 125 | *.txt -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.renderWhitespace": "all" 3 | } -------------------------------------------------------------------------------- /A_D_Reports/hmda-spark-reporting/Jenkinsfile: -------------------------------------------------------------------------------- 1 | pipeline { 2 | agent { 3 | label 'hmdaops' 4 | } 5 | 6 | stages { 7 | stage('init') { 8 | steps { 9 | script { 10 | library identifier: "hmdaUtils@master", retriever: modernSCM ( 11 | [ 12 | $class: 'GitSCMSource', 13 | remote: 'https://github.cfpb.gov/HMDA-Operations/hmda-devops.git' 14 | ] 15 | ) 16 | 17 | init.setEnvironment('hmda_spark_reporting') 18 | init.checkForChanges('hmda-spark-reporting') 19 | } 20 | } 21 | } 22 | 23 | stage('Build Scala Code and Generate Dockerfile') { 24 | agent { 25 | docker { 26 | image 'hmda/sbt' 27 | reuseNode true 28 | } 29 | } 30 | steps { 31 | sh "sbt -Dsbt.global.base=.sbt -Dsbt.boot.directory=.sbt -Dsbt.ivy.home=.ivy2 -batch clean hmda-spark-reporting/docker:stage" 32 | } 33 | } 34 | 35 | stage('Build And Publish Docker Image') { 36 | steps { 37 | script { 38 | withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'dockerhub', 39 | usernameVariable: 'DOCKER_HUB_USER', passwordVariable: 'DOCKER_HUB_PASSWORD']]) { 40 | withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'hmda-platform-jenkins-service', 41 | usernameVariable: 'DTR_USER', passwordVariable: 'DTR_PASSWORD']]) { 42 | withCredentials([string(credentialsId: 'internal-docker-registry', variable: 'DOCKER_REGISTRY_URL')]){ 43 | dockerBuild.sbtBuild('hmda-spark-reporting') 44 | scanImage('hmda-spark-reporting', env.DOCKER_TAG) 45 | } 46 | } 47 | } 48 | } 49 | } 50 | } 51 | 52 | stage('Deploy') { 53 | agent { 54 | docker { 55 | image 'hmda/helm' 56 | reuseNode true 57 | args '--entrypoint=\'\'' 58 | } 59 | } 60 | steps { 61 | script { 62 | withCredentials([file(credentialsId: 'hmda-ops-kubeconfig', variable: 'KUBECONFIG')]) { 63 | if (env.DOCKER_PUSH == 'true') { 64 | helm.deploy('hmda-spark-reporting') 65 | } 66 | } 67 | } 68 | } 69 | } 70 | 71 | stage('Docker Clean Up') { 72 | steps { 73 | script { 74 | dockerBuild.clean() 75 | } 76 | } 77 | } 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /A_D_Reports/hmda-spark-reporting/build.sbt: -------------------------------------------------------------------------------- 1 | import sbtdocker.Instructions.User 2 | 3 | packageName in Docker := "hmda-spark-reporting" 4 | 5 | version := "0.1" 6 | 7 | dockerfile in docker := { 8 | // The assembly task generates a fat JAR file 9 | val artifact: File = assembly.value 10 | val artifactTargetPath = s"/opt/spark/cfpb/hmda/jars/${name.value}.jar" 11 | 12 | new Dockerfile { 13 | from("lightbend/spark:2.1.0-OpenShift-2.4.0-rh-2.12") //Uses 2.12 Scala 14 | User("jboss") 15 | run( 16 | "rm", 17 | "/opt/spark/jars/kubernetes-client-3.0.0.jar" 18 | ) 19 | run( 20 | "curl", 21 | "https://repo1.maven.org/maven2/io/fabric8/kubernetes-client/4.4.2/kubernetes-client-4.4.2.jar", 22 | "--output", 23 | "/opt/spark/jars/kubernetes-client-4.4.2.jar", 24 | "--silent" 25 | ) 26 | run( 27 | "curl", 28 | "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/2.7.3/hadoop-aws-2.7.3.jar", 29 | "--output", 30 | "/opt/spark/jars/hadoop-aws-2.7.3.jar", 31 | "--silent" 32 | ) 33 | run( 34 | "curl", 35 | "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk/1.7.4/aws-java-sdk-1.7.4.jar", 36 | "--output", 37 | "/opt/spark/jars/aws-java-sdk-1.7.4.jar", 38 | "--silent" 39 | ) 40 | run("mkdir", "-p", "/opt/spark/cfpb/hmda/jars/") 41 | 42 | add(artifact, artifactTargetPath) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /A_D_Reports/hmda-spark-reporting/spark-pi.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2017 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | apiVersion: "sparkoperator.k8s.io/v1beta1" 17 | kind: SparkApplication 18 | metadata: 19 | name: spark-pi 20 | namespace: default 21 | spec: 22 | type: Scala 23 | mode: cluster 24 | image: "gcr.io/spark-operator/spark:v2.4.4" 25 | imagePullPolicy: Always 26 | mainClass: org.apache.spark.examples.SparkPi 27 | mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.11-2.4.4.jar" 28 | sparkVersion: "2.4.4" 29 | restartPolicy: 30 | type: Never 31 | volumes: 32 | - name: "test-volume" 33 | hostPath: 34 | path: "/tmp" 35 | type: Directory 36 | driver: 37 | cores: 1 38 | coreLimit: "1200m" 39 | memory: "512m" 40 | labels: 41 | version: 2.4.4 42 | serviceAccount: spark-operator-spark 43 | volumeMounts: 44 | - name: "test-volume" 45 | mountPath: "/tmp" 46 | executor: 47 | cores: 1 48 | instances: 1 49 | memory: "512m" 50 | labels: 51 | version: 2.4.4 52 | volumeMounts: 53 | - name: "test-volume" 54 | mountPath: "/tmp" 55 | -------------------------------------------------------------------------------- /A_D_Reports/hmda-spark-reporting/src/main/resources/application.conf: -------------------------------------------------------------------------------- 1 | akka { 2 | log-dead-letters = 10 3 | log-dead-letters-during-shutdown = on 4 | } -------------------------------------------------------------------------------- /A_D_Reports/kubernetes/hmda-spark-reporting/hmda-spark-aggregate.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "sparkoperator.k8s.io/v1beta1" 2 | kind: SparkApplication 3 | metadata: 4 | name: hmda-spark-aggregate 5 | namespace: default 6 | spec: 7 | type: Scala 8 | mode: cluster 9 | sparkConf: 10 | "spark.streaming.receiver.maxRate": "1" 11 | "spark.streaming.receiver.maxRate": "1" 12 | "spark.streaming.backpressure.initialRate": "1" 13 | "spark.streaming.backpressure.enabled": "true" 14 | image: "hmda/hmda-spark-reporting" 15 | imagePullPolicy: Always 16 | mainClass: com.hmda.reports.AggregateReports 17 | mainApplicationFile: "local:///opt/spark/cfpb/hmda/jars/hmda-spark-reporting.jar" 18 | sparkVersion: "2.4.0" 19 | restartPolicy: 20 | type: Never 21 | # volumes: 22 | # - name: "test-volume" 23 | # hostPath: 24 | # path: "/tmp" 25 | # type: Directory 26 | driver: 27 | cores: 1 28 | coreLimit: "4000m" 29 | memory: "4g" 30 | javaOptions: "-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap" 31 | labels: 32 | version: 2.4.0 33 | serviceAccount: spark-operator-spark 34 | configMaps: 35 | - name: kafka-configmap 36 | path: /mnt/kafka-config-maps 37 | envSecretKeyRefs: 38 | JDBC_URL: 39 | name: inst-postgres-credentials 40 | key: url 41 | ACCESS_KEY: 42 | name: aws-credentials 43 | key: aws-access-key-id 44 | SECRET_KEY: 45 | name: aws-credentials 46 | key: aws-secret-access-key 47 | KAFKA_HOSTS: 48 | name: kafka-hosts 49 | key: kafka-hosts 50 | AWS_ENV: 51 | name: aws-env 52 | key: aws-env 53 | volumeMounts: 54 | - name: "test-volume" 55 | mountPath: "/tmp" 56 | executor: 57 | cores: 2 58 | instances: 2 59 | memory: "8g" 60 | javaOptions: "-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap" 61 | labels: 62 | version: 2.4.0 63 | configMaps: 64 | - name: kafka-configmap 65 | path: /mnt/kafka-config-maps 66 | envSecretKeyRefs: 67 | JDBC_URL: 68 | name: inst-postgres-credentials 69 | key: url 70 | ACCESS_KEY: 71 | name: aws-credentials 72 | key: aws-access-key-id 73 | SECRET_KEY: 74 | name: aws-credentials 75 | key: aws-secret-access-key 76 | KAFKA_HOSTS: 77 | name: kafka-hosts 78 | key: kafka-hosts 79 | AWS_ENV: 80 | name: aws-env 81 | key: aws-env 82 | # volumeMounts: 83 | # - name: "test-volume" 84 | # mountPath: "/tmp" 85 | # volumes: 86 | # - name: config-vol 87 | # configMap: 88 | # name: dummy-cm 89 | -------------------------------------------------------------------------------- /A_D_Reports/kubernetes/hmda-spark-reporting/hmda-spark-disclosure-scheduled.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "sparkoperator.k8s.io/v1beta1" 2 | kind: ScheduledSparkApplication 3 | metadata: 4 | name: disclosure-scheduled 5 | namespace: default 6 | spec: 7 | schedule: "@every 30m" 8 | concurrencyPolicy: Allow 9 | successfulRunHistoryLimit: 1 10 | failedRunHistoryLimit: 3 11 | template: 12 | type: Scala 13 | mode: cluster 14 | image: "hmda/hmda-spark-reporting" 15 | imagePullPolicy: Always 16 | mainClass: com.hmda.reports.DisclosureReports 17 | mainApplicationFile: "local:///opt/spark/cfpb/hmda/jars/hmda-spark-reporting.jar" 18 | sparkVersion: "2.4.0" 19 | restartPolicy: 20 | type: Never 21 | driver: 22 | cores: 2 23 | coreLimit: "4000m" 24 | memory: "3g" 25 | labels: 26 | version: 2.4.0 27 | serviceAccount: spark-operator-spark 28 | configMaps: 29 | - name: kafka-configmap 30 | path: /mnt/kafka-config-maps 31 | envSecretKeyRefs: 32 | JDBC_URL: 33 | name: inst-postgres-credentials 34 | key: url 35 | ACCESS_KEY: 36 | name: aws-credentials 37 | key: aws-access-key-id 38 | SECRET_KEY: 39 | name: aws-credentials 40 | key: aws-secret-access-key 41 | KAFKA_HOSTS: 42 | name: kafka-hosts 43 | key: kafka-hosts 44 | AWS_ENV: 45 | name: aws-env 46 | key: aws-env 47 | executor: 48 | cores: 2 49 | instances: 2 50 | memory: "6g" 51 | labels: 52 | version: 2.4.0 53 | configMaps: 54 | - name: kafka-configmap 55 | path: /mnt/kafka-config-maps 56 | envSecretKeyRefs: 57 | JDBC_URL: 58 | name: inst-postgres-credentials 59 | key: url 60 | ACCESS_KEY: 61 | name: aws-credentials 62 | key: aws-access-key-id 63 | SECRET_KEY: 64 | name: aws-credentials 65 | key: aws-secret-access-key 66 | AWS_ENV: 67 | name: aws-env 68 | key: aws-env 69 | KAFKA_HOSTS: 70 | name: kafka-hosts 71 | key: kafka-hosts -------------------------------------------------------------------------------- /A_D_Reports/kubernetes/hmda-spark-reporting/hmda-spark-disclosure.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "sparkoperator.k8s.io/v1beta1" 2 | kind: SparkApplication 3 | metadata: 4 | name: hmda-spark-disclosure 5 | namespace: default 6 | spec: 7 | type: Scala 8 | mode: cluster 9 | image: "hmda/hmda-spark-reporting:newk8" 10 | imagePullPolicy: Always 11 | mainClass: com.hmda.reports.DisclosureReports 12 | mainApplicationFile: "local:///opt/spark/cfpb/hmda/jars/hmda-spark-reporting.jar" 13 | sparkVersion: "2.4.5" 14 | restartPolicy: 15 | type: Never 16 | # volumes: 17 | # - name: "test-volume" 18 | # hostPath: 19 | # path: "/tmp" 20 | # type: Directory 21 | driver: 22 | cores: 2 23 | coreLimit: "4000m" 24 | memory: "2g" 25 | labels: 26 | version: 2.4.0 27 | # affinity: 28 | # podAffinity: 29 | # requiredDuringSchedulingIgnoredDuringExecution: 30 | # - labelSelector: 31 | # matchExpressions: 32 | # - key: app 33 | # operator: In 34 | # values: 35 | # - "hmda-platform" 36 | # - "keycloak" 37 | # topologyKey: kubernetes.io/hostname 38 | serviceAccount: spark-operator-spark 39 | configMaps: 40 | - name: kafka-configmap 41 | path: /mnt/kafka-config-maps 42 | envSecretKeyRefs: 43 | JDBC_URL: 44 | name: inst-postgres-credentials 45 | key: url 46 | ACCESS_KEY: 47 | name: aws-credentials 48 | key: aws-access-key-id 49 | SECRET_KEY: 50 | name: aws-credentials 51 | key: aws-secret-access-key 52 | KAFKA_HOSTS: 53 | name: kafka-hosts 54 | key: kafka-hosts 55 | AWS_ENV: 56 | name: aws-env 57 | key: aws-env 58 | # volumeMounts: 59 | # - name: "test-volume" 60 | # mountPath: "/tmp" 61 | executor: 62 | cores: 2 63 | instances: 1 64 | memory: "2g" 65 | labels: 66 | version: 2.4.0 67 | # affinity: 68 | # podAntiAffinity: 69 | # requiredDuringSchedulingIgnoredDuringExecution: 70 | # - labelSelector: 71 | # matchExpressions: 72 | # - key: app 73 | # operator: In 74 | # values: 75 | # - "hmda-platform" 76 | # - "keycloak" 77 | # topologyKey: kubernetes.io/hostname 78 | configMaps: 79 | - name: kafka-configmap 80 | path: /mnt/kafka-config-maps 81 | envSecretKeyRefs: 82 | JDBC_URL: 83 | name: inst-postgres-credentials 84 | key: url 85 | ACCESS_KEY: 86 | name: aws-credentials 87 | key: aws-access-key-id 88 | SECRET_KEY: 89 | name: aws-credentials 90 | key: aws-secret-access-key 91 | AWS_ENV: 92 | name: aws-env 93 | key: aws-env 94 | KAFKA_HOSTS: 95 | name: kafka-hosts 96 | key: kafka-hosts 97 | # volumeMounts: 98 | # - name: "test-volume" 99 | # mountPath: "/tmp" 100 | # volumes: 101 | # - name: config-vol 102 | # configMap: 103 | # name: dummy-cm -------------------------------------------------------------------------------- /A_D_Reports/kubernetes/hmda-spark-reporting/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.2.6 2 | -------------------------------------------------------------------------------- /A_D_Reports/kubernetes/hmda-spark-reporting/spark-pi.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2017 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | apiVersion: "sparkoperator.k8s.io/v1beta2" 17 | kind: SparkApplication 18 | metadata: 19 | name: spark-pi 20 | namespace: default 21 | spec: 22 | type: Scala 23 | mode: cluster 24 | image: "gcr.io/spark-operator/spark:v2.4.4" 25 | imagePullPolicy: Always 26 | mainClass: org.apache.spark.examples.SparkPi 27 | mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.11-2.4.4.jar" 28 | sparkVersion: "2.4.4" 29 | restartPolicy: 30 | type: Never 31 | volumes: 32 | - name: "test-volume" 33 | hostPath: 34 | path: "/tmp" 35 | type: Directory 36 | driver: 37 | cores: 1 38 | coreLimit: "1200m" 39 | memory: "512m" 40 | labels: 41 | version: 2.4.4 42 | serviceAccount: spark-operator-spark 43 | volumeMounts: 44 | - name: "test-volume" 45 | mountPath: "/tmp" 46 | executor: 47 | cores: 1 48 | instances: 1 49 | memory: "512m" 50 | labels: 51 | version: 2.4.4 52 | volumeMounts: 53 | - name: "test-volume" 54 | mountPath: "/tmp" 55 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | appnope = "==0.1.0" 8 | argon2-cffi = "==20.1.0" 9 | attrs = "==20.1.0" 10 | backcall = "==0.2.0" 11 | bleach = "==3.1.5" 12 | cffi = "==1.14.2" 13 | decorator = "==4.4.2" 14 | defusedxml = "==0.6.0" 15 | entrypoints = "==0.3" 16 | importlib-metadata = "==1.7.0" 17 | ipykernel = "==5.3.4" 18 | ipython = "==7.16.1" 19 | ipython-genutils = "==0.2.0" 20 | ipywidgets = "==7.5.1" 21 | jedi = "==0.17.2" 22 | jinja2 = "==2.11.2" 23 | jsonschema = "==3.2.0" 24 | jupyter = "==1.0.0" 25 | jupyter-client = "==6.1.6" 26 | jupyter-console = "==6.1.0" 27 | jupyter-core = "==4.6.3" 28 | markupsafe = "==1.1.1" 29 | mistune = "==0.8.4" 30 | nbconvert = "==5.6.1" 31 | nbformat = "==5.0.7" 32 | notebook = "==6.1.3" 33 | numpy = "==1.19.1" 34 | packaging = "==20.4" 35 | pandas = "==1.1.1" 36 | pandocfilters = "==1.4.2" 37 | parso = "==0.7.1" 38 | pexpect = "==4.8.0" 39 | pickleshare = "==0.7.5" 40 | prometheus-client = "==0.8.0" 41 | prompt-toolkit = "==3.0.6" 42 | psycopg2 = "==2.8.5" 43 | ptyprocess = "==0.6.0" 44 | pycparser = "==2.20" 45 | pygments = "==2.6.1" 46 | pyparsing = "==2.4.7" 47 | pyrsistent = "==0.16.0" 48 | python-dateutil = "==2.8.1" 49 | pytz = "==2020.1" 50 | pyzmq = "==19.0.2" 51 | qtconsole = "==4.7.6" 52 | qtpy = "==1.9.0" 53 | send2trash = "==1.5.0" 54 | six = "==1.15.0" 55 | terminado = "==0.8.3" 56 | testpath = "==0.4.4" 57 | tornado = "==6.0.4" 58 | traitlets = "==4.3.3" 59 | wcwidth = "==0.2.5" 60 | webencodings = "==0.5.1" 61 | widgetsnbextension = "==3.5.1" 62 | zipp = "==3.1.0" 63 | 64 | [dev-packages] 65 | 66 | [requires] 67 | python_version = "3.9" 68 | -------------------------------------------------------------------------------- /TERMS.md: -------------------------------------------------------------------------------- 1 | As a work of the United States Government, this package (excluding any 2 | exceptions listed below) is in the public domain within the United States. 3 | Additionally, we waive copyright and related rights in the work worldwide 4 | through the [CC0 1.0 Universal public domain dedication][CC0]. 5 | 6 | Software source code previously released under an open source license and then 7 | modified by CFPB staff or its contractors is considered a "joint work" 8 | (see 17 USC § 101); it is partially copyrighted, partially public domain, 9 | and as a whole is protected by the copyrights of the non-government authors and 10 | must be released according to the terms of the original open-source license. 11 | Segments written by CFPB staff, and by contractors who are developing software 12 | on behalf of CFPB are also in the public domain, and copyright and related 13 | rights for that work are waived through the CC0 1.0 Universal dedication. 14 | 15 | For further details, please see the CFPB [Source Code Policy][policy]. 16 | 17 | 18 | ## CC0 1.0 Universal Summary 19 | 20 | This is a human-readable summary of the [Legal Code (read the full text)][CC0]. 21 | 22 | ### No Copyright 23 | 24 | The person who associated a work with this deed has dedicated the work to 25 | the public domain by waiving all of his or her rights to the work worldwide 26 | under copyright law, including all related and neighboring rights, to the 27 | extent allowed by law. 28 | 29 | You can copy, modify, distribute and perform the work, even for commercial 30 | purposes, all without asking permission. See Other Information below. 31 | 32 | ### Other Information 33 | 34 | In no way are the patent or trademark rights of any person affected by CC0, 35 | nor are the rights that other persons may have in the work or in how the 36 | work is used, such as publicity or privacy rights. 37 | 38 | Unless expressly stated otherwise, the person who associated a work with 39 | this deed makes no warranties about the work, and disclaims liability for 40 | all uses of the work, to the fullest extent permitted by applicable law. 41 | When using or citing the work, you should not imply endorsement by the 42 | author or the affirmer. 43 | 44 | [policy]: https://github.com/cfpb/source-code-policy/ 45 | [CC0]: http://creativecommons.org/publicdomain/zero/1.0/legalcode 46 | 47 | 48 | ## Exceptions 49 | 50 | _Source code or other assets that are excluded from the TERMS should be listed 51 | here. These may include dependencies that may be licensed differently or are 52 | not in the public domain._ -------------------------------------------------------------------------------- /analysis_examples/analysis_output/charts/1_filer_count_time_series.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/b053b1a6ebc4c264cc12ca3888d955cee4be4857/analysis_examples/analysis_output/charts/1_filer_count_time_series.png -------------------------------------------------------------------------------- /analysis_examples/analysis_output/charts/2_filer_count_by_category.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/b053b1a6ebc4c264cc12ca3888d955cee4be4857/analysis_examples/analysis_output/charts/2_filer_count_by_category.png -------------------------------------------------------------------------------- /analysis_examples/analysis_output/tables/1_filer_count_time_series.txt: -------------------------------------------------------------------------------- 1 | activity_year|count 2 | 2004|8871 3 | 2005|8873 4 | 2006|8902 5 | 2007|8636 6 | 2008|8422 7 | 2009|8151 8 | 2010|7952 9 | 2011|7676 10 | 2012|7435 11 | 2013|7214 12 | 2014|7062 13 | 2015|6913 14 | 2016|6762 15 | 2017|5852 16 | -------------------------------------------------------------------------------- /analysis_examples/analysis_output/tables/2_filer_count_by_category.txt: -------------------------------------------------------------------------------- 1 | activity_year|filer_count 2 | 2004|351 3 | 2005|343 4 | 2006|326 5 | 2007|309 6 | 2008|290 7 | 2009|287 8 | 2010|287 9 | 2011|299 10 | 2012|290 11 | 2013|290 12 | 2014|288 13 | 2015|282 14 | 2016|287 15 | 2017|244 16 | -------------------------------------------------------------------------------- /analysis_examples/sql_commands/1_filer_count_time_series.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | activity_year, 3 | COUNT (*) 4 | FROM 5 | hmda_public.ts_{year} 6 | GROUP BY 7 | activity_year; -------------------------------------------------------------------------------- /analysis_examples/sql_commands/2_filer_count_by_category.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | activity_year, COUNT (*) AS filer_count 3 | FROM 4 | hmda_public.ts_{year} 5 | {extention} 6 | ; -------------------------------------------------------------------------------- /census/readme.me: -------------------------------------------------------------------------------- 1 | 2 | The FFIEC Census flat file fields of interest are (column number :name): 3 | 0:HMDA Year, 4 | 1:MSA, 2:State, 5 | 3:County, 4:Tract, 6 | 14:Area Population, 7 | 20: Minority Population Percentage, 8 | 915:Number of Owner-Occupied Units, 9 | 899:Number of 1-4 Family Units, 10 | 580:Tract Median Family Income, 11 | 12:Tract to MSA Median Family Income Percentage, 12 | 952:Median Age of Housing Stock, 13 | 13:FFIEC Median Family Income. 14 | 15 | 16 | With the exception of FFIEC median family income percentage, the base data are available in the American Community Survey (ACS) or the Summary File from the [US Census Bureau](). The FFIEC median family income data are made available by the FFIEC on this [website](https://www.ffiec.gov/Medianincome.htm). Please note that the data provided by the FRB on behalf of the FFIEC incorporates annual changes made to geographic boundaries. These provided data update Census base data on a 5 year cycle, for example activity years 2012-2016 used the 2010 SF1/ACS 5 year datasets and the 2017 activity year changes to the 2015 ACS 5 year dataset. Some reference fields also change during this update cycle. For these reasons it is recommended to use the FFIEC Census data as these data will provide consistency with other HMDA publications, such as Aggregate and Disclosure reports. -------------------------------------------------------------------------------- /documentation_resources/ModifiedLarWithExcel.md: -------------------------------------------------------------------------------- 1 | # Opening Modified LAR Text Files With Excel 2 | Modified LAR is available as a pipe-delimited text file with the .txt extension and has no header row. To read this file in Excel, the following steps would need to be taken. 3 | 4 | 1. Navigate to the Modified Loan/Application Register (LAR) page [here](https://ffiec.cfpb.gov/data-publication/modified-lar/2018). Enter the institution name in the search box, and click `Download Modified LAR`. 5 | 6 | 2. Open Excel. Select the `File` menu, and then `Open`. In the Open menu, double-click on `Browse` and view the Downloads folder. Select the modified LAR file in the Downloads folder as shown below. 7 | 8 | ![alt text](https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/master/documentation_resources/example_images/mlar_tutorial_images/Step2.JPG) 9 | 10 | 3. A window will open that tells Excel how to read the file. Select `Delimited` in the first window and click `Next.` 11 | 12 | ![alt text](https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/master/documentation_resources/example_images/mlar_tutorial_images/Step3.JPG) 13 | 14 | 4. In the next window, specify the pipe delimiter. Select `Other` under the `Delimiters` column and place a pipe (`|`) character in the text box field. Click `Next.` 15 | 16 | ![alt text](https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/master/documentation_resources/example_images/mlar_tutorial_images/Step4.JPG) 17 | 18 | 5. In the final window, select `General` for data type, and select `Finish.` 19 | At this point, the data will populate into their own cells. 20 | 21 | ![alt text](https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/master/documentation_resources/example_images/mlar_tutorial_images/Step5.JPG) 22 | 23 | 6. To add a header, right click on the row number to highlight the top row of data, and select `Insert` to add a blank row. 24 | 25 | ![alt text](https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/master/documentation_resources/example_images/mlar_tutorial_images/Step6.JPG) 26 | 27 | 7. Select the modified LAR header for the appropriate activity year and open in Excel. These headers are available [here](https://github.com/cfpb/HMDA_Data_Science_Kit/tree/master/documentation_resources/schemas/mlar/headers/). To download from Github, click "Raw" at the top of the file. 28 | 29 | ![alt text](https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/master/documentation_resources/example_images/mlar_tutorial_images/Step_7_1.JPG) 30 | 31 | Right click the raw text file page and save as a csv in the Downloads folder. 32 | 33 | ![alt text](https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/master/documentation_resources/example_images/mlar_tutorial_images/Step_7_2.JPG) 34 | 35 | 8. Highlight the first row of the header file, right click, and select `Copy`. 36 | 37 | ![alt text](https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/master/documentation_resources/example_images/mlar_tutorial_images/Step8.JPG) 38 | 39 | 9. Navigate back to the modified LAR excel sheet, highlight the first row, right click, and select `Insert Copied Cells`. 40 | 41 | ![alt text](https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/master/documentation_resources/example_images/mlar_tutorial_images/Step9.JPG) 42 | -------------------------------------------------------------------------------- /documentation_resources/example_images/mlar_tutorial_images/Step2.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/b053b1a6ebc4c264cc12ca3888d955cee4be4857/documentation_resources/example_images/mlar_tutorial_images/Step2.JPG -------------------------------------------------------------------------------- /documentation_resources/example_images/mlar_tutorial_images/Step3.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/b053b1a6ebc4c264cc12ca3888d955cee4be4857/documentation_resources/example_images/mlar_tutorial_images/Step3.JPG -------------------------------------------------------------------------------- /documentation_resources/example_images/mlar_tutorial_images/Step4.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/b053b1a6ebc4c264cc12ca3888d955cee4be4857/documentation_resources/example_images/mlar_tutorial_images/Step4.JPG -------------------------------------------------------------------------------- /documentation_resources/example_images/mlar_tutorial_images/Step5.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/b053b1a6ebc4c264cc12ca3888d955cee4be4857/documentation_resources/example_images/mlar_tutorial_images/Step5.JPG -------------------------------------------------------------------------------- /documentation_resources/example_images/mlar_tutorial_images/Step6.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/b053b1a6ebc4c264cc12ca3888d955cee4be4857/documentation_resources/example_images/mlar_tutorial_images/Step6.JPG -------------------------------------------------------------------------------- /documentation_resources/example_images/mlar_tutorial_images/Step8.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/b053b1a6ebc4c264cc12ca3888d955cee4be4857/documentation_resources/example_images/mlar_tutorial_images/Step8.JPG -------------------------------------------------------------------------------- /documentation_resources/example_images/mlar_tutorial_images/Step9.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/b053b1a6ebc4c264cc12ca3888d955cee4be4857/documentation_resources/example_images/mlar_tutorial_images/Step9.JPG -------------------------------------------------------------------------------- /documentation_resources/example_images/mlar_tutorial_images/Step_7_1.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/b053b1a6ebc4c264cc12ca3888d955cee4be4857/documentation_resources/example_images/mlar_tutorial_images/Step_7_1.JPG -------------------------------------------------------------------------------- /documentation_resources/example_images/mlar_tutorial_images/Step_7_2.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/b053b1a6ebc4c264cc12ca3888d955cee4be4857/documentation_resources/example_images/mlar_tutorial_images/Step_7_2.JPG -------------------------------------------------------------------------------- /documentation_resources/schemas/lar/lar_2004_2013.csv: -------------------------------------------------------------------------------- 1 |  Data Field Number , Data Field Name , Data Field Type , Start Position , Stop Position , Field Length 2 | 1, As of Date , Numeric ,1,4,4 3 | 2, Respondent ID , Alphanumeric ,5,14,10 4 | 3, Agency Code , Numeric ,15,15,1 5 | 4, Loan Type , Numeric ,16,16,1 6 | 5, Loan Purpose , Numeric ,17,17,1 7 | 6, Occupancy , Numeric ,18,18,1 8 | 7, Loan Amount (Thousands) , Numeric ,19,23,5 9 | 8, Action Taken , Numeric ,24,24,1 10 | 9, MSA/MD , Alphanumeric ,25,29,5 11 | 10, State Code , Alphanumeric ,30,31,2 12 | 11, County Code , Alphanumeric ,32,34,3 13 | 12, Census Tract , Alphanumeric ,35,41,7 14 | 13, Applicant Sex , Numeric ,42,42,1 15 | 14, Co Applicant Sex , Numeric ,43,43,1 16 | 15, Income , Alphanumeric ,44,47,4 17 | 16, Purchaser Type , Numeric ,48,48,1 18 | 17, Denial Reason 1 , Numeric ,49,49,1 19 | 18, Denial Reason 2 , Numeric ,50,50,1 20 | 19, Denial Reason 3 , Numeric ,51,51,1 21 | 20, Edit Status , Numeric ,52,52,1 22 | 21, Property Type , Numeric ,53,53,1 23 | 22, Preapproval , Numeric ,54,54,1 24 | 23, Applicant Ethnicity , Numeric ,55,55,1 25 | 24, Co Applicant Ethnicity , Numeric ,56,56,1 26 | 25, Applicant Race 1 , Numeric ,57,57,1 27 | 26, Applicant Race 2 , Numeric ,58,58,1 28 | 27, Applicant Race 3 , Numeric ,59,59,1 29 | 28, Applicant Race 4 , Numeric ,60,60,1 30 | 29, Applicant Race 5 , Numeric ,61,61,1 31 | 30, Co Applicant Race 1 , Numeric ,62,62,1 32 | 31, Co Applicant Race 2 , Numeric ,63,63,1 33 | 32, Co Applicant Race 3 , Numeric ,64,64,1 34 | 33, Co Applicant Race 4 , Numeric ,65,65,1 35 | 34, Co Applicant Race 5 , Numeric ,66,66,1 36 | 35, Rate Spread , Alphanumeric ,67,71,5 37 | 36, HOEPA Status , Numeric ,72,72,1 38 | 37, Lien Status , Numeric ,73,73,1 39 | 38, Sequence Number , Numeric ,74,80,7 -------------------------------------------------------------------------------- /documentation_resources/schemas/lar/lar_2004_2013.md: -------------------------------------------------------------------------------- 1 | |Data Field Number|Data Field Name|Data Field Type|Start Position|Stop Position|Field Length| 2 | |-----------------|---------------|---------------|--------------|-------------|------------| 3 | |1|As of Date|Numeric|1|4|4| 4 | |2|Respondent ID|Alphanumeric|5|14|10| 5 | |3|Agency Code|Numeric|15|15|1| 6 | |4|Loan Type|Numeric|16|16|1| 7 | |5|Loan Purpose|Numeric|17|17|1| 8 | |6|Occupancy|Numeric|18|18|1| 9 | |7|Loan Amount (Thousands)|Numeric|19|23|5| 10 | |8|Action Taken|Numeric|24|24|1| 11 | |9|MSA/MD|Alphanumeric|25|29|5| 12 | |10|State Code|Alphanumeric|30|31|2| 13 | |11|County Code|Alphanumeric|32|34|3| 14 | |12|Census Tract|Alphanumeric|35|41|7| 15 | |13|Applicant Sex|Numeric|42|42|1| 16 | |14|Co Applicant Sex|Numeric|43|43|1| 17 | |15|Income|Alphanumeric|44|47|4| 18 | |16|Purchaser Type|Numeric|48|48|1| 19 | |17|Denial Reason 1|Numeric|49|49|1| 20 | |18|Denial Reason 2|Numeric|50|50|1| 21 | |19|Denial Reason 3|Numeric|51|51|1| 22 | |20|Edit Status| Numeric|52|52|1| 23 | |21|Property Type| Numeric|53|53|1| 24 | |22|Preapproval| Numeric|54|54|1| 25 | |23|Applicant Ethnicity|Numeric|55|55|1| 26 | |24|Co Applicant Ethnicity|Numeric|56|56|1| 27 | |25|Applicant Race 1|Numeric|57|57|1| 28 | |26|Applicant Race 2|Numeric|58|58|1| 29 | |27|Applicant Race 3|Numeric|59|59|1| 30 | |28|Applicant Race 4|Numeric|60|60|1| 31 | |29|Applicant Race 5|Numeric|61|61|1| 32 | |30|Co Applicant Race 1|Numeric|62|62|1| 33 | |31|Co Applicant Race 2|Numeric|63|63|1| 34 | |32|Co Applicant Race 3|Numeric|64|64|1| 35 | |33|Co Applicant Race 4|Numeric|65|65|1| 36 | |34|Co Applicant Race 5|Numeric|66|66|1| 37 | |35|Rate Spread|Alphanumeric|67|71|5| 38 | |36|HOEPA Status|Numeric|72|72|1| 39 | |37|Lien Status|Numeric|73|73|1| 40 | |38|Sequence Number|Numeric|74|80|7| -------------------------------------------------------------------------------- /documentation_resources/schemas/lar/lar_2014_2016.csv: -------------------------------------------------------------------------------- 1 |  Data Field Number , Data Field Name , Data Field Type 2 | 1, As of Year , Numeric 3 | 2, Respondent ID , Alphanumeric 4 | 3, Agency Code , Numeric 5 | 4, Loan Type , Numeric 6 | 5, Property Type , Numeric 7 | 6, Loan Purpose , Numeric 8 | 7, Occupancy , Numeric 9 | 8, Loan Amount (thousands) , Numeric 10 | 9, Preapproval , Numeric 11 | 10, Action Type , Numeric 12 | 11, MSA/MD , Alphanumeric 13 | 12, State Code , Alphanumeric 14 | 13, County Code , Alphanumeric 15 | 14, Census Tract Number , Alphanumeric 16 | 15, Applicant Ethnicity , Numeric 17 | 16, Co Applicant Ethnicity , Numeric 18 | 17, Applicant Race 1 , Numeric 19 | 18, Applicant Race 2 , Numeric 20 | 19, Applicant Race 3 , Numeric 21 | 20, Applicant Race 4 , Numeric 22 | 21, Applicant Race 5 , Numeric 23 | 22, Co Applicant Race 1 , Numeric 24 | 23, Co Applicant Race 2 , Numeric 25 | 24, Co Applicant Race 3 , Numeric 26 | 25, Co Applicant Race 4 , Numeric 27 | 26, Co Applicant Race 5 , Numeric 28 | 27, Applicant Sex , Numeric 29 | 28, Co Applicant Sex , Numeric 30 | 29, Applicant Income (thousands) , Alphanumeric 31 | 30, Purchaser Type , Numeric 32 | 31, Denial Reason 1 , Numeric 33 | 32, Denial Reason 2 , Numeric 34 | 33, Denial Reason 3 , Numeric 35 | 34, Rate Spread , Alphanumeric 36 | 35, HOEPA Status , Numeric 37 | 36, Lien Status , Numeric 38 | 37, Edit Status , Numeric 39 | 38, Sequence Number , Numeric 40 | 39, Population , Alphanumeric 41 | 40, Minority Population Percent , Alphanumeric 42 | 41, FFIEC Median Family Income , Alphanumeric 43 | 42, Tract to MSA/MD Income Percent , Alphanumeric 44 | 43, Number of Owner-occupied Units , Alphanumeric 45 | 44, Number of 1-to-4 Family Units , Alphanumeric 46 | 45, Application Date Indicator , Numeric -------------------------------------------------------------------------------- /documentation_resources/schemas/lar/lar_2014_2016.md: -------------------------------------------------------------------------------- 1 | |Data Field Number|Data Field Name|Data Field Type| 2 | |-----------------|---------------|---------------| 3 | |1|As of Year|Numeric| 4 | |2|Respondent ID|Alphanumeric| 5 | |3|Agency Code|Numeric| 6 | |4|Loan Type|Numeric| 7 | |5|Property Type|Numeric| 8 | |6|Loan Purpose|Numeric| 9 | |7|Occupancy|Numeric| 10 | |8|Loan Amount (thousands)|Numeric| 11 | |9|Preapproval|Numeric| 12 | |10|Action Type|Numeric| 13 | |11|MSA/MD|Alphanumeric| 14 | |12|State Code|Alphanumeric| 15 | |13|County Code|Alphanumeric| 16 | |14|Census Tract Number|Alphanumeric| 17 | |15|Applicant Ethnicity|Numeric| 18 | |16|Co Applicant Ethnicity|Numeric| 19 | |17|Applicant Race 1|Numeric| 20 | |18|Applicant Race 2|Numeric| 21 | |19|Applicant Race 3|Numeric| 22 | |20|Applicant Race 4|Numeric| 23 | |21|Applicant Race 5|Numeric| 24 | |22|Co Applicant Race 1|Numeric| 25 | |23|Co Applicant Race 2|Numeric| 26 | |24|Co Applicant Race 3|Numeric| 27 | |25|Co Applicant Race 4|Numeric| 28 | |26|Co Applicant Race 5|Numeric| 29 | |27|Applicant Sex|Numeric| 30 | |28|Co Applicant Sex|Numeric| 31 | |29|Applicant Income (thousands)|Alphanumeric| 32 | |30|Purchaser Type|Numeric| 33 | |31|Denial Reason 1|Numeric| 34 | |32|Denial Reason 2|Numeric| 35 | |33|Denial Reason 3|Numeric| 36 | |34|Rate Spread|Alphanumeric| 37 | |35|HOEPA Status|Numeric| 38 | |36|Lien Status|Numeric| 39 | |37|Edit Status|Numeric| 40 | |38|Sequence Number|Numeric| 41 | |39|Population|Alphanumeric| 42 | |40|Minority Population Percent|Alphanumeric| 43 | |41|FFIEC Median Family Income|Alphanumeric| 44 | |42|Tract to MSA/MD Income Percent|Alphanumeric| 45 | |43|Number of Owner-occupied Units|Alphanumeric| 46 | |44|Number of 1-to-4 Family Units|Alphanumeric| 47 | |45|Application Date Indicator|Numeric| -------------------------------------------------------------------------------- /documentation_resources/schemas/lar/lar_2017.csv: -------------------------------------------------------------------------------- 1 |  Data Field Number , Data Field Name , Data Field Type 2 | 1, Record Identifier - Value is 2 , Numeric 3 | 2, Respondent-ID , Alphanumeric 4 | 3, Agency Code , Numeric 5 | 4, Loan Type , Numeric 6 | 5, Property Type , Numeric 7 | 6, Loan Purpose , Numeric 8 | 7, Owner Occupancy , Numeric 9 | 8, Loan Amount , Numeric 10 | 9, Preapprovals , Numeric 11 | 10, Type of Action Taken , Numeric 12 | 11, Metropolitan Statistical Area/Metropolitan Division , Alphanumeric 13 | 12, State Code , Alphanumeric 14 | 13, County Code , Alphanumeric 15 | 14, Census Tract , Alphanumeric 16 | 15, Applicant Ethnicity , Numeric 17 | 16, Co-applicant Ethnicity , Numeric 18 | 17, Applicant Race: 1 , Numeric 19 | 18, Applicant Race: 2 , Numeric 20 | 19, Applicant Race: 3 , Numeric 21 | 20, Applicant Race: 4 , Numeric 22 | 21, Applicant Race: 5 , Numeric 23 | 22, Co-applicant Race: 1 , Numeric 24 | 23, Co-applicant Race: 2 , Numeric 25 | 24, Co-applicant Race: 3 , Numeric 26 | 25, Co-applicant Race: 4 , Numeric 27 | 26, Co-applicant Race: 5 , Numeric 28 | 27, Applicant Sex , Numeric 29 | 28, Co-applicant Sex , Numeric 30 | 29, Applicant Income , Alphanumeric 31 | 30, Type of Purchaser , Numeric 32 | 31, Denial Reason: 1 , Numeric 33 | 32, Denial Reason: 2 , Numeric 34 | 33, Denial Reason: 3 , Numeric 35 | 34, Rate Spread , Alphanumeric 36 | 35, HOEPA Status , Numeric 37 | 36, Lien Status , Numeric 38 | 37, Population , Numeric 39 | 38, Minority Population % , Numeric 40 | 39, FFIEC Median Family Income , Numeric 41 | 40, Tract to MSA/MD Median Family Income % , Numeric 42 | 41, Number of Owner Occupied Units , Numeric 43 | 42, Number of 1- to 4-Family units , Numeric -------------------------------------------------------------------------------- /documentation_resources/schemas/lar/lar_2017.md: -------------------------------------------------------------------------------- 1 | 2 | | Data Field Number | Data Field Name | Data Field Type | 3 | |-------------------|-----------------|-----------------| 4 | |1|Record Identifier|Numeric| 5 | |2|Respondent-ID|Alphanumeric| 6 | |3|Agency Code|Numeric| 7 | |4|Loan Type|Numeric| 8 | |5|Property Type|Numeric| 9 | |6|Loan Purpose|Numeric| 10 | |7|Owner Occupancy|Numeric| 11 | |8|Loan Amount|Numeric| 12 | |9|Preapprovals|Numeric| 13 | |10|Type of Action Taken|Numeric| 14 | |11|MSA/MD|Alphanumeric| 15 | |12|State Code|Alphanumeric| 16 | |13|County Code|Alphanumeric| 17 | |14|Census Tract|Alphanumeric| 18 | |15|Applicant Ethnicity|Numeric| 19 | |16|Co-applicant Ethnicity|Numeric| 20 | |17|Applicant Race: 1|Numeric| 21 | |18|Applicant Race: 2|Numeric| 22 | |19|Applicant Race: 3|Numeric| 23 | |20|Applicant Race: 4|Numeric| 24 | |21|Applicant Race: 5|Numeric| 25 | |22|Co-applicant Race: 1|Numeric| 26 | |23|Co-applicant Race: 2|Numeric| 27 | |24|Co-applicant Race: 3|Numeric| 28 | |25|Co-applicant Race: 4|Numeric| 29 | |26|Co-applicant Race: 5|Numeric| 30 | |27|Applicant Sex|Numeric| 31 | |28|Co-applicant Sex|Numeric| 32 | |29|Applicant Income|Alphanumeric| 33 | |30|Type of Purchaser |Numeric| 34 | |31|Denial Reason: 1|Numeric| 35 | |32|Denial Reason: 2|Numeric| 36 | |33|Denial Reason: 3|Numeric| 37 | |34|Rate Spread |Alphanumeric| 38 | |35|HOEPA Status|Numeric| 39 | |36|Lien Status|Numeric| 40 | |37|Population|Numeric| 41 | |38|Minority Population Pct|Numeric| 42 | |39|FFIEC Median Family Income|Numeric| 43 | |40|Tract to MSA/MD Median Family Income Pct|Numeric| 44 | |41|Number of Owner Occupied Units|Numeric| 45 | |42|Number of 1- to 4-Family units|Numeric| -------------------------------------------------------------------------------- /documentation_resources/schemas/mlar/mlar_header_2017.csv: -------------------------------------------------------------------------------- 1 |  Record Identifier, Respondent-ID, Agency Code, Loan Type, Property Type, Loan Purpose, Owner Occupancy, Loan Amount, Preapprovals, Type of Action Taken, Metropolitan Statistical Area/Metropolitan Division, State Code, County Code, Census Tract, Applicant Ethnicity, Co-applicant Ethnicity, Applicant Race: 1, Applicant Race: 2, Applicant Race: 3, Applicant Race: 4, Applicant Race: 5, Co-applicant Race: 1, Co-applicant Race: 2, Co-applicant Race: 3, Co-applicant Race: 4, Co-applicant Race: 5, Applicant Sex, Co-applicant Sex, Applicant Income, Type of Purchaser, Denial Reason: 1, Denial Reason: 2, Denial Reason: 3, Rate Spread, HOEPA Status, Lien Status -------------------------------------------------------------------------------- /documentation_resources/schemas/mlar/mlar_header_2018.csv: -------------------------------------------------------------------------------- 1 | Record Identifier,Legal Entity Identifier (LEI),Loan Type ,Loan Purpose,Preapproval ,Construction Method ,Occupancy Type ,Loan Amount,Action Taken ,State ,County,Census Tract ,Ethnicity of Applicant or Borrower: 1 ,Ethnicity of Applicant or Borrower: 2 ,Ethnicity of Applicant or Borrower: 3,Ethnicity of Applicant or Borrower: 4 ,Ethnicity of Applicant or Borrower: 5 ,Ethnicity of Co-Applicant or Co-Borrower: 1,Ethnicity of Co-Applicant or Co-Borrower: 2 ,Ethnicity of Co-Applicant or Co-Borrower: 3,Ethnicity of Co-Applicant or Co-Borrower: 4 ,Ethnicity of Co-Applicant or Co-Borrower: 5 ,Ethnicity of Applicant or Borrower Collected on the Basis of Visual Observation or Surname ,Ethnicity of Co-Applicant or Co-Borrower Collected on the Basis of Visual Observation or Surname ,Race of Applicant or Borrower: 1 ,Race of Applicant or Borrower: 2 ,Race of Applicant or Borrower: 3 ,Race of Applicant or Borrower: 4 ,Race of Applicant or Borrower: 5 ,Race of Co-Applicant or Co-Borrower: 1 ,Race of Co-Applicant or Co-Borrower: 2 ,Race of Co-Applicant or Co-Borrower: 3 ,Race of Co-Applicant or Co-Borrower: 4 ,Race of Co-Applicant or Co-Borrower: 5 ,Race of Applicant or Borrower Collected on the Basis of Visual Observation or Surname ,Race of Co-Applicant or Co-Borrower Collected on the Basis of Visual Observation or Surname ,Sex of Applicant or Borrower,Sex of Co-Applicant or Co-Borrower ,Sex of Applicant or Borrower Collected on the Basis of Visual Observation or Surname ,Sex of Co-Applicant or Co-Borrower Collected on the Basis of Visual Observation or Surname ,Age of Applicant or Borrower,Age of Applicant >= 62 ,Age of Co-Applicant or Co-Borrower ,Age of Co-Applicant >= 62 ,Income,Type of Purchaser ,Rate Spread ,HOEPA Status ,Lien Status ,Applicant or Borrower - Name and Version of Credit Scoring Model,Co-Applicant or Co-Borrower - Name and Version of Credit Scoring Model ,Reason for Denial: 1,Reason for Denial: 2 ,Reason for Denial: 3,Reason for Denial: 4,Total Loan Costs ,Total Points and Fees ,Origination Charges ,Discount Points ,Lender Credits ,Interest Rate,Prepayment Penalty Term ,Debt-to-Income Ratio ,Combined Loan-to-Value Ratio ,Loan Term,Introductory Rate Period,Balloon Payment ,Interest-Only Payments ,Negative Amortization ,Other Non-amortizing Features ,Property Value ,Manufactured Home Secured Property Type ,Manufactured Home Land Property Interest ,Total Units,Multifamily Affordable Units ,Submission of Application ,Initially Payable to Your Institution ,Automated Underwriting System: 1 ,Automated Underwriting System: 2 ,Automated Underwriting System: 3 ,Automated Underwriting System: 4 ,Automated Underwriting System: 5,Reverse Mortgage,Open-End Line of Credit ,Business or Commercial Purpose -------------------------------------------------------------------------------- /documentation_resources/schemas/mlar/mlar_schema_2017.csv: -------------------------------------------------------------------------------- 1 | Data Field Name,Data Field Type,Valid Values 2 | Record Identifier,Numeric,2 3 | Respondent-ID,Alphanumeric,Identification Numbers Based on the FIG 4 | Agency Code,Numeric,1 2 3 5 7 9 5 | Loan Type,Numeric,1 2 3 4 6 | Property Type,Numeric,1 2 3 7 | Loan Purpose,Numeric,1 2 3 8 | Owner Occupancy,Numeric,1 2 3 9 | Loan Amount,Numeric,Integer Reported in thousands. Rounded to the nearest thousand without leading zeros and without commas. 10 | Preapprovals,Numeric,1 2 3 11 | Type of Action Taken,Numeric,1 2 3 4 5 6 7 8 12 | Metropolitan Statistical Area/Metropolitan Division,Character Varying,MSA code or 'NA' 13 | State Code,Character Varying,2 digit FIPS State Code or 'NA' 14 | County Code,Character Varying,3 digit FIPS County Code or 'NA' 15 | Census Tract,Character Varying ,6 digit tract number (include decimal point) or 'NA' 16 | Applicant Ethnicity,Numeric,1 2 3 4 17 | Co-applicant Ethnicity,Numeric,1 2 3 4 5 18 | Applicant Race: 1,Numeric,1 2 3 4 5 6 7 19 | Applicant Race: 2,Character Varying,1 2 3 4 5 blank 20 | Applicant Race: 3,Character Varying,1 2 3 4 5 blank 21 | Applicant Race: 4,Character Varying,1 2 3 4 5 blank 22 | Applicant Race: 5,Character Varying,1 2 3 4 5 blank 23 | Co-applicant Race: 1,Numeric,1 2 3 4 5 6 7 8 24 | Co-applicant Race: 2,Character Varying,1 2 3 4 5 blank 25 | Co-applicant Race: 3,Character Varying,1 2 3 4 5 blank 26 | Co-applicant Race: 4,Character Varying,1 2 3 4 5 blank 27 | Co-applicant Race: 5,Character Varying,1 2 3 4 5 blank 28 | Applicant Sex,Numeric,1 2 3 4 29 | Co-applicant Sex,Numeric,1 2 3 4 5 30 | Applicant Income,Alphanumeric,Integer reported in thousands and rounded to the nearest thousand and without commas or 'NA.' 31 | Type of Purchaser,Numeric,0 1 2 3 4 5 6 7 8 9 32 | Denial Reason: 1,Character Varying,1 2 3 4 5 6 7 8 9 blank 33 | Denial Reason: 2,Character Varying,1 2 3 4 5 6 7 8 9 blank 34 | Denial Reason: 3,Character Varying,1 2 3 4 5 6 7 8 9 blank 35 | Rate Spread,Alphanumeric,Double or 'NA' 36 | HOEPA Status,Numeric,1 2 37 | Lien Status,Numeric,1 2 3 4 -------------------------------------------------------------------------------- /documentation_resources/schemas/panel/panel_2004_2013.csv: -------------------------------------------------------------------------------- 1 |  Field Number , Field Name , Field Data Type , Start , Stop , Length 2 | 1, Respondent ID , Alphanumeric ,1,10,10 3 | 2, MSA , Alphanumeric ,11,15,5 4 | 3, Agency Code , Numeric ,16,16,1 5 | 4, Agency Group Code , Numeric ,17,18,2 6 | 5, Respondent Name , Alphanumeric ,19,48,30 7 | 6, Respondent City , Alphanumeric ,49,73,25 8 | 7, Respondent State , Alphanumeric ,74,75,2 9 | 8, Respondent State FIPS , Alphanumeric ,76,77,2 10 | 9, Assets , Numeric ,78,87,10 11 | 10, Other Lender Code , Numeric ,88,88,1 12 | 11, Parent ID , Alphanumeric ,89,98,10 13 | 12, Parent Name , Alphanumeric ,99,128,30 14 | 13, Parent City , Alphanumeric ,129,153,25 15 | 14, Parent State , Alphanumeric ,154,155,2 16 | 15, As Of Year , Numeric ,156,159,4 17 | 16, Respondent RSSD , Numeric ,160,169,10 -------------------------------------------------------------------------------- /documentation_resources/schemas/panel/panel_2004_2013.md: -------------------------------------------------------------------------------- 1 | File format is fixed width 2 | 3 | File specifications are available from the National Archives. Please see [this link](https://github.com/cfpb/HMDA_Data_Science_Kit/blob/master/hmda_data_links.md) for a list of available NARA HMDA resources. 4 | 5 | | Field Number | Field Name | Field Data Type | Start | Stop | Length | 6 | |--------------|-----------------------|-----------------|-------|------|--------| 7 | | 1 | Respondent ID | Alphanumeric | 1 | 10 | 10 | 8 | | 2 | MSA | Alphanumeric | 11 | 15 | 5 | 9 | | 3 | Agency Code | Numeric | 16 | 16 | 1 | 10 | | 4 | Agency Group Code | Numeric | 17 | 18 | 2 | 11 | | 5 | Respondent Name | Alphanumeric | 19 | 48 | 30 | 12 | | 6 | Respondent City | Alphanumeric | 49 | 73 | 25 | 13 | | 7 | Respondent State | Alphanumeric | 74 | 75 | 2 | 14 | | 8 | Respondent State FIPS | Alphanumeric | 76 | 77 | 2 | 15 | | 9 | Assets | Numeric | 78 | 87 | 10 | 16 | | 10 | Other Lender Code | Numeric | 88 | 88 | 1 | 17 | | 11 | Parent ID | Alphanumeric | 89 | 98 | 10 | 18 | | 12 | Parent Name | Alphanumeric | 99 | 128 | 30 | 19 | | 13 | Parent City | Alphanumeric | 129 | 153 | 25 | 20 | | 14 | Parent State | Alphanumeric | 154 | 155 | 2 | 21 | | 15 | As Of Year | Numeric | 156 | 159 | 4 | 22 | | 16 | Respondent RSSD | Numeric | 160 | 169 | 10 | -------------------------------------------------------------------------------- /documentation_resources/schemas/panel/panel_2014_2016.csv: -------------------------------------------------------------------------------- 1 |  Field Number , Field Name , Data Type , Start , End , Length 2 | 1, Activity Year , Numeric ,1,4,4 3 | 2, Respondent ID , Alphanumeric ,5,14,10 4 | 3, Agency code , Numeric ,15,15,1 5 | 4, Parent Respondent ID , Alphanumeric ,16,25,10 6 | 5, Parent Name , Alphanumeric ,26,55,30 7 | 6, Parent City , Alphanumeric ,56,80,25 8 | 7, Parent State , Alphanumeric ,81,82,2 9 | 8, Region , Numeric ,83,84,2 10 | 9, Assets , Numeric ,85,94,10 11 | 10, Other Lender Code , Numeric ,95,95,1 12 | 11, Respondent Name , Alphanumeric ,96,125,30 13 | 12, Filler , Alphanumeric ,126,165,30 14 | 13, Respondent City , Alphanumeric ,166,190,25 15 | 14, Respondent State , Alphanumeric ,191,192,2 16 | 15, Filler , Alphanumeric ,193,202,10 17 | 16, Filler , Alphanumeric ,203,212,10 18 | 17, Top Holder RSSD , Numeric ,213,222,10 19 | 18, Top Holder Name , Alphanumeric ,223,252,30 20 | 19, Top Holder City , Alphanumeric ,253,277,25 21 | 20, Top Holder State , Alphanumeric ,278,279,2 22 | 21, Top Holder Country , Alphanumeric ,280,319,40 23 | 22, Respondent RSSD , Numeric ,320,329,10 24 | 23, Parent RSSD , Numeric ,330,339,10 25 | 24, Respondent FIPS State Number , Alphanumeric ,340,341,2 -------------------------------------------------------------------------------- /documentation_resources/schemas/panel/panel_2014_2016.md: -------------------------------------------------------------------------------- 1 | File format is fixed width 2 | 3 | Specs for this data were pulled from [this](https://www.ffiec.gov/hmda/hmdaflat.htm) FFIEC HMDA site. 4 | - [2016](https://www.ffiec.gov/hmdarawdata/FORMATS/2016HMDAReporterPanel.pdf) 5 | - [2015](https://www.ffiec.gov/hmdarawdata/FORMATS/2015HMDAReporterPanel.pdf) 6 | - [2014](https://www.ffiec.gov/hmdarawdata/FORMATS/2014HMDAReporterPanel.pdf) 7 | 8 | | Field Number | Field Name | Data Type | Start | End | Length | 9 | |--------------|------------------------------|--------------|-------|-----|--------| 10 | | 1 | Activity Year | Numeric | 1 | 4 | 4 | 11 | | 2 | Respondent ID | Alphanumeric | 5 | 14 | 10 | 12 | | 3 | Agency code | Numeric | 15 | 15 | 1 | 13 | | 4 | Parent Respondent ID | Alphanumeric | 16 | 25 | 10 | 14 | | 5 | Parent Name | Alphanumeric | 26 | 55 | 30 | 15 | | 6 | Parent City | Alphanumeric | 56 | 80 | 25 | 16 | | 7 | Parent State | Alphanumeric | 81 | 82 | 2 | 17 | | 8 | Region | Numeric | 83 | 84 | 2 | 18 | | 9 | Assets | Numeric | 85 | 94 | 10 | 19 | | 10 | Other Lender Code | Numeric | 95 | 95 | 1 | 20 | | 11 | Respondent Name | Alphanumeric | 96 | 125 | 30 | 21 | | 12 | Filler | Alphanumeric | 126 | 165 | 30 | 22 | | 13 | Respondent City | Alphanumeric | 166 | 190 | 25 | 23 | | 14 | Respondent State | Alphanumeric | 191 | 192 | 2 | 24 | | 15 | Filler | Alphanumeric | 193 | 202 | 10 | 25 | | 16 | Filler | Alphanumeric | 203 | 212 | 10 | 26 | | 17 | Top Holder RSSD | Numeric | 213 | 222 | 10 | 27 | | 18 | Top Holder Name | Alphanumeric | 223 | 252 | 30 | 28 | | 19 | Top Holder City | Alphanumeric | 253 | 277 | 25 | 29 | | 20 | Top Holder State | Alphanumeric | 278 | 279 | 2 | 30 | | 21 | Top Holder Country | Alphanumeric | 280 | 319 | 40 | 31 | | 22 | Respondent RSSD | Numeric | 320 | 329 | 10 | 32 | | 23 | Parent RSSD | Numeric | 330 | 339 | 10 | 33 | | 24 | Respondent FIPS State Number | Alphanumeric | 340 | 341 | 2 | -------------------------------------------------------------------------------- /documentation_resources/schemas/panel/panel_2017.csv: -------------------------------------------------------------------------------- 1 |  Field Number , Field Name , Data Type , Length 2 | 1, Activity Year , Numeric ,4 3 | 2, Respondent ID , Alphanumeric ,10 4 | 3, Agency code , Numeric ,1 5 | 4, Parent Respondent ID , Alphanumeric ,10 6 | 5, Parent Name , Alphanumeric ,30 7 | 6, Parent City , Alphanumeric ,25 8 | 7, Parent State , Alphanumeric ,2 9 | 8, Region , Numeric ,2 10 | 9, Assets , Numeric ,10 11 | 10, Other Lender Code , Numeric ,1 12 | 11, Respondent Name , Alphanumeric ,30 13 | 13, Respondent City , Alphanumeric ,25 14 | 14, Respondent State , Alphanumeric ,2 15 | 17, Top Holder RSSD , Numeric ,10 16 | 18, Top Holder Name , Alphanumeric ,30 17 | 19, Top Holder City , Alphanumeric ,25 18 | 20, Top Holder State , Alphanumeric ,2 19 | 21, Top Holder Country , Alphanumeric ,40 20 | 22, Respondent RSSD , Numeric ,10 21 | 23, Parent RSSD , Numeric ,10 22 | 24, Respondent FIPS State Number , Alphanumeric ,2 -------------------------------------------------------------------------------- /documentation_resources/schemas/panel/panel_2017.md: -------------------------------------------------------------------------------- 1 | File format is pipe delimited 2 | 3 | The file specification is pulled from [here](https://s3.amazonaws.com/cfpb-hmda-public/prod/snapshot-data/2017_publicstatic_dataformat.zip). 4 | 5 | | Field Number | Field Name | Data Type | Length | 6 | |--------------|------------------------------|--------------|--------| 7 | | 1 | Activity Year | Numeric | 4 | 8 | | 2 | Respondent ID | Alphanumeric | 10 | 9 | | 3 | Agency code | Numeric | 1 | 10 | | 4 | Parent Respondent ID | Alphanumeric | 10 | 11 | | 5 | Parent Name | Alphanumeric | 30 | 12 | | 6 | Parent City | Alphanumeric | 25 | 13 | | 7 | Parent State | Alphanumeric | 2 | 14 | | 8 | Region | Numeric | 2 | 15 | | 9 | Assets | Numeric | 10 | 16 | | 10 | Other Lender Code | Numeric | 1 | 17 | | 11 | Respondent Name | Alphanumeric | 30 | 18 | | 13 | Respondent City | Alphanumeric | 25 | 19 | | 14 | Respondent State | Alphanumeric | 2 | 20 | | 17 | Top Holder RSSD | Numeric | 10 | 21 | | 18 | Top Holder Name | Alphanumeric | 30 | 22 | | 19 | Top Holder City | Alphanumeric | 25 | 23 | | 20 | Top Holder State | Alphanumeric | 2 | 24 | | 21 | Top Holder Country | Alphanumeric | 40 | 25 | | 22 | Respondent RSSD | Numeric | 10 | 26 | | 23 | Parent RSSD | Numeric | 10 | 27 | | 24 | Respondent FIPS State Number | Alphanumeric | 2 | -------------------------------------------------------------------------------- /documentation_resources/schemas/panel/panel_2018_2019.csv: -------------------------------------------------------------------------------- 1 |  Field Number , Field Name ,Data Type, Length 2 | 1,activity_year,Numeric,4 3 | 2,lei,Alphanumeric,20 4 | 3,tax_id,Alphanumeric,10 5 | 4,agency_code,Numeric,1 6 | 5,id_2017,Alphanumeric,10 7 | 6,arid_2017,Alphanumeric,11 8 | 7,respondent_rssd,Numeric, 9 | 8,respondent_name,Alphanumeric, 10 | 9,respondent_state,Alphanumeric,2 11 | 10,respondent_city,Alphanumeric, 12 | 11,assets,Numeric, 13 | 13,other_lender_code,Numeric,1 14 | 14,parent_rssd,Numeric, 15 | 17,parent_name,Alphanumeric, 16 | 18,top_holder_rssd,Numeric, 17 | 19,top_holder_name,Alphanumeric, -------------------------------------------------------------------------------- /documentation_resources/schemas/panel/panel_2018_2019.md: -------------------------------------------------------------------------------- 1 | |Field Number|Field Name|Data Type|Length 2 | |------------|----------|---------|------ 3 | |1|activity_year|Numeric|4 4 | |2|lei|Alphanumeric|20 5 | |3|tax_id|Alphanumeric|10 6 | |4|agency_code|Numeric|1 7 | |5|id_2017|Alphanumeric|10 8 | |6|arid_2017|Alphanumeric|11 9 | |7|respondent_rssd|Numeric| 10 | |8|respondent_name|Alphanumeric| 11 | |9|respondent_state|Alphanumeric|2 12 | |10|respondent_city|Alphanumeric| 13 | |11|assets|Numeric| 14 | |13|other_lender_code|Numeric|1 15 | |14|parent_rssd|Numeric| 16 | |17|parent_name|Alphanumeric| 17 | |18|top_holder_rssd|Numeric| 18 | |19|top_holder_name|Alphanumeric| -------------------------------------------------------------------------------- /documentation_resources/schemas/ts/ts_2014_2016.csv: -------------------------------------------------------------------------------- 1 |  Field Number , Field Name , Data Type , Max Length 2 | 1, Activity Year , Numeric ,4 3 | 2, Respondent ID , Alphanumeric ,10 4 | 3, Agency Code , Numeric ,1 5 | 4, Tax ID , Alphanumeric ,10 6 | 5, Respondent Name , Alphanumeric ,30 7 | 6, Respondent Address , Alphanumeric ,40 8 | 7, Respondent City , Alphanumeric ,25 9 | 8, Respondent State , Alphanumeric ,2 10 | 9, Respondent ZIP , Alphanumeric ,10 11 | 10, Parent Name , Alphanumeric ,30 12 | 11, Parent Address , Alphanumeric ,40 13 | 12, Parent City , Alphanumeric ,25 14 | 13, Parent State , Alphanumeric ,2 15 | 14, Parent ZIP , Alphanumeric ,10 16 | 15, Respondent Name (Panel) , Alphanumeric ,30 17 | 16, Respondent City (Panel) , Alphanumeric ,25 18 | 17, Respondent State (Panel) , Alphanumeric ,2 19 | 18, Assets (Panel) , Numeric ,10 20 | 19, Other Lender Code (Panel) , Numeric ,1 21 | 20, Region Code (Panel) , Numeric ,2 22 | 21, LAR Count , Numeric ,10 23 | 22, Validity Error , Alphanumeric ,1 -------------------------------------------------------------------------------- /documentation_resources/schemas/ts/ts_2014_2016.md: -------------------------------------------------------------------------------- 1 | |Field Number|Field Name|Data Type|Max Length| 2 | |------------|----------|---------|----------| 3 | |1|Activity Year|Numeric|4| 4 | |2|Respondent ID|Alphanumeric|10| 5 | |3|Agency Code|Numeric|1| 6 | |4|Tax ID|Alphanumeric|10| 7 | |5|Respondent Name|Alphanumeric|30| 8 | |6|Respondent Address|Alphanumeric|40| 9 | |7|Respondent City| Alphanumeric|25| 10 | |8|Respondent State|Alphanumeric|2| 11 | |9|Respondent ZIP|Alphanumeric|10| 12 | |10|Parent Name|Alphanumeric|30| 13 | |11|Parent Address|Alphanumeric|40| 14 | |12|Parent City|Alphanumeric|25| 15 | |13|Parent State|Alphanumeric|2| 16 | |14|Parent ZIP|Alphanumeric|10| 17 | |15|Respondent Name (Panel)|Alphanumeric|30| 18 | |16|Respondent City (Panel)|Alphanumeric|25| 19 | |17|Respondent State (Panel)|Alphanumeric|2| 20 | |18|Assets (Panel)|Numeric|10| 21 | |19|Other Lender Code (Panel)|Numeric|1| 22 | |20|Region Code (Panel)|Numeric|2| 23 | |21|LAR Count|Numeric|10| 24 | |22|Validity Error|Alphanumeric|1| -------------------------------------------------------------------------------- /documentation_resources/schemas/ts/ts_2017.csv: -------------------------------------------------------------------------------- 1 |  Field Number , Field Name , Data Type 2 | 1, Record ID , Numeric 3 | 2, Respondent ID , Alphanumeric 4 | 3, Agency Code , Numeric 5 | 4, Activity Year , Numeric 6 | 5, Tax ID , Alphanumeric 7 | 6, LAR Entries , Numeric 8 | 7, Respondent Name , Alphanumeric 9 | 8, Respondent Address , Alphanumeric 10 | 9, Respondent City , Alphanumeric 11 | 10, Respondent State , Alphanumeric 12 | 11, Respondent ZIP , Alphanumeric 13 | 12, Parent Name , Alphanumeric 14 | 13, Parent Address , Alpnanumeric 15 | 14, Parent City , Alphanumeric 16 | 15, Parent State , Alphanumric 17 | 16, Parent ZIP , Alphanumeric -------------------------------------------------------------------------------- /documentation_resources/schemas/ts/ts_2017.md: -------------------------------------------------------------------------------- 1 | |Field Number|Field Name|Data Type| 2 | |------------|----------|---------| 3 | |1|Record ID|Numeric| 4 | |2|Respondent ID|Alphanumeric| 5 | |3|Agency Code|Numeric| 6 | |4|Activity Year|Numeric| 7 | |5|Tax ID|Alphanumeric| 8 | |6|LAR Entries|Numeric| 9 | |7|Respondent Name|Alphanumeric| 10 | |8|Respondent Address|Alphanumeric| 11 | |9|Respondent City|Alphanumeric| 12 | |10|Respondent State|Alphanumeric| 13 | |11|Respondent ZIP|Alphanumeric| 14 | |12|Parent Name|Alphanumeric| 15 | |13|Parent Address|Alpnanumeric| 16 | |14|Parent City|Alphanumeric| 17 | |15|Parent State|Alphanumric| 18 | |16|Parent ZIP|Alphanumeric| -------------------------------------------------------------------------------- /documentation_resources/schemas/ts/ts_2018_2019.csv: -------------------------------------------------------------------------------- 1 |  Field Number , Field Name , Data Type ,Length 2 | 1,activity_year,Numeric,4 3 | 2,calendar_quarter,Numeric,1 4 | 3,lei,Alphanumeric,20 5 | 4,tax_id,Alphanumeric,10 6 | 5,agency_code,Numeric,1 7 | 6,respondent_name,Alphanumeric, 8 | 7,respondent_state,Alphanumeric, 9 | 8,respondent_city,Alphanumeric, 10 | 9,respondent_zip_code,Alphanumeric, 11 | 10,lar_count,Numeric, -------------------------------------------------------------------------------- /documentation_resources/schemas/ts/ts_2018_2019.md: -------------------------------------------------------------------------------- 1 | |Field Number|Field Name|Data Type|Length 2 | |------------|----------|---------|------ 3 | |1|activity_year|Numeric|4 4 | |2|calendar_quarter|Numeric|1 5 | |3|lei|Alphanumeric|20 6 | |4|tax_id|Alphanumeric|10 7 | |5|agency_code|Numeric|1 8 | |6|respondent_name|Alphanumeric| 9 | |7|respondent_state|Alphanumeric| 10 | |8|respondent_city|Alphanumeric| 11 | |9|respondent_zip_code|Alphanumeric| 12 | |10|lar_count|Numeric| -------------------------------------------------------------------------------- /download_scripts/unzip_all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | bash download_scripts/unzip_and_rename_lar.sh 3 | bash download_scripts/unzip_ts.sh 4 | bash download_scripts/unzip_panel.sh -------------------------------------------------------------------------------- /download_scripts/unzip_and_rename_lar.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | #This file unzips and renames the LAR files in the data/lar directory. 4 | #unzip and rename files 5 | unzip data/lar/lar_2022.zip -d data/lar 6 | unzip data/lar/lar_2021.zip -d data/lar 7 | unzip data/lar/lar_2020.zip -d data/lar 8 | unzip data/lar/lar_2019.zip -d data/lar 9 | unzip data/lar/lar_2018.zip -d data/lar 10 | unzip data/lar/lar_2017.zip -d data/lar 11 | unzip data/lar/lar_2016.zip -d data/lar 12 | unzip data/lar/lar_2015.zip -d data/lar 13 | unzip data/lar/lar_2014.zip -d data/lar 14 | unzip data/lar/lar_2013.zip -d data/lar 15 | unzip data/lar/lar_2012.zip -d data/lar 16 | unzip data/lar/lar_2011.zip -d data/lar 17 | unzip data/lar/lar_2010.zip -d data/lar 18 | unzip data/lar/lar_2009.zip -d data/lar 19 | unzip data/lar/lar_2008.zip -d data/lar 20 | unzip data/lar/lar_2007.zip -d data/lar 21 | unzip data/lar/lar_2006.zip -d data/lar 22 | unzip data/lar/lar_2005.zip -d data/lar 23 | unzip data/lar/lar_2004.zip -d data/lar 24 | 25 | mv data/lar/2022_public_lar_pipe.txt data/lar/lar_2022.txt 26 | mv data/lar/2021_public_lar_psv.txt data/lar/lar_2021.txt 27 | mv data/lar/2020_lar_pipe.txt data/lar/lar_2020.txt 28 | mv data/lar/2019_public_lar_three_year_pipe.txt data/lar/lar_2019.txt 29 | mv data/lar/2018_public__lar_three_year_psv.txt data/lar/lar_2018.txt 30 | mv data/lar/2017_public_lar_three_year.txt data/lar/lar_2017.txt 31 | mv data/lar/2016HMDALAR\ -\ National.csv data/lar/lar_2016.csv 32 | mv data/lar/2015HMDALAR\ -\ National.csv data/lar/lar_2015.csv 33 | mv data/lar/Lars_ultimate_2014.txt data/lar/lar_2014.txt 34 | mv data/lar/Lars.ultimate.2013.dat data/lar/lar_2013.dat 35 | mv data/lar/Lars.ultimate.2012.dat data/lar/lar_2012.dat 36 | mv data/lar/Lars.ultimate.2011.dat data/lar/lar_2011.dat 37 | mv data/lar/Lars.ultimate.2010.dat data/lar/lar_2010.dat 38 | mv data/lar/2009_Ultimate_PUBLIC_LAR.dat data/lar/lar_2009.dat 39 | mv data/lar/Lars.ultimate.2008.dat data/lar/lar_2008.dat 40 | mv data/lar/Lars.ultimate.2007.dat data/lar/lar_2007.dat 41 | mv data/lar/Lars.ultimate.2006.dat data/lar/lar_2006.dat 42 | mv data/lar/LARS.ULTIMATE.2005.DAT data/lar/lar_2005.dat 43 | mv data/lar/u2004lar.public.dat data/lar/lar_2004.dat 44 | -------------------------------------------------------------------------------- /download_scripts/unzip_panel.sh: -------------------------------------------------------------------------------- 1 | #!bin/bash 2 | #This file unzips the compressed Panel files in the data/panel directory. 3 | echo "Unzipping Panel files" 4 | for f in data/panel/*.zip; do unzip -p "$f" > "${f%.zip}.txt"; done 5 | -------------------------------------------------------------------------------- /download_scripts/unzip_ts.sh: -------------------------------------------------------------------------------- 1 | #!bin/bash 2 | #this file unzips the compressed Transmittal Sheet files in the data/ts directory 3 | echo "Unzipping TS files" 4 | for f in data/ts/*.zip; do unzip -p "$f" > "${f%.zip}.txt"; done -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_lar_2004.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.lar_2004; 2 | CREATE TABLE hmda_public.lar_2004( 3 | activity_year VARCHAR, 4 | respondent_id VARCHAR, 5 | agency_code VARCHAR, 6 | loan_type VARCHAR, 7 | loan_purpose VARCHAR, 8 | occupancy_type VARCHAR, 9 | loan_amount VARCHAR, 10 | action_taken VARCHAR, 11 | msa_md VARCHAR, 12 | state_code VARCHAR, 13 | county_code VARCHAR, 14 | census_tract VARCHAR, 15 | applicant_sex VARCHAR, 16 | co_applicant_sex VARCHAR, 17 | income VARCHAR, 18 | purchaser_type VARCHAR, 19 | denial_reason_1 VARCHAR, 20 | denial_reason_2 VARCHAR, 21 | denial_reason_3 VARCHAR, 22 | edit_status VARCHAR, 23 | property_type VARCHAR, 24 | preapproval VARCHAR, 25 | applicant_ethnicity VARCHAR, 26 | co_applicant_ethnicity VARCHAR, 27 | applicant_race_1 VARCHAR, 28 | applicant_race_2 VARCHAR, 29 | applicant_race_3 VARCHAR, 30 | applicant_race_4 VARCHAR, 31 | applicant_race_5 VARCHAR, 32 | co_applicant_race_1 VARCHAR, 33 | co_applicant_race_2 VARCHAR, 34 | co_applicant_race_3 VARCHAR, 35 | co_applicant_race_4 VARCHAR, 36 | co_applicant_race_5 VARCHAR, 37 | rate_spread VARCHAR, 38 | hoepa_status VARCHAR, 39 | lien_status VARCHAR, 40 | sequence_number VARCHAR 41 | ); 42 | 43 | COMMIT; 44 | 45 | CREATE TEMPORARY TABLE lar_load 46 | (LAR VARCHAR); -- LAR contains an entire LAR record 47 | 48 | COPY lar_load 49 | -- Change this path to your local data path. 50 | FROM '{data_path}/data/lar/lar_2004.dat'; 51 | 52 | COMMIT; 53 | 54 | INSERT INTO hmda_public.lar_2004 ( 55 | activity_year, 56 | respondent_id, 57 | agency_code, 58 | loan_type, 59 | loan_purpose, 60 | occupancy_type, 61 | loan_amount, 62 | action_taken, 63 | msa_md, 64 | state_code, 65 | county_code, 66 | census_tract, 67 | applicant_sex, 68 | co_applicant_sex, 69 | income, 70 | purchaser_type, 71 | denial_reason_1, 72 | denial_reason_2, 73 | denial_reason_3, 74 | edit_status, 75 | property_type, 76 | preapproval, 77 | applicant_ethnicity, 78 | co_applicant_ethnicity, 79 | applicant_race_1, 80 | applicant_race_2, 81 | applicant_race_3, 82 | applicant_race_4, 83 | applicant_race_5, 84 | co_applicant_race_1, 85 | co_applicant_race_2, 86 | co_applicant_race_3, 87 | co_applicant_race_4, 88 | co_applicant_race_5, 89 | rate_spread, 90 | hoepa_status, 91 | lien_status, 92 | sequence_number 93 | ) 94 | 95 | SELECT 96 | SUBSTRING(LAR, 1,4), 97 | SUBSTRING(LAR, 5,10), 98 | SUBSTRING(LAR, 15,1), 99 | SUBSTRING(LAR, 16,1), 100 | SUBSTRING(LAR, 17,1), 101 | SUBSTRING(LAR, 18,1), 102 | SUBSTRING(LAR, 19,5), 103 | SUBSTRING(LAR, 24,1), 104 | SUBSTRING(LAR, 25,5), 105 | SUBSTRING(LAR, 30,2), 106 | SUBSTRING(LAR, 32,3), 107 | SUBSTRING(LAR, 35,7), 108 | SUBSTRING(LAR, 42,1), 109 | SUBSTRING(LAR, 43,1), 110 | SUBSTRING(LAR, 44,4), 111 | SUBSTRING(LAR, 48,1), 112 | SUBSTRING(LAR, 49,1), 113 | SUBSTRING(LAR, 50,1), 114 | SUBSTRING(LAR, 51,1), 115 | SUBSTRING(LAR, 52,1), 116 | SUBSTRING(LAR, 53,1), 117 | SUBSTRING(LAR, 54,1), 118 | SUBSTRING(LAR, 55,1), 119 | SUBSTRING(LAR, 56,1), 120 | SUBSTRING(LAR, 57,1), 121 | SUBSTRING(LAR, 58,1), 122 | SUBSTRING(LAR, 59,1), 123 | SUBSTRING(LAR, 60,1), 124 | SUBSTRING(LAR, 61,1), 125 | SUBSTRING(LAR, 62,1), 126 | SUBSTRING(LAR, 63,1), 127 | SUBSTRING(LAR, 64,1), 128 | SUBSTRING(LAR, 65,1), 129 | SUBSTRING(LAR, 66,1), 130 | SUBSTRING(LAR, 67,5), 131 | SUBSTRING(LAR, 72,1), 132 | SUBSTRING(LAR, 73,1), 133 | SUBSTRING(LAR, 74,7) 134 | 135 | FROM lar_load; 136 | COMMIT; 137 | DROP TABLE IF EXISTS lar_load; 138 | COMMIT; 139 | 140 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_lar_2005.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.lar_2005; 2 | CREATE TABLE hmda_public.lar_2005( 3 | activity_year VARCHAR, 4 | respondent_id VARCHAR, 5 | agency_code VARCHAR, 6 | loan_type VARCHAR, 7 | loan_purpose VARCHAR, 8 | occupancy_type VARCHAR, 9 | loan_amount VARCHAR, 10 | action_taken VARCHAR, 11 | msa_md VARCHAR, 12 | state_code VARCHAR, 13 | county_code VARCHAR, 14 | census_tract VARCHAR, 15 | applicant_sex VARCHAR, 16 | co_applicant_sex VARCHAR, 17 | income VARCHAR, 18 | purchaser_type VARCHAR, 19 | denial_reason_1 VARCHAR, 20 | denial_reason_2 VARCHAR, 21 | denial_reason_3 VARCHAR, 22 | edit_status VARCHAR, 23 | property_type VARCHAR, 24 | preapproval VARCHAR, 25 | applicant_ethnicity VARCHAR, 26 | co_applicant_ethnicity VARCHAR, 27 | applicant_race_1 VARCHAR, 28 | applicant_race_2 VARCHAR, 29 | applicant_race_3 VARCHAR, 30 | applicant_race_4 VARCHAR, 31 | applicant_race_5 VARCHAR, 32 | co_applicant_race_1 VARCHAR, 33 | co_applicant_race_2 VARCHAR, 34 | co_applicant_race_3 VARCHAR, 35 | co_applicant_race_4 VARCHAR, 36 | co_applicant_race_5 VARCHAR, 37 | rate_spread VARCHAR, 38 | hoepa_status VARCHAR, 39 | lien_status VARCHAR, 40 | sequence_number VARCHAR 41 | ); 42 | 43 | CREATE TEMPORARY TABLE lar_load 44 | (LAR VARCHAR); -- LAR contains an entire LAR record 45 | 46 | COPY lar_load 47 | -- Change this path to your local data path. 48 | FROM '{data_path}/data/lar/lar_2005.dat'; 49 | 50 | COMMIT; 51 | 52 | INSERT INTO hmda_public.lar_2005 ( 53 | activity_year, 54 | respondent_id, 55 | agency_code, 56 | loan_type, 57 | loan_purpose, 58 | occupancy_type, 59 | loan_amount, 60 | action_taken, 61 | msa_md, 62 | state_code, 63 | county_code, 64 | census_tract, 65 | applicant_sex, 66 | co_applicant_sex, 67 | income, 68 | purchaser_type, 69 | denial_reason_1, 70 | denial_reason_2, 71 | denial_reason_3, 72 | edit_status, 73 | property_type, 74 | preapproval, 75 | applicant_ethnicity, 76 | co_applicant_ethnicity, 77 | applicant_race_1, 78 | applicant_race_2, 79 | applicant_race_3, 80 | applicant_race_4, 81 | applicant_race_5, 82 | co_applicant_race_1, 83 | co_applicant_race_2, 84 | co_applicant_race_3, 85 | co_applicant_race_4, 86 | co_applicant_race_5, 87 | rate_spread, 88 | hoepa_status, 89 | lien_status, 90 | sequence_number 91 | ) 92 | 93 | SELECT 94 | SUBSTRING(LAR, 1,4), 95 | SUBSTRING(LAR, 5,10), 96 | SUBSTRING(LAR, 15,1), 97 | SUBSTRING(LAR, 16,1), 98 | SUBSTRING(LAR, 17,1), 99 | SUBSTRING(LAR, 18,1), 100 | SUBSTRING(LAR, 19,5), 101 | SUBSTRING(LAR, 24,1), 102 | SUBSTRING(LAR, 25,5), 103 | SUBSTRING(LAR, 30,2), 104 | SUBSTRING(LAR, 32,3), 105 | SUBSTRING(LAR, 35,7), 106 | SUBSTRING(LAR, 42,1), 107 | SUBSTRING(LAR, 43,1), 108 | SUBSTRING(LAR, 44,4), 109 | SUBSTRING(LAR, 48,1), 110 | SUBSTRING(LAR, 49,1), 111 | SUBSTRING(LAR, 50,1), 112 | SUBSTRING(LAR, 51,1), 113 | SUBSTRING(LAR, 52,1), 114 | SUBSTRING(LAR, 53,1), 115 | SUBSTRING(LAR, 54,1), 116 | SUBSTRING(LAR, 55,1), 117 | SUBSTRING(LAR, 56,1), 118 | SUBSTRING(LAR, 57,1), 119 | SUBSTRING(LAR, 58,1), 120 | SUBSTRING(LAR, 59,1), 121 | SUBSTRING(LAR, 60,1), 122 | SUBSTRING(LAR, 61,1), 123 | SUBSTRING(LAR, 62,1), 124 | SUBSTRING(LAR, 63,1), 125 | SUBSTRING(LAR, 64,1), 126 | SUBSTRING(LAR, 65,1), 127 | SUBSTRING(LAR, 66,1), 128 | SUBSTRING(LAR, 67,5), 129 | SUBSTRING(LAR, 72,1), 130 | SUBSTRING(LAR, 73,1), 131 | SUBSTRING(LAR, 74,7) 132 | 133 | FROM lar_load; 134 | COMMIT; 135 | DROP TABLE IF EXISTS lar_load; 136 | COMMIT; 137 | 138 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_lar_2006.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.lar_2006; 2 | CREATE TABLE hmda_public.lar_2006( 3 | activity_year VARCHAR, 4 | respondent_id VARCHAR, 5 | agency_code VARCHAR, 6 | loan_type VARCHAR, 7 | loan_purpose VARCHAR, 8 | occupancy_type VARCHAR, 9 | loan_amount VARCHAR, 10 | action_taken VARCHAR, 11 | msa_md VARCHAR, 12 | state_code VARCHAR, 13 | county_code VARCHAR, 14 | census_tract VARCHAR, 15 | applicant_sex VARCHAR, 16 | co_applicant_sex VARCHAR, 17 | income VARCHAR, 18 | purchaser_type VARCHAR, 19 | denial_reason_1 VARCHAR, 20 | denial_reason_2 VARCHAR, 21 | denial_reason_3 VARCHAR, 22 | edit_status VARCHAR, 23 | property_type VARCHAR, 24 | preapproval VARCHAR, 25 | applicant_ethnicity VARCHAR, 26 | co_applicant_ethnicity VARCHAR, 27 | applicant_race_1 VARCHAR, 28 | applicant_race_2 VARCHAR, 29 | applicant_race_3 VARCHAR, 30 | applicant_race_4 VARCHAR, 31 | applicant_race_5 VARCHAR, 32 | co_applicant_race_1 VARCHAR, 33 | co_applicant_race_2 VARCHAR, 34 | co_applicant_race_3 VARCHAR, 35 | co_applicant_race_4 VARCHAR, 36 | co_applicant_race_5 VARCHAR, 37 | rate_spread VARCHAR, 38 | hoepa_status VARCHAR, 39 | lien_status VARCHAR, 40 | sequence_number VARCHAR 41 | ); 42 | 43 | COMMIT; 44 | 45 | CREATE TEMPORARY TABLE lar_load 46 | (LAR VARCHAR); -- LAR contains an entire LAR record 47 | 48 | COPY lar_load 49 | -- Change this path to your local data path. 50 | FROM '{data_path}/data/lar/lar_2006.dat'; 51 | 52 | COMMIT; 53 | 54 | INSERT INTO hmda_public.lar_2006 ( 55 | activity_year, 56 | respondent_id, 57 | agency_code, 58 | loan_type, 59 | loan_purpose, 60 | occupancy_type, 61 | loan_amount, 62 | action_taken, 63 | msa_md, 64 | state_code, 65 | county_code, 66 | census_tract, 67 | applicant_sex, 68 | co_applicant_sex, 69 | income, 70 | purchaser_type, 71 | denial_reason_1, 72 | denial_reason_2, 73 | denial_reason_3, 74 | edit_status, 75 | property_type, 76 | preapproval, 77 | applicant_ethnicity, 78 | co_applicant_ethnicity, 79 | applicant_race_1, 80 | applicant_race_2, 81 | applicant_race_3, 82 | applicant_race_4, 83 | applicant_race_5, 84 | co_applicant_race_1, 85 | co_applicant_race_2, 86 | co_applicant_race_3, 87 | co_applicant_race_4, 88 | co_applicant_race_5, 89 | rate_spread, 90 | hoepa_status, 91 | lien_status, 92 | sequence_number 93 | ) 94 | 95 | SELECT 96 | SUBSTRING(LAR, 1,4), 97 | SUBSTRING(LAR, 5,10), 98 | SUBSTRING(LAR, 15,1), 99 | SUBSTRING(LAR, 16,1), 100 | SUBSTRING(LAR, 17,1), 101 | SUBSTRING(LAR, 18,1), 102 | SUBSTRING(LAR, 19,5), 103 | SUBSTRING(LAR, 24,1), 104 | SUBSTRING(LAR, 25,5), 105 | SUBSTRING(LAR, 30,2), 106 | SUBSTRING(LAR, 32,3), 107 | SUBSTRING(LAR, 35,7), 108 | SUBSTRING(LAR, 42,1), 109 | SUBSTRING(LAR, 43,1), 110 | SUBSTRING(LAR, 44,4), 111 | SUBSTRING(LAR, 48,1), 112 | SUBSTRING(LAR, 49,1), 113 | SUBSTRING(LAR, 50,1), 114 | SUBSTRING(LAR, 51,1), 115 | SUBSTRING(LAR, 52,1), 116 | SUBSTRING(LAR, 53,1), 117 | SUBSTRING(LAR, 54,1), 118 | SUBSTRING(LAR, 55,1), 119 | SUBSTRING(LAR, 56,1), 120 | SUBSTRING(LAR, 57,1), 121 | SUBSTRING(LAR, 58,1), 122 | SUBSTRING(LAR, 59,1), 123 | SUBSTRING(LAR, 60,1), 124 | SUBSTRING(LAR, 61,1), 125 | SUBSTRING(LAR, 62,1), 126 | SUBSTRING(LAR, 63,1), 127 | SUBSTRING(LAR, 64,1), 128 | SUBSTRING(LAR, 65,1), 129 | SUBSTRING(LAR, 66,1), 130 | SUBSTRING(LAR, 67,5), 131 | SUBSTRING(LAR, 72,1), 132 | SUBSTRING(LAR, 73,1), 133 | SUBSTRING(LAR, 74,7) 134 | 135 | FROM lar_load; 136 | COMMIT; 137 | DROP TABLE IF EXISTS lar_load; 138 | COMMIT; 139 | 140 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_lar_2007.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.lar_2007; 2 | CREATE TABLE hmda_public.lar_2007( 3 | activity_year VARCHAR, 4 | respondent_id VARCHAR, 5 | agency_code VARCHAR, 6 | loan_type VARCHAR, 7 | loan_purpose VARCHAR, 8 | occupancy_type VARCHAR, 9 | loan_amount VARCHAR, 10 | action_taken VARCHAR, 11 | msa_md VARCHAR, 12 | state_code VARCHAR, 13 | county_code VARCHAR, 14 | census_tract VARCHAR, 15 | applicant_sex VARCHAR, 16 | co_applicant_sex VARCHAR, 17 | income VARCHAR, 18 | purchaser_type VARCHAR, 19 | denial_reason_1 VARCHAR, 20 | denial_reason_2 VARCHAR, 21 | denial_reason_3 VARCHAR, 22 | edit_status VARCHAR, 23 | property_type VARCHAR, 24 | preapproval VARCHAR, 25 | applicant_ethnicity VARCHAR, 26 | co_applicant_ethnicity VARCHAR, 27 | applicant_race_1 VARCHAR, 28 | applicant_race_2 VARCHAR, 29 | applicant_race_3 VARCHAR, 30 | applicant_race_4 VARCHAR, 31 | applicant_race_5 VARCHAR, 32 | co_applicant_race_1 VARCHAR, 33 | co_applicant_race_2 VARCHAR, 34 | co_applicant_race_3 VARCHAR, 35 | co_applicant_race_4 VARCHAR, 36 | co_applicant_race_5 VARCHAR, 37 | rate_spread VARCHAR, 38 | hoepa_status VARCHAR, 39 | lien_status VARCHAR, 40 | sequence_number VARCHAR 41 | ); 42 | 43 | COMMIT; 44 | 45 | CREATE TEMPORARY TABLE lar_load 46 | (LAR VARCHAR); -- LAR contains an entire LAR record 47 | 48 | COPY lar_load 49 | -- Change this path to your local data path. 50 | FROM '{data_path}/data/lar/lar_2007.dat'; 51 | 52 | COMMIT; 53 | 54 | INSERT INTO hmda_public.lar_2007 ( 55 | activity_year, 56 | respondent_id, 57 | agency_code, 58 | loan_type, 59 | loan_purpose, 60 | occupancy_type, 61 | loan_amount, 62 | action_taken, 63 | msa_md, 64 | state_code, 65 | county_code, 66 | census_tract, 67 | applicant_sex, 68 | co_applicant_sex, 69 | income, 70 | purchaser_type, 71 | denial_reason_1, 72 | denial_reason_2, 73 | denial_reason_3, 74 | edit_status, 75 | property_type, 76 | preapproval, 77 | applicant_ethnicity, 78 | co_applicant_ethnicity, 79 | applicant_race_1, 80 | applicant_race_2, 81 | applicant_race_3, 82 | applicant_race_4, 83 | applicant_race_5, 84 | co_applicant_race_1, 85 | co_applicant_race_2, 86 | co_applicant_race_3, 87 | co_applicant_race_4, 88 | co_applicant_race_5, 89 | rate_spread, 90 | hoepa_status, 91 | lien_status, 92 | sequence_number 93 | ) 94 | 95 | SELECT 96 | SUBSTRING(LAR, 1,4), 97 | SUBSTRING(LAR, 5,10), 98 | SUBSTRING(LAR, 15,1), 99 | SUBSTRING(LAR, 16,1), 100 | SUBSTRING(LAR, 17,1), 101 | SUBSTRING(LAR, 18,1), 102 | SUBSTRING(LAR, 19,5), 103 | SUBSTRING(LAR, 24,1), 104 | SUBSTRING(LAR, 25,5), 105 | SUBSTRING(LAR, 30,2), 106 | SUBSTRING(LAR, 32,3), 107 | SUBSTRING(LAR, 35,7), 108 | SUBSTRING(LAR, 42,1), 109 | SUBSTRING(LAR, 43,1), 110 | SUBSTRING(LAR, 44,4), 111 | SUBSTRING(LAR, 48,1), 112 | SUBSTRING(LAR, 49,1), 113 | SUBSTRING(LAR, 50,1), 114 | SUBSTRING(LAR, 51,1), 115 | SUBSTRING(LAR, 52,1), 116 | SUBSTRING(LAR, 53,1), 117 | SUBSTRING(LAR, 54,1), 118 | SUBSTRING(LAR, 55,1), 119 | SUBSTRING(LAR, 56,1), 120 | SUBSTRING(LAR, 57,1), 121 | SUBSTRING(LAR, 58,1), 122 | SUBSTRING(LAR, 59,1), 123 | SUBSTRING(LAR, 60,1), 124 | SUBSTRING(LAR, 61,1), 125 | SUBSTRING(LAR, 62,1), 126 | SUBSTRING(LAR, 63,1), 127 | SUBSTRING(LAR, 64,1), 128 | SUBSTRING(LAR, 65,1), 129 | SUBSTRING(LAR, 66,1), 130 | SUBSTRING(LAR, 67,5), 131 | SUBSTRING(LAR, 72,1), 132 | SUBSTRING(LAR, 73,1), 133 | SUBSTRING(LAR, 74,7) 134 | 135 | FROM lar_load; 136 | COMMIT; 137 | DROP TABLE IF EXISTS lar_load; 138 | COMMIT; 139 | 140 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_lar_2008.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.lar_2008; 2 | CREATE TABLE hmda_public.lar_2008( 3 | activity_year VARCHAR, 4 | respondent_id VARCHAR, 5 | agency_code VARCHAR, 6 | loan_type VARCHAR, 7 | loan_purpose VARCHAR, 8 | occupancy_type VARCHAR, 9 | loan_amount VARCHAR, 10 | action_taken VARCHAR, 11 | msa_md VARCHAR, 12 | state_code VARCHAR, 13 | county_code VARCHAR, 14 | census_tract VARCHAR, 15 | applicant_sex VARCHAR, 16 | co_applicant_sex VARCHAR, 17 | income VARCHAR, 18 | purchaser_type VARCHAR, 19 | denial_reason_1 VARCHAR, 20 | denial_reason_2 VARCHAR, 21 | denial_reason_3 VARCHAR, 22 | edit_status VARCHAR, 23 | property_type VARCHAR, 24 | preapproval VARCHAR, 25 | applicant_ethnicity VARCHAR, 26 | co_applicant_ethnicity VARCHAR, 27 | applicant_race_1 VARCHAR, 28 | applicant_race_2 VARCHAR, 29 | applicant_race_3 VARCHAR, 30 | applicant_race_4 VARCHAR, 31 | applicant_race_5 VARCHAR, 32 | co_applicant_race_1 VARCHAR, 33 | co_applicant_race_2 VARCHAR, 34 | co_applicant_race_3 VARCHAR, 35 | co_applicant_race_4 VARCHAR, 36 | co_applicant_race_5 VARCHAR, 37 | rate_spread VARCHAR, 38 | hoepa_status VARCHAR, 39 | lien_status VARCHAR, 40 | sequence_number VARCHAR 41 | ); 42 | 43 | CREATE TEMPORARY TABLE lar_load 44 | (LAR VARCHAR); -- LAR contains an entire LAR record 45 | 46 | COPY lar_load 47 | -- Change this path to your local data path. 48 | FROM '{data_path}/data/lar/lar_2008.dat'; 49 | 50 | COMMIT; 51 | 52 | INSERT INTO hmda_public.lar_2008 ( 53 | activity_year, 54 | respondent_id, 55 | agency_code, 56 | loan_type, 57 | loan_purpose, 58 | occupancy_type, 59 | loan_amount, 60 | action_taken, 61 | msa_md, 62 | state_code, 63 | county_code, 64 | census_tract, 65 | applicant_sex, 66 | co_applicant_sex, 67 | income, 68 | purchaser_type, 69 | denial_reason_1, 70 | denial_reason_2, 71 | denial_reason_3, 72 | edit_status, 73 | property_type, 74 | preapproval, 75 | applicant_ethnicity, 76 | co_applicant_ethnicity, 77 | applicant_race_1, 78 | applicant_race_2, 79 | applicant_race_3, 80 | applicant_race_4, 81 | applicant_race_5, 82 | co_applicant_race_1, 83 | co_applicant_race_2, 84 | co_applicant_race_3, 85 | co_applicant_race_4, 86 | co_applicant_race_5, 87 | rate_spread, 88 | hoepa_status, 89 | lien_status, 90 | sequence_number 91 | ) 92 | 93 | SELECT 94 | SUBSTRING(LAR, 1,4), 95 | SUBSTRING(LAR, 5,10), 96 | SUBSTRING(LAR, 15,1), 97 | SUBSTRING(LAR, 16,1), 98 | SUBSTRING(LAR, 17,1), 99 | SUBSTRING(LAR, 18,1), 100 | SUBSTRING(LAR, 19,5), 101 | SUBSTRING(LAR, 24,1), 102 | SUBSTRING(LAR, 25,5), 103 | SUBSTRING(LAR, 30,2), 104 | SUBSTRING(LAR, 32,3), 105 | SUBSTRING(LAR, 35,7), 106 | SUBSTRING(LAR, 42,1), 107 | SUBSTRING(LAR, 43,1), 108 | SUBSTRING(LAR, 44,4), 109 | SUBSTRING(LAR, 48,1), 110 | SUBSTRING(LAR, 49,1), 111 | SUBSTRING(LAR, 50,1), 112 | SUBSTRING(LAR, 51,1), 113 | SUBSTRING(LAR, 52,1), 114 | SUBSTRING(LAR, 53,1), 115 | SUBSTRING(LAR, 54,1), 116 | SUBSTRING(LAR, 55,1), 117 | SUBSTRING(LAR, 56,1), 118 | SUBSTRING(LAR, 57,1), 119 | SUBSTRING(LAR, 58,1), 120 | SUBSTRING(LAR, 59,1), 121 | SUBSTRING(LAR, 60,1), 122 | SUBSTRING(LAR, 61,1), 123 | SUBSTRING(LAR, 62,1), 124 | SUBSTRING(LAR, 63,1), 125 | SUBSTRING(LAR, 64,1), 126 | SUBSTRING(LAR, 65,1), 127 | SUBSTRING(LAR, 66,1), 128 | SUBSTRING(LAR, 67,5), 129 | SUBSTRING(LAR, 72,1), 130 | SUBSTRING(LAR, 73,1), 131 | SUBSTRING(LAR, 74,7) 132 | 133 | FROM lar_load; 134 | COMMIT; 135 | DROP TABLE IF EXISTS lar_load; 136 | COMMIT; 137 | 138 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_lar_2009.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.lar_2009; 2 | 3 | CREATE TABLE hmda_public.lar_2009( 4 | activity_year VARCHAR, 5 | respondent_id VARCHAR, 6 | agency_code VARCHAR, 7 | loan_type VARCHAR, 8 | loan_purpose VARCHAR, 9 | occupancy_type VARCHAR, 10 | loan_amount VARCHAR, 11 | action_taken VARCHAR, 12 | msa_md VARCHAR, 13 | state_code VARCHAR, 14 | county_code VARCHAR, 15 | census_tract VARCHAR, 16 | applicant_sex VARCHAR, 17 | co_applicant_sex VARCHAR, 18 | income VARCHAR, 19 | purchaser_type VARCHAR, 20 | denial_reason_1 VARCHAR, 21 | denial_reason_2 VARCHAR, 22 | denial_reason_3 VARCHAR, 23 | edit_status VARCHAR, 24 | property_type VARCHAR, 25 | preapproval VARCHAR, 26 | applicant_ethnicity VARCHAR, 27 | co_applicant_ethnicity VARCHAR, 28 | applicant_race_1 VARCHAR, 29 | applicant_race_2 VARCHAR, 30 | applicant_race_3 VARCHAR, 31 | applicant_race_4 VARCHAR, 32 | applicant_race_5 VARCHAR, 33 | co_applicant_race_1 VARCHAR, 34 | co_applicant_race_2 VARCHAR, 35 | co_applicant_race_3 VARCHAR, 36 | co_applicant_race_4 VARCHAR, 37 | co_applicant_race_5 VARCHAR, 38 | rate_spread VARCHAR, 39 | hoepa_status VARCHAR, 40 | lien_status VARCHAR, 41 | sequence_number VARCHAR 42 | ); 43 | 44 | CREATE TEMPORARY TABLE lar_load 45 | (LAR VARCHAR); -- LAR contains an entire LAR record 46 | 47 | COPY lar_load 48 | -- Change this path to your local data path. 49 | FROM '{data_path}/data/lar/lar_2009.dat'; 50 | 51 | COMMIT; 52 | 53 | INSERT INTO hmda_public.lar_2009 ( 54 | activity_year, 55 | respondent_id, 56 | agency_code, 57 | loan_type, 58 | loan_purpose, 59 | occupancy_type, 60 | loan_amount, 61 | action_taken, 62 | msa_md, 63 | state_code, 64 | county_code, 65 | census_tract, 66 | applicant_sex, 67 | co_applicant_sex, 68 | income, 69 | purchaser_type, 70 | denial_reason_1, 71 | denial_reason_2, 72 | denial_reason_3, 73 | edit_status, 74 | property_type, 75 | preapproval, 76 | applicant_ethnicity, 77 | co_applicant_ethnicity, 78 | applicant_race_1, 79 | applicant_race_2, 80 | applicant_race_3, 81 | applicant_race_4, 82 | applicant_race_5, 83 | co_applicant_race_1, 84 | co_applicant_race_2, 85 | co_applicant_race_3, 86 | co_applicant_race_4, 87 | co_applicant_race_5, 88 | rate_spread, 89 | hoepa_status, 90 | lien_status, 91 | sequence_number 92 | ) 93 | 94 | SELECT 95 | SUBSTRING(LAR, 1,4), 96 | SUBSTRING(LAR, 5,10), 97 | SUBSTRING(LAR, 15,1), 98 | SUBSTRING(LAR, 16,1), 99 | SUBSTRING(LAR, 17,1), 100 | SUBSTRING(LAR, 18,1), 101 | SUBSTRING(LAR, 19,5), 102 | SUBSTRING(LAR, 24,1), 103 | SUBSTRING(LAR, 25,5), 104 | SUBSTRING(LAR, 30,2), 105 | SUBSTRING(LAR, 32,3), 106 | SUBSTRING(LAR, 35,7), 107 | SUBSTRING(LAR, 42,1), 108 | SUBSTRING(LAR, 43,1), 109 | SUBSTRING(LAR, 44,4), 110 | SUBSTRING(LAR, 48,1), 111 | SUBSTRING(LAR, 49,1), 112 | SUBSTRING(LAR, 50,1), 113 | SUBSTRING(LAR, 51,1), 114 | SUBSTRING(LAR, 52,1), 115 | SUBSTRING(LAR, 53,1), 116 | SUBSTRING(LAR, 54,1), 117 | SUBSTRING(LAR, 55,1), 118 | SUBSTRING(LAR, 56,1), 119 | SUBSTRING(LAR, 57,1), 120 | SUBSTRING(LAR, 58,1), 121 | SUBSTRING(LAR, 59,1), 122 | SUBSTRING(LAR, 60,1), 123 | SUBSTRING(LAR, 61,1), 124 | SUBSTRING(LAR, 62,1), 125 | SUBSTRING(LAR, 63,1), 126 | SUBSTRING(LAR, 64,1), 127 | SUBSTRING(LAR, 65,1), 128 | SUBSTRING(LAR, 66,1), 129 | SUBSTRING(LAR, 67,5), 130 | SUBSTRING(LAR, 72,1), 131 | SUBSTRING(LAR, 73,1), 132 | SUBSTRING(LAR, 74,7) 133 | 134 | FROM lar_load; 135 | COMMIT; 136 | DROP TABLE IF EXISTS lar_load; 137 | COMMIT; 138 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_lar_2010.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.lar_2010; 2 | CREATE TABLE hmda_public.lar_2010( 3 | activity_year VARCHAR, 4 | respondent_id VARCHAR, 5 | agency_code VARCHAR, 6 | loan_type VARCHAR, 7 | loan_purpose VARCHAR, 8 | occupancy_type VARCHAR, 9 | loan_amount VARCHAR, 10 | action_taken VARCHAR, 11 | msa_md VARCHAR, 12 | state_code VARCHAR, 13 | county_code VARCHAR, 14 | census_tract VARCHAR, 15 | applicant_sex VARCHAR, 16 | co_applicant_sex VARCHAR, 17 | income VARCHAR, 18 | purchaser_type VARCHAR, 19 | denial_reason_1 VARCHAR, 20 | denial_reason_2 VARCHAR, 21 | denial_reason_3 VARCHAR, 22 | edit_status VARCHAR, 23 | property_type VARCHAR, 24 | preapproval VARCHAR, 25 | applicant_ethnicity VARCHAR, 26 | co_applicant_ethnicity VARCHAR, 27 | applicant_race_1 VARCHAR, 28 | applicant_race_2 VARCHAR, 29 | applicant_race_3 VARCHAR, 30 | applicant_race_4 VARCHAR, 31 | applicant_race_5 VARCHAR, 32 | co_applicant_race_1 VARCHAR, 33 | co_applicant_race_2 VARCHAR, 34 | co_applicant_race_3 VARCHAR, 35 | co_applicant_race_4 VARCHAR, 36 | co_applicant_race_5 VARCHAR, 37 | rate_spread VARCHAR, 38 | hoepa_status VARCHAR, 39 | lien_status VARCHAR, 40 | sequence_number VARCHAR 41 | ); 42 | 43 | CREATE TEMPORARY TABLE lar_load 44 | (LAR VARCHAR); -- LAR contains an entire LAR record 45 | 46 | COPY lar_load 47 | -- Change this path to your local data path. 48 | FROM '{data_path}/data/lar/lar_2010.dat'; 49 | 50 | COMMIT; 51 | 52 | INSERT INTO hmda_public.lar_2010 ( 53 | activity_year, 54 | respondent_id, 55 | agency_code, 56 | loan_type, 57 | loan_purpose, 58 | occupancy_type, 59 | loan_amount, 60 | action_taken, 61 | msa_md, 62 | state_code, 63 | county_code, 64 | census_tract, 65 | applicant_sex, 66 | co_applicant_sex, 67 | income, 68 | purchaser_type, 69 | denial_reason_1, 70 | denial_reason_2, 71 | denial_reason_3, 72 | edit_status, 73 | property_type, 74 | preapproval, 75 | applicant_ethnicity, 76 | co_applicant_ethnicity, 77 | applicant_race_1, 78 | applicant_race_2, 79 | applicant_race_3, 80 | applicant_race_4, 81 | applicant_race_5, 82 | co_applicant_race_1, 83 | co_applicant_race_2, 84 | co_applicant_race_3, 85 | co_applicant_race_4, 86 | co_applicant_race_5, 87 | rate_spread, 88 | hoepa_status, 89 | lien_status, 90 | sequence_number 91 | ) 92 | 93 | SELECT 94 | SUBSTRING(LAR, 1,4), 95 | SUBSTRING(LAR, 5,10), 96 | SUBSTRING(LAR, 15,1), 97 | SUBSTRING(LAR, 16,1), 98 | SUBSTRING(LAR, 17,1), 99 | SUBSTRING(LAR, 18,1), 100 | SUBSTRING(LAR, 19,5), 101 | SUBSTRING(LAR, 24,1), 102 | SUBSTRING(LAR, 25,5), 103 | SUBSTRING(LAR, 30,2), 104 | SUBSTRING(LAR, 32,3), 105 | SUBSTRING(LAR, 35,7), 106 | SUBSTRING(LAR, 42,1), 107 | SUBSTRING(LAR, 43,1), 108 | SUBSTRING(LAR, 44,4), 109 | SUBSTRING(LAR, 48,1), 110 | SUBSTRING(LAR, 49,1), 111 | SUBSTRING(LAR, 50,1), 112 | SUBSTRING(LAR, 51,1), 113 | SUBSTRING(LAR, 52,1), 114 | SUBSTRING(LAR, 53,1), 115 | SUBSTRING(LAR, 54,1), 116 | SUBSTRING(LAR, 55,1), 117 | SUBSTRING(LAR, 56,1), 118 | SUBSTRING(LAR, 57,1), 119 | SUBSTRING(LAR, 58,1), 120 | SUBSTRING(LAR, 59,1), 121 | SUBSTRING(LAR, 60,1), 122 | SUBSTRING(LAR, 61,1), 123 | SUBSTRING(LAR, 62,1), 124 | SUBSTRING(LAR, 63,1), 125 | SUBSTRING(LAR, 64,1), 126 | SUBSTRING(LAR, 65,1), 127 | SUBSTRING(LAR, 66,1), 128 | SUBSTRING(LAR, 67,5), 129 | SUBSTRING(LAR, 72,1), 130 | SUBSTRING(LAR, 73,1), 131 | SUBSTRING(LAR, 74,7) 132 | 133 | FROM lar_load; 134 | COMMIT; 135 | DROP TABLE IF EXISTS lar_load; 136 | COMMIT; 137 | 138 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_lar_2011.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.lar_2011; 2 | CREATE TABLE hmda_public.lar_2011( 3 | activity_year VARCHAR, 4 | respondent_id VARCHAR, 5 | agency_code VARCHAR, 6 | loan_type VARCHAR, 7 | loan_purpose VARCHAR, 8 | occupancy_type VARCHAR, 9 | loan_amount VARCHAR, 10 | action_taken VARCHAR, 11 | msa_md VARCHAR, 12 | state_code VARCHAR, 13 | county_code VARCHAR, 14 | census_tract VARCHAR, 15 | applicant_sex VARCHAR, 16 | co_applicant_sex VARCHAR, 17 | income VARCHAR, 18 | purchaser_type VARCHAR, 19 | denial_reason_1 VARCHAR, 20 | denial_reason_2 VARCHAR, 21 | denial_reason_3 VARCHAR, 22 | edit_status VARCHAR, 23 | property_type VARCHAR, 24 | preapproval VARCHAR, 25 | applicant_ethnicity VARCHAR, 26 | co_applicant_ethnicity VARCHAR, 27 | applicant_race_1 VARCHAR, 28 | applicant_race_2 VARCHAR, 29 | applicant_race_3 VARCHAR, 30 | applicant_race_4 VARCHAR, 31 | applicant_race_5 VARCHAR, 32 | co_applicant_race_1 VARCHAR, 33 | co_applicant_race_2 VARCHAR, 34 | co_applicant_race_3 VARCHAR, 35 | co_applicant_race_4 VARCHAR, 36 | co_applicant_race_5 VARCHAR, 37 | rate_spread VARCHAR, 38 | hoepa_status VARCHAR, 39 | lien_status VARCHAR, 40 | sequence_number VARCHAR 41 | ); 42 | 43 | COMMIT; 44 | 45 | CREATE TEMPORARY TABLE lar_load 46 | (LAR VARCHAR); -- LAR contains an entire LAR record 47 | 48 | COPY lar_load 49 | -- Change this path to your local data path. 50 | FROM '{data_path}/data/lar/lar_2011.dat'; 51 | 52 | COMMIT; 53 | 54 | INSERT INTO hmda_public.lar_2011 ( 55 | activity_year, 56 | respondent_id, 57 | agency_code, 58 | loan_type, 59 | loan_purpose, 60 | occupancy_type, 61 | loan_amount, 62 | action_taken, 63 | msa_md, 64 | state_code, 65 | county_code, 66 | census_tract, 67 | applicant_sex, 68 | co_applicant_sex, 69 | income, 70 | purchaser_type, 71 | denial_reason_1, 72 | denial_reason_2, 73 | denial_reason_3, 74 | edit_status, 75 | property_type, 76 | preapproval, 77 | applicant_ethnicity, 78 | co_applicant_ethnicity, 79 | applicant_race_1, 80 | applicant_race_2, 81 | applicant_race_3, 82 | applicant_race_4, 83 | applicant_race_5, 84 | co_applicant_race_1, 85 | co_applicant_race_2, 86 | co_applicant_race_3, 87 | co_applicant_race_4, 88 | co_applicant_race_5, 89 | rate_spread, 90 | hoepa_status, 91 | lien_status, 92 | sequence_number 93 | ) 94 | 95 | SELECT 96 | SUBSTRING(LAR, 1,4), 97 | SUBSTRING(LAR, 5,10), 98 | SUBSTRING(LAR, 15,1), 99 | SUBSTRING(LAR, 16,1), 100 | SUBSTRING(LAR, 17,1), 101 | SUBSTRING(LAR, 18,1), 102 | SUBSTRING(LAR, 19,5), 103 | SUBSTRING(LAR, 24,1), 104 | SUBSTRING(LAR, 25,5), 105 | SUBSTRING(LAR, 30,2), 106 | SUBSTRING(LAR, 32,3), 107 | SUBSTRING(LAR, 35,7), 108 | SUBSTRING(LAR, 42,1), 109 | SUBSTRING(LAR, 43,1), 110 | SUBSTRING(LAR, 44,4), 111 | SUBSTRING(LAR, 48,1), 112 | SUBSTRING(LAR, 49,1), 113 | SUBSTRING(LAR, 50,1), 114 | SUBSTRING(LAR, 51,1), 115 | SUBSTRING(LAR, 52,1), 116 | SUBSTRING(LAR, 53,1), 117 | SUBSTRING(LAR, 54,1), 118 | SUBSTRING(LAR, 55,1), 119 | SUBSTRING(LAR, 56,1), 120 | SUBSTRING(LAR, 57,1), 121 | SUBSTRING(LAR, 58,1), 122 | SUBSTRING(LAR, 59,1), 123 | SUBSTRING(LAR, 60,1), 124 | SUBSTRING(LAR, 61,1), 125 | SUBSTRING(LAR, 62,1), 126 | SUBSTRING(LAR, 63,1), 127 | SUBSTRING(LAR, 64,1), 128 | SUBSTRING(LAR, 65,1), 129 | SUBSTRING(LAR, 66,1), 130 | SUBSTRING(LAR, 67,5), 131 | SUBSTRING(LAR, 72,1), 132 | SUBSTRING(LAR, 73,1), 133 | SUBSTRING(LAR, 74,7) 134 | 135 | FROM lar_load; 136 | COMMIT; 137 | DROP TABLE IF EXISTS lar_load; 138 | COMMIT; 139 | 140 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_lar_2012.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.lar_2012; 2 | CREATE TABLE hmda_public.lar_2012( 3 | activity_year VARCHAR, 4 | respondent_id VARCHAR, 5 | agency_code VARCHAR, 6 | loan_type VARCHAR, 7 | loan_purpose VARCHAR, 8 | occupancy_type VARCHAR, 9 | loan_amount VARCHAR, 10 | action_taken VARCHAR, 11 | msa_md VARCHAR, 12 | state_code VARCHAR, 13 | county_code VARCHAR, 14 | census_tract VARCHAR, 15 | applicant_sex VARCHAR, 16 | co_applicant_sex VARCHAR, 17 | income VARCHAR, 18 | purchaser_type VARCHAR, 19 | denial_reason_1 VARCHAR, 20 | denial_reason_2 VARCHAR, 21 | denial_reason_3 VARCHAR, 22 | edit_status VARCHAR, 23 | property_type VARCHAR, 24 | preapproval VARCHAR, 25 | applicant_ethnicity VARCHAR, 26 | co_applicant_ethnicity VARCHAR, 27 | applicant_race_1 VARCHAR, 28 | applicant_race_2 VARCHAR, 29 | applicant_race_3 VARCHAR, 30 | applicant_race_4 VARCHAR, 31 | applicant_race_5 VARCHAR, 32 | co_applicant_race_1 VARCHAR, 33 | co_applicant_race_2 VARCHAR, 34 | co_applicant_race_3 VARCHAR, 35 | co_applicant_race_4 VARCHAR, 36 | co_applicant_race_5 VARCHAR, 37 | rate_spread VARCHAR, 38 | hoepa_status VARCHAR, 39 | lien_status VARCHAR, 40 | sequence_number VARCHAR 41 | ); 42 | 43 | COMMIT; 44 | 45 | CREATE TEMPORARY TABLE lar_load 46 | (LAR VARCHAR); -- LAR contains an entire LAR record 47 | 48 | COPY lar_load 49 | -- Change this path to your local data path. 50 | FROM '{data_path}/data/lar/lar_2012.dat'; 51 | 52 | COMMIT; 53 | 54 | INSERT INTO hmda_public.lar_2012 ( 55 | activity_year, 56 | respondent_id, 57 | agency_code, 58 | loan_type, 59 | loan_purpose, 60 | occupancy_type, 61 | loan_amount, 62 | action_taken, 63 | msa_md, 64 | state_code, 65 | county_code, 66 | census_tract, 67 | applicant_sex, 68 | co_applicant_sex, 69 | income, 70 | purchaser_type, 71 | denial_reason_1, 72 | denial_reason_2, 73 | denial_reason_3, 74 | edit_status, 75 | property_type, 76 | preapproval, 77 | applicant_ethnicity, 78 | co_applicant_ethnicity, 79 | applicant_race_1, 80 | applicant_race_2, 81 | applicant_race_3, 82 | applicant_race_4, 83 | applicant_race_5, 84 | co_applicant_race_1, 85 | co_applicant_race_2, 86 | co_applicant_race_3, 87 | co_applicant_race_4, 88 | co_applicant_race_5, 89 | rate_spread, 90 | hoepa_status, 91 | lien_status, 92 | sequence_number 93 | ) 94 | 95 | SELECT 96 | SUBSTRING(LAR, 1,4), 97 | SUBSTRING(LAR, 5,10), 98 | SUBSTRING(LAR, 15,1), 99 | SUBSTRING(LAR, 16,1), 100 | SUBSTRING(LAR, 17,1), 101 | SUBSTRING(LAR, 18,1), 102 | SUBSTRING(LAR, 19,5), 103 | SUBSTRING(LAR, 24,1), 104 | SUBSTRING(LAR, 25,5), 105 | SUBSTRING(LAR, 30,2), 106 | SUBSTRING(LAR, 32,3), 107 | SUBSTRING(LAR, 35,7), 108 | SUBSTRING(LAR, 42,1), 109 | SUBSTRING(LAR, 43,1), 110 | SUBSTRING(LAR, 44,4), 111 | SUBSTRING(LAR, 48,1), 112 | SUBSTRING(LAR, 49,1), 113 | SUBSTRING(LAR, 50,1), 114 | SUBSTRING(LAR, 51,1), 115 | SUBSTRING(LAR, 52,1), 116 | SUBSTRING(LAR, 53,1), 117 | SUBSTRING(LAR, 54,1), 118 | SUBSTRING(LAR, 55,1), 119 | SUBSTRING(LAR, 56,1), 120 | SUBSTRING(LAR, 57,1), 121 | SUBSTRING(LAR, 58,1), 122 | SUBSTRING(LAR, 59,1), 123 | SUBSTRING(LAR, 60,1), 124 | SUBSTRING(LAR, 61,1), 125 | SUBSTRING(LAR, 62,1), 126 | SUBSTRING(LAR, 63,1), 127 | SUBSTRING(LAR, 64,1), 128 | SUBSTRING(LAR, 65,1), 129 | SUBSTRING(LAR, 66,1), 130 | SUBSTRING(LAR, 67,5), 131 | SUBSTRING(LAR, 72,1), 132 | SUBSTRING(LAR, 73,1), 133 | SUBSTRING(LAR, 74,7) 134 | 135 | FROM lar_load; 136 | COMMIT; 137 | DROP TABLE IF EXISTS lar_load; 138 | COMMIT; 139 | 140 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_lar_2013.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.lar_2013; 2 | CREATE TABLE hmda_public.lar_2013( 3 | activity_year VARCHAR, 4 | respondent_id VARCHAR, 5 | agency_code VARCHAR, 6 | loan_type VARCHAR, 7 | loan_purpose VARCHAR, 8 | occupancy_type VARCHAR, 9 | loan_amount VARCHAR, 10 | action_taken VARCHAR, 11 | msa_md VARCHAR, 12 | state_code VARCHAR, 13 | county_code VARCHAR, 14 | census_tract VARCHAR, 15 | applicant_sex VARCHAR, 16 | co_applicant_sex VARCHAR, 17 | income VARCHAR, 18 | purchaser_type VARCHAR, 19 | denial_reason_1 VARCHAR, 20 | denial_reason_2 VARCHAR, 21 | denial_reason_3 VARCHAR, 22 | edit_status VARCHAR, 23 | property_type VARCHAR, 24 | preapproval VARCHAR, 25 | applicant_ethnicity VARCHAR, 26 | co_applicant_ethnicity VARCHAR, 27 | applicant_race_1 VARCHAR, 28 | applicant_race_2 VARCHAR, 29 | applicant_race_3 VARCHAR, 30 | applicant_race_4 VARCHAR, 31 | applicant_race_5 VARCHAR, 32 | co_applicant_race_1 VARCHAR, 33 | co_applicant_race_2 VARCHAR, 34 | co_applicant_race_3 VARCHAR, 35 | co_applicant_race_4 VARCHAR, 36 | co_applicant_race_5 VARCHAR, 37 | rate_spread VARCHAR, 38 | hoepa_status VARCHAR, 39 | lien_status VARCHAR, 40 | sequence_number VARCHAR 41 | ); 42 | 43 | CREATE TEMPORARY TABLE lar_load 44 | (LAR VARCHAR); -- LAR contains an entire LAR record 45 | 46 | COPY lar_load 47 | -- Change this path to your local data path. 48 | FROM '{data_path}/data/lar/lar_2013.dat'; 49 | 50 | COMMIT; 51 | 52 | INSERT INTO hmda_public.lar_2013 ( 53 | activity_year, 54 | respondent_id, 55 | agency_code, 56 | loan_type, 57 | loan_purpose, 58 | occupancy_type, 59 | loan_amount, 60 | action_taken, 61 | msa_md, 62 | state_code, 63 | county_code, 64 | census_tract, 65 | applicant_sex, 66 | co_applicant_sex, 67 | income, 68 | purchaser_type, 69 | denial_reason_1, 70 | denial_reason_2, 71 | denial_reason_3, 72 | edit_status, 73 | property_type, 74 | preapproval, 75 | applicant_ethnicity, 76 | co_applicant_ethnicity, 77 | applicant_race_1, 78 | applicant_race_2, 79 | applicant_race_3, 80 | applicant_race_4, 81 | applicant_race_5, 82 | co_applicant_race_1, 83 | co_applicant_race_2, 84 | co_applicant_race_3, 85 | co_applicant_race_4, 86 | co_applicant_race_5, 87 | rate_spread, 88 | hoepa_status, 89 | lien_status, 90 | sequence_number 91 | ) 92 | 93 | SELECT 94 | SUBSTRING(LAR, 1,4), 95 | SUBSTRING(LAR, 5,10), 96 | SUBSTRING(LAR, 15,1), 97 | SUBSTRING(LAR, 16,1), 98 | SUBSTRING(LAR, 17,1), 99 | SUBSTRING(LAR, 18,1), 100 | SUBSTRING(LAR, 19,5), 101 | SUBSTRING(LAR, 24,1), 102 | SUBSTRING(LAR, 25,5), 103 | SUBSTRING(LAR, 30,2), 104 | SUBSTRING(LAR, 32,3), 105 | SUBSTRING(LAR, 35,7), 106 | SUBSTRING(LAR, 42,1), 107 | SUBSTRING(LAR, 43,1), 108 | SUBSTRING(LAR, 44,4), 109 | SUBSTRING(LAR, 48,1), 110 | SUBSTRING(LAR, 49,1), 111 | SUBSTRING(LAR, 50,1), 112 | SUBSTRING(LAR, 51,1), 113 | SUBSTRING(LAR, 52,1), 114 | SUBSTRING(LAR, 53,1), 115 | SUBSTRING(LAR, 54,1), 116 | SUBSTRING(LAR, 55,1), 117 | SUBSTRING(LAR, 56,1), 118 | SUBSTRING(LAR, 57,1), 119 | SUBSTRING(LAR, 58,1), 120 | SUBSTRING(LAR, 59,1), 121 | SUBSTRING(LAR, 60,1), 122 | SUBSTRING(LAR, 61,1), 123 | SUBSTRING(LAR, 62,1), 124 | SUBSTRING(LAR, 63,1), 125 | SUBSTRING(LAR, 64,1), 126 | SUBSTRING(LAR, 65,1), 127 | SUBSTRING(LAR, 66,1), 128 | SUBSTRING(LAR, 67,5), 129 | SUBSTRING(LAR, 72,1), 130 | SUBSTRING(LAR, 73,1), 131 | SUBSTRING(LAR, 74,7) 132 | 133 | FROM lar_load; 134 | COMMIT; 135 | DROP TABLE IF EXISTS lar_load; 136 | COMMIT; 137 | 138 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_lar_2014.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.lar_2014; 2 | CREATE TABLE hmda_public.lar_2014( 3 | activity_year VARCHAR, 4 | respondent_id VARCHAR, 5 | agency_code VARCHAR, 6 | loan_type VARCHAR, 7 | loan_purpose VARCHAR, 8 | occupancy_type VARCHAR, 9 | loan_amount VARCHAR, 10 | action_taken VARCHAR, 11 | msa_md VARCHAR, 12 | state_code VARCHAR, 13 | county_code VARCHAR, 14 | census_tract VARCHAR, 15 | applicant_sex VARCHAR, 16 | co_applicant_sex VARCHAR, 17 | income VARCHAR, 18 | purchaser_type VARCHAR, 19 | denial_reason_1 VARCHAR, 20 | denial_reason_2 VARCHAR, 21 | denial_reason_3 VARCHAR, 22 | edit_status VARCHAR, 23 | property_type VARCHAR, 24 | preapproval VARCHAR, 25 | applicant_ethnicity VARCHAR, 26 | co_applicant_ethnicity VARCHAR, 27 | applicant_race_1 VARCHAR, 28 | applicant_race_2 VARCHAR, 29 | applicant_race_3 VARCHAR, 30 | applicant_race_4 VARCHAR, 31 | applicant_race_5 VARCHAR, 32 | co_applicant_race_1 VARCHAR, 33 | co_applicant_race_2 VARCHAR, 34 | co_applicant_race_3 VARCHAR, 35 | co_applicant_race_4 VARCHAR, 36 | co_applicant_race_5 VARCHAR, 37 | rate_spread VARCHAR, 38 | hoepa_status VARCHAR, 39 | lien_status VARCHAR, 40 | sequence_number VARCHAR 41 | ); 42 | 43 | CREATE TEMPORARY TABLE lar_load 44 | (LAR VARCHAR); -- LAR contains an entire LAR record 45 | 46 | COPY lar_load 47 | -- Change this path to your local data path. 48 | FROM '{data_path}/data/lar/lar_2014.txt'; 49 | 50 | COMMIT; 51 | 52 | INSERT INTO hmda_public.lar_2014 ( 53 | activity_year, 54 | respondent_id, 55 | agency_code, 56 | loan_type, 57 | loan_purpose, 58 | occupancy_type, 59 | loan_amount, 60 | action_taken, 61 | msa_md, 62 | state_code, 63 | county_code, 64 | census_tract, 65 | applicant_sex, 66 | co_applicant_sex, 67 | income, 68 | purchaser_type, 69 | denial_reason_1, 70 | denial_reason_2, 71 | denial_reason_3, 72 | edit_status, 73 | property_type, 74 | preapproval, 75 | applicant_ethnicity, 76 | co_applicant_ethnicity, 77 | applicant_race_1, 78 | applicant_race_2, 79 | applicant_race_3, 80 | applicant_race_4, 81 | applicant_race_5, 82 | co_applicant_race_1, 83 | co_applicant_race_2, 84 | co_applicant_race_3, 85 | co_applicant_race_4, 86 | co_applicant_race_5, 87 | rate_spread, 88 | hoepa_status, 89 | lien_status, 90 | sequence_number 91 | ) 92 | 93 | SELECT 94 | SUBSTRING(LAR, 1,4), 95 | SUBSTRING(LAR, 5,10), 96 | SUBSTRING(LAR, 15,1), 97 | SUBSTRING(LAR, 16,1), 98 | SUBSTRING(LAR, 17,1), 99 | SUBSTRING(LAR, 18,1), 100 | SUBSTRING(LAR, 19,5), 101 | SUBSTRING(LAR, 24,1), 102 | SUBSTRING(LAR, 25,5), 103 | SUBSTRING(LAR, 30,2), 104 | SUBSTRING(LAR, 32,3), 105 | SUBSTRING(LAR, 35,7), 106 | SUBSTRING(LAR, 42,1), 107 | SUBSTRING(LAR, 43,1), 108 | SUBSTRING(LAR, 44,4), 109 | SUBSTRING(LAR, 48,1), 110 | SUBSTRING(LAR, 49,1), 111 | SUBSTRING(LAR, 50,1), 112 | SUBSTRING(LAR, 51,1), 113 | SUBSTRING(LAR, 52,1), 114 | SUBSTRING(LAR, 53,1), 115 | SUBSTRING(LAR, 54,1), 116 | SUBSTRING(LAR, 55,1), 117 | SUBSTRING(LAR, 56,1), 118 | SUBSTRING(LAR, 57,1), 119 | SUBSTRING(LAR, 58,1), 120 | SUBSTRING(LAR, 59,1), 121 | SUBSTRING(LAR, 60,1), 122 | SUBSTRING(LAR, 61,1), 123 | SUBSTRING(LAR, 62,1), 124 | SUBSTRING(LAR, 63,1), 125 | SUBSTRING(LAR, 64,1), 126 | SUBSTRING(LAR, 65,1), 127 | SUBSTRING(LAR, 66,1), 128 | SUBSTRING(LAR, 67,5), 129 | SUBSTRING(LAR, 72,1), 130 | SUBSTRING(LAR, 73,1), 131 | SUBSTRING(LAR, 74,7) 132 | 133 | FROM lar_load; 134 | COMMIT; 135 | DROP TABLE IF EXISTS lar_load; 136 | COMMIT; 137 | 138 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_lar_2015.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.lar_2015; 2 | CREATE TABLE hmda_public.lar_2015 ( 3 | activity_year VARCHAR, 4 | respondent_id VARCHAR, 5 | agency_code VARCHAR, 6 | loan_type VARCHAR, 7 | property_type VARCHAR, 8 | loan_purpose VARCHAR, 9 | occupancy_type VARCHAR, 10 | loan_amount VARCHAR, 11 | preapproval VARCHAR, 12 | action_taken VARCHAR, 13 | msa_md VARCHAR, 14 | state_code VARCHAR, 15 | county_code VARCHAR, 16 | census_tract VARCHAR, 17 | applicant_ethnicity VARCHAR, 18 | co_applicant_ethnicity VARCHAR, 19 | applicant_race_1 VARCHAR, 20 | applicant_race_2 VARCHAR, 21 | applicant_race_3 VARCHAR, 22 | applicant_race_4 VARCHAR, 23 | applicant_race_5 VARCHAR, 24 | co_applicant_race_1 VARCHAR, 25 | co_applicant_race_2 VARCHAR, 26 | co_applicant_race_3 VARCHAR, 27 | co_applicant_race_4 VARCHAR, 28 | co_applicant_race_5 VARCHAR, 29 | applicant_sex VARCHAR, 30 | co_applicant_sex VARCHAR, 31 | income VARCHAR, 32 | purchaser_type VARCHAR, 33 | denial_reason_1 VARCHAR, 34 | denial_reason_2 VARCHAR, 35 | denial_reason_3 VARCHAR, 36 | rate_spread VARCHAR, 37 | hoepa_status VARCHAR, 38 | lien_status VARCHAR, 39 | edit_status VARCHAR, 40 | sequence_number VARCHAR, 41 | tract_population VARCHAR, 42 | tract_minority_population_percent VARCHAR, 43 | ffiec_msa_md_median_family_income VARCHAR, 44 | tract_to_msa_income_percentage VARCHAR, 45 | tract_owner_occupied_units VARCHAR, 46 | tract_one_to_four_family_units VARCHAR, 47 | application_date_indicator VARCHAR 48 | ); 49 | 50 | COPY hmda_public.lar_2015 51 | FROM '{data_path}/data/lar/lar_2015.csv' 52 | DELIMITER ',' ENCODING 'latin1'; 53 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_lar_2016.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.lar_2016; 2 | CREATE TABLE hmda_public.lar_2016 ( 3 | activity_year VARCHAR, 4 | respondent_id VARCHAR, 5 | agency_code VARCHAR, 6 | loan_type VARCHAR, 7 | property_type VARCHAR, 8 | loan_purpose VARCHAR, 9 | occupancy_type VARCHAR, 10 | loan_amount VARCHAR, 11 | preapproval VARCHAR, 12 | action_taken VARCHAR, 13 | msa_md VARCHAR, 14 | state_code VARCHAR, 15 | county_code VARCHAR, 16 | census_tract VARCHAR, 17 | applicant_ethnicity VARCHAR, 18 | co_applicant_ethnicity VARCHAR, 19 | applicant_race_1 VARCHAR, 20 | applicant_race_2 VARCHAR, 21 | applicant_race_3 VARCHAR, 22 | applicant_race_4 VARCHAR, 23 | applicant_race_5 VARCHAR, 24 | co_applicant_race_1 VARCHAR, 25 | co_applicant_race_2 VARCHAR, 26 | co_applicant_race_3 VARCHAR, 27 | co_applicant_race_4 VARCHAR, 28 | co_applicant_race_5 VARCHAR, 29 | applicant_sex VARCHAR, 30 | co_applicant_sex VARCHAR, 31 | income VARCHAR, 32 | purchaser_type VARCHAR, 33 | denial_reason_1 VARCHAR, 34 | denial_reason_2 VARCHAR, 35 | denial_reason_3 VARCHAR, 36 | rate_spread VARCHAR, 37 | hoepa_status VARCHAR, 38 | lien_status VARCHAR, 39 | edit_status VARCHAR, 40 | sequence_number VARCHAR, 41 | tract_population VARCHAR, 42 | tract_minority_population_percent VARCHAR, 43 | ffiec_msa_md_median_family_income VARCHAR, 44 | tract_to_msa_income_percentage VARCHAR, 45 | tract_owner_occupied_units VARCHAR, 46 | tract_one_to_four_family_units VARCHAR, 47 | application_date_indicator VARCHAR 48 | ); 49 | 50 | COPY hmda_public.lar_2016 51 | FROM '{data_path}/data/lar/lar_2016.csv' 52 | DELIMITER ',' ENCODING 'latin1'; 53 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_lar_2017.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.lar_2017; 2 | CREATE TABLE hmda_public.lar_2017 ( 3 | activity_year VARCHAR, 4 | respondent_id VARCHAR, 5 | agency_code VARCHAR, 6 | loan_type VARCHAR, 7 | property_type VARCHAR, 8 | loan_purpose VARCHAR, 9 | occupancy_type VARCHAR, 10 | loan_amount VARCHAR, 11 | preapproval VARCHAR, 12 | action_taken VARCHAR, 13 | msa_md VARCHAR, 14 | state_code VARCHAR, 15 | county_code VARCHAR, 16 | census_tract VARCHAR, 17 | applicant_ethnicity VARCHAR, 18 | co_applicant_ethnicity VARCHAR, 19 | applicant_race_1 VARCHAR, 20 | applicant_race_2 VARCHAR, 21 | applicant_race_3 VARCHAR, 22 | applicant_race_4 VARCHAR, 23 | applicant_race_5 VARCHAR, 24 | co_applicant_race_1 VARCHAR, 25 | co_applicant_race_2 VARCHAR, 26 | co_applicant_race_3 VARCHAR, 27 | co_applicant_race_4 VARCHAR, 28 | co_applicant_race_5 VARCHAR, 29 | applicant_sex VARCHAR, 30 | co_applicant_sex VARCHAR, 31 | income VARCHAR, 32 | purchaser_type VARCHAR, 33 | denial_reason_1 VARCHAR, 34 | denial_reason_2 VARCHAR, 35 | denial_reason_3 VARCHAR, 36 | rate_spread VARCHAR, 37 | hoepa_status VARCHAR, 38 | lien_status VARCHAR, 39 | tract_population VARCHAR, 40 | tract_minority_population_percent VARCHAR, 41 | ffiec_msa_md_median_family_income VARCHAR, 42 | tract_to_msa_income_percentage VARCHAR, 43 | tract_owner_occupied_units VARCHAR, 44 | tract_one_to_four_family_units VARCHAR, 45 | derived_loan_product_type VARCHAR, 46 | derived_dwelling_category VARCHAR, 47 | derived_ethnicity VARCHAR, 48 | derived_race VARCHAR, 49 | derived_sex VARCHAR 50 | ); 51 | 52 | COPY hmda_public.lar_2017 53 | 54 | FROM '{data_path}/data/lar/lar_2017.txt' 55 | 56 | DELIMITER '|' ENCODING 'latin1'; 57 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_modified_lar_2017.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS mlar_2017_{institution_id}; 2 | CREATE TABLE mlar_2017_{institution_id} ( 3 | record_id NUMERIC, 4 | respondent_id VARCHAR, 5 | agency_code NUMERIC, 6 | loan_type NUMERIC, 7 | property_type NUMERIC, 8 | loan_purpose NUMERIC, 9 | occupancy_type NUMERIC, 10 | loan_amount NUMERIC, 11 | preapproval NUMERIC, 12 | action_taken NUMERIC, 13 | msa_md VARCHAR, 14 | state_code VARCHAR, 15 | county_code VARCHAR, 16 | census_tract VARCHAR, 17 | applicant_ethnicity NUMERIC, 18 | co_applicant_ethnicity NUMERIC, 19 | applicant_race_1 NUMERIC, 20 | applicant_race_2 VARCHAR, 21 | applicant_race_3 VARCHAR, 22 | applicant_race_4 VARCHAR, 23 | applicant_race_5 VARCHAR, 24 | co_applicant_race_1 NUMERIC, 25 | co_applicant_race_2 VARCHAR, 26 | co_applicant_race_3 VARCHAR, 27 | co_applicant_race_4 VARCHAR, 28 | co_applicant_race_5 VARCHAR, 29 | applicant_sex NUMERIC, 30 | co_applicant_sex NUMERIC, 31 | income NUMERIC, 32 | purchaser_type NUMERIC, 33 | denial_reason_1 VARCHAR, 34 | denial_reason_2 VARCHAR, 35 | denial_reason_3 VARCHAR, 36 | rate_spread VARCHAR, 37 | hoepa_status NUMERIC, 38 | lien_status NUMERIC 39 | ) 40 | 41 | COPY hmda_public.mlar_2017_{institution_id} 42 | FROM '{data_path}/data/mlar/2017/{institution_id}.txt' 43 | DELIMITER '|' ENCODING 'latin1'; 44 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_modified_lar_2018.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS mlar_2018_{lei}; 2 | CREATE TABLE mlar_2018_{lei} ( 3 | record_id NUMERIC, 4 | lei VARCHAR, 5 | loan_type NUMERIC, 6 | loan_purpose NUMERIC, 7 | preapproval NUMERIC, 8 | construction_method NUMERIC, 9 | occupancy_type NUMERIC, 10 | loan_amount NUMERIC, 11 | action_taken NUMERIC, 12 | state_code VARCHAR, 13 | county_code VARCHAR, 14 | census_tract VARCHAR, 15 | applicant_ethnicity_1 VARCHAR, 16 | applicant_ethnicity_2 VARCHAR, 17 | applicant_ethnicity_3 VARCHAR, 18 | applicant_ethnicity_4 VARCHAR, 19 | applicant_ethnicity_5 VARCHAR, 20 | co_applicant_ethnicity_1 VARCHAR, 21 | co_applicant_ethnicity_2 VARCHAR, 22 | co_applicant_ethnicity_3 VARCHAR, 23 | co_applicant_ethnicity_4 VARCHAR, 24 | co_applicant_ethnicity_5 VARCHAR, 25 | applicant_ethnicity_observed NUMERIC, 26 | co_applicant_ethnicity_observed NUMERIC, 27 | applicant_race_1 VARCHAR, 28 | applicant_race_2 VARCHAR, 29 | applicant_race_3 VARCHAR, 30 | applicant_race_4 VARCHAR, 31 | applicant_race_5 VARCHAR, 32 | co_applicant_race_1 VARCHAR, 33 | co_applicant_race_2 VARCHAR, 34 | co_applicant_race_3 VARCHAR, 35 | co_applicant_race_4 VARCHAR, 36 | co_applicant_race_5 VARCHAR, 37 | applicant_race_observed NUMERIC, 38 | co_applicant_race_observed NUMERIC, 39 | applicant_sex NUMERIC, 40 | co_applicant_sex NUMERIC, 41 | applicant_sex_observed NUMERIC, 42 | co_applicant_sex_observed NUMERIC, 43 | applicant_age VARCHAR, 44 | applicant_age_above_62 VARCHAR, 45 | co_applicant_age VARCHAR, 46 | co_applicant_age_above_62 VARCHAR, 47 | income VARCHAR, 48 | purchaser_type NUMERIC, 49 | rate_spread VARCHAR, 50 | hoepa_status NUMERIC, 51 | lien_status NUMERIC, 52 | applicant_credit_scoring_model NUMERIC, 53 | co_applicant_credit_scoring_model NUMERIC, 54 | denial_reason_1 NUMERIC, 55 | denial_reason_2 VARCHAR, 56 | denial_reason_3 VARCHAR, 57 | denial_reason_4 VARCHAR, 58 | total_loan_costs VARCHAR, 59 | total_points_and_fees VARCHAR, 60 | origination_charges VARCHAR, 61 | discount_points VARCHAR, 62 | lender_credits VARCHAR, 63 | interest_rate VARCHAR, 64 | prepayment_penalty_term VARCHAR, 65 | debt_to_income_ratio VARCHAR, 66 | combined_loan_to_value_ratio VARCHAR, 67 | loan_term VARCHAR, 68 | intro_rate_period VARCHAR, 69 | balloon_payment NUMERIC, 70 | interest_only_payment NUMERIC, 71 | negative_amortization NUMERIC, 72 | other_non_amortizing_features NUMERIC, 73 | property_value VARCHAR, 74 | manufactured_home_secured_property_type NUMERIC, 75 | manufactured_home_land_property_interest NUMERIC, 76 | total_units VARCHAR, 77 | multifamily_affordable_units VARCHAR, 78 | submission_of_application NUMERIC, 79 | initially_payable_to_institution NUMERIC, 80 | aus_1 NUMERIC, 81 | aus_2 VARCHAR, 82 | aus_3 VARCHAR, 83 | aus_4 VARCHAR, 84 | aus_5 VARCHAR, 85 | reverse_mortgage NUMERIC, 86 | open_end_credit NUMERIC, 87 | business_or_commercial_purpose NUMERIC 88 | ); 89 | 90 | COPY hmda_public.mlar_2018_{lei} 91 | FROM '{data_path}/data/mlar/2018/{lei}.txt' 92 | DELIMITER '|' ENCODING 'latin1'; 93 | COMMIT; 94 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_modified_lar_2019.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS mlar_2019_{lei}; 2 | CREATE TABLE mlar_2019_{lei} ( 3 | record_id NUMERIC, 4 | lei VARCHAR, 5 | loan_type NUMERIC, 6 | loan_purpose NUMERIC, 7 | preapproval NUMERIC, 8 | construction_method NUMERIC, 9 | occupancy_type NUMERIC, 10 | loan_amount NUMERIC, 11 | action_taken NUMERIC, 12 | state_code VARCHAR, 13 | county_code VARCHAR, 14 | census_tract VARCHAR, 15 | applicant_ethnicity_1 VARCHAR, 16 | applicant_ethnicity_2 VARCHAR, 17 | applicant_ethnicity_3 VARCHAR, 18 | applicant_ethnicity_4 VARCHAR, 19 | applicant_ethnicity_5 VARCHAR, 20 | co_applicant_ethnicity_1 VARCHAR, 21 | co_applicant_ethnicity_2 VARCHAR, 22 | co_applicant_ethnicity_3 VARCHAR, 23 | co_applicant_ethnicity_4 VARCHAR, 24 | co_applicant_ethnicity_5 VARCHAR, 25 | applicant_ethnicity_observed NUMERIC, 26 | co_applicant_ethnicity_observed NUMERIC, 27 | applicant_race_1 VARCHAR, 28 | applicant_race_2 VARCHAR, 29 | applicant_race_3 VARCHAR, 30 | applicant_race_4 VARCHAR, 31 | applicant_race_5 VARCHAR, 32 | co_applicant_race_1 VARCHAR, 33 | co_applicant_race_2 VARCHAR, 34 | co_applicant_race_3 VARCHAR, 35 | co_applicant_race_4 VARCHAR, 36 | co_applicant_race_5 VARCHAR, 37 | applicant_race_observed NUMERIC, 38 | co_applicant_race_observed NUMERIC, 39 | applicant_sex NUMERIC, 40 | co_applicant_sex NUMERIC, 41 | applicant_sex_observed NUMERIC, 42 | co_applicant_sex_observed NUMERIC, 43 | applicant_age VARCHAR, 44 | applicant_age_above_62 VARCHAR, 45 | co_applicant_age VARCHAR, 46 | co_applicant_age_above_62 VARCHAR, 47 | income VARCHAR, 48 | purchaser_type NUMERIC, 49 | rate_spread VARCHAR, 50 | hoepa_status NUMERIC, 51 | lien_status NUMERIC, 52 | applicant_credit_scoring_model NUMERIC, 53 | co_applicant_credit_scoring_model NUMERIC, 54 | denial_reason_1 NUMERIC, 55 | denial_reason_2 VARCHAR, 56 | denial_reason_3 VARCHAR, 57 | denial_reason_4 VARCHAR, 58 | total_loan_costs VARCHAR, 59 | total_points_and_fees VARCHAR, 60 | origination_charges VARCHAR, 61 | discount_points VARCHAR, 62 | lender_credits VARCHAR, 63 | interest_rate VARCHAR, 64 | prepayment_penalty_term VARCHAR, 65 | debt_to_income_ratio VARCHAR, 66 | combined_loan_to_value_ratio VARCHAR, 67 | loan_term VARCHAR, 68 | intro_rate_period VARCHAR, 69 | balloon_payment NUMERIC, 70 | interest_only_payment NUMERIC, 71 | negative_amortization NUMERIC, 72 | other_non_amortizing_features NUMERIC, 73 | property_value VARCHAR, 74 | manufactured_home_secured_property_type NUMERIC, 75 | manufactured_home_land_property_interest NUMERIC, 76 | total_units VARCHAR, 77 | multifamily_affordable_units VARCHAR, 78 | submission_of_application NUMERIC, 79 | initially_payable_to_institution NUMERIC, 80 | aus_1 NUMERIC, 81 | aus_2 VARCHAR, 82 | aus_3 VARCHAR, 83 | aus_4 VARCHAR, 84 | aus_5 VARCHAR, 85 | reverse_mortgage NUMERIC, 86 | open_end_credit NUMERIC, 87 | business_or_commercial_purpose NUMERIC 88 | ); 89 | 90 | COPY hmda_public.mlar_2019_{lei} 91 | FROM '{data_path}/data/mlar/2019/{lei}.txt' 92 | DELIMITER '|' ENCODING 'latin1'; 93 | COMMIT; 94 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_panel_2004.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.panel_2004; 2 | DROP TABLE IF EXISTS panel_load; 3 | CREATE TABLE hmda_public.panel_2004 ( 4 | respondent_id VARCHAR, 5 | msa VARCHAR, 6 | agency_code VARCHAR, 7 | agency_group_code VARCHAR, 8 | respondent_name VARCHAR, 9 | respondent_city VARCHAR, 10 | respondent_state VARCHAR, 11 | respondent_state_fips_number VARCHAR, 12 | assets VARCHAR, 13 | other_lender_code VARCHAR, 14 | parent_respondent_id VARCHAR, 15 | parent_name VARCHAR, 16 | parent_city VARCHAR, 17 | parent_state VARCHAR, 18 | activity_year VARCHAR, 19 | respondent_rssd VARCHAR); 20 | CREATE TEMPORARY TABLE panel_load 21 | (PANEL VARCHAR); -- LAR contains an entire LAR record 22 | COPY panel_load 23 | --Change to your local path 24 | FROM '{data_path}/data/panel/panel_2004.dat' 25 | ENCODING 'latin1'; 26 | COMMIT; 27 | INSERT INTO hmda_public.panel_2004 ( 28 | respondent_id, 29 | msa, 30 | agency_code, 31 | agency_group_code, 32 | respondent_name, 33 | respondent_city, 34 | respondent_state, 35 | respondent_state_fips_number, 36 | assets, 37 | other_lender_code, 38 | parent_respondent_id, 39 | parent_name, 40 | parent_city, 41 | parent_state, 42 | activity_year, 43 | respondent_rssd 44 | ) 45 | SELECT 46 | SUBSTRING(PANEL, 1, 10), 47 | SUBSTRING(PANEL, 11, 5), 48 | SUBSTRING(PANEL, 16, 1), 49 | SUBSTRING(PANEL, 17, 2), 50 | SUBSTRING(PANEL, 19, 30), 51 | SUBSTRING(PANEL, 49, 25), 52 | SUBSTRING(PANEL, 74, 2), 53 | SUBSTRING(PANEL, 76, 2), 54 | SUBSTRING(PANEL, 78, 10), 55 | SUBSTRING(PANEL, 88, 1), 56 | SUBSTRING(PANEL, 89, 10), 57 | SUBSTRING(PANEL, 99, 30), 58 | SUBSTRING(PANEL, 129, 25), 59 | SUBSTRING(PANEL, 154, 2), 60 | SUBSTRING(PANEL, 156, 4), 61 | SUBSTRING(PANEL, 160, 10) 62 | FROM panel_load; 63 | COMMIT; 64 | DROP TABLE IF EXISTS panel_load; 65 | COMMIT; 66 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_panel_2005.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.panel_2005; 2 | DROP TABLE IF EXISTS panel_load; 3 | CREATE TABLE hmda_public.panel_2005 ( 4 | respondent_id VARCHAR, 5 | msa VARCHAR, 6 | agency_code VARCHAR, 7 | agency_group_code VARCHAR, 8 | respondent_name VARCHAR, 9 | respondent_city VARCHAR, 10 | respondent_state VARCHAR, 11 | respondent_state_fips_number VARCHAR, 12 | assets VARCHAR, 13 | other_lender_code VARCHAR, 14 | parent_respondent_id VARCHAR, 15 | parent_name VARCHAR, 16 | parent_city VARCHAR, 17 | parent_state VARCHAR, 18 | activity_year VARCHAR, 19 | respondent_rssd VARCHAR); 20 | CREATE TEMPORARY TABLE panel_load 21 | (PANEL VARCHAR); -- LAR contains an entire LAR record 22 | COPY panel_load 23 | --Change to your local path 24 | FROM '{data_path}/data/panel/panel_2005.dat' 25 | ENCODING 'latin1'; 26 | COMMIT; 27 | INSERT INTO hmda_public.panel_2005 ( 28 | respondent_id, 29 | msa, 30 | agency_code, 31 | agency_group_code, 32 | respondent_name, 33 | respondent_city, 34 | respondent_state, 35 | respondent_state_fips_number, 36 | assets, 37 | other_lender_code, 38 | parent_respondent_id, 39 | parent_name, 40 | parent_city, 41 | parent_state, 42 | activity_year, 43 | respondent_rssd 44 | ) 45 | SELECT 46 | SUBSTRING(PANEL, 1, 10), 47 | SUBSTRING(PANEL, 11, 5), 48 | SUBSTRING(PANEL, 16, 1), 49 | SUBSTRING(PANEL, 17, 2), 50 | SUBSTRING(PANEL, 19, 30), 51 | SUBSTRING(PANEL, 49, 25), 52 | SUBSTRING(PANEL, 74, 2), 53 | SUBSTRING(PANEL, 76, 2), 54 | SUBSTRING(PANEL, 78, 10), 55 | SUBSTRING(PANEL, 88, 1), 56 | SUBSTRING(PANEL, 89, 10), 57 | SUBSTRING(PANEL, 99, 30), 58 | SUBSTRING(PANEL, 129, 25), 59 | SUBSTRING(PANEL, 154, 2), 60 | SUBSTRING(PANEL, 156, 4), 61 | SUBSTRING(PANEL, 160, 10) 62 | FROM panel_load; 63 | COMMIT; 64 | DROP TABLE IF EXISTS panel_load; 65 | COMMIT; 66 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_panel_2006.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.panel_2006; 2 | DROP TABLE IF EXISTS panel_load; 3 | CREATE TABLE hmda_public.panel_2006 ( 4 | respondent_id VARCHAR, 5 | msa VARCHAR, 6 | agency_code VARCHAR, 7 | agency_group_code VARCHAR, 8 | respondent_name VARCHAR, 9 | respondent_city VARCHAR, 10 | respondent_state VARCHAR, 11 | respondent_state_fips_number VARCHAR, 12 | assets VARCHAR, 13 | other_lender_code VARCHAR, 14 | parent_respondent_id VARCHAR, 15 | parent_name VARCHAR, 16 | parent_city VARCHAR, 17 | parent_state VARCHAR, 18 | activity_year VARCHAR, 19 | respondent_rssd VARCHAR); 20 | CREATE TEMPORARY TABLE panel_load 21 | (PANEL VARCHAR); -- LAR contains an entire LAR record 22 | COPY panel_load 23 | --Change to your local path 24 | FROM '{data_path}/data/panel/panel_2006.dat' 25 | ENCODING 'latin1'; 26 | COMMIT; 27 | INSERT INTO hmda_public.panel_2006 ( 28 | respondent_id, 29 | msa, 30 | agency_code, 31 | agency_group_code, 32 | respondent_name, 33 | respondent_city, 34 | respondent_state, 35 | respondent_state_fips_number, 36 | assets, 37 | other_lender_code, 38 | parent_respondent_id, 39 | parent_name, 40 | parent_city, 41 | parent_state, 42 | activity_year, 43 | respondent_rssd 44 | ) 45 | SELECT 46 | SUBSTRING(PANEL, 1, 10), 47 | SUBSTRING(PANEL, 11, 5), 48 | SUBSTRING(PANEL, 16, 1), 49 | SUBSTRING(PANEL, 17, 2), 50 | SUBSTRING(PANEL, 19, 30), 51 | SUBSTRING(PANEL, 49, 25), 52 | SUBSTRING(PANEL, 74, 2), 53 | SUBSTRING(PANEL, 76, 2), 54 | SUBSTRING(PANEL, 78, 10), 55 | SUBSTRING(PANEL, 88, 1), 56 | SUBSTRING(PANEL, 89, 10), 57 | SUBSTRING(PANEL, 99, 30), 58 | SUBSTRING(PANEL, 129, 25), 59 | SUBSTRING(PANEL, 154, 2), 60 | SUBSTRING(PANEL, 156, 4), 61 | SUBSTRING(PANEL, 160, 10) 62 | FROM panel_load; 63 | COMMIT; 64 | DROP TABLE IF EXISTS panel_load; 65 | COMMIT; 66 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_panel_2007.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.panel_2007; 2 | DROP TABLE IF EXISTS panel_load; 3 | CREATE TABLE hmda_public.panel_2007 ( 4 | respondent_id VARCHAR, 5 | msa VARCHAR, 6 | agency_code VARCHAR, 7 | agency_group_code VARCHAR, 8 | respondent_name VARCHAR, 9 | respondent_city VARCHAR, 10 | respondent_state VARCHAR, 11 | respondent_state_fips_number VARCHAR, 12 | assets VARCHAR, 13 | other_lender_code VARCHAR, 14 | parent_respondent_id VARCHAR, 15 | parent_name VARCHAR, 16 | parent_city VARCHAR, 17 | parent_state VARCHAR, 18 | activity_year VARCHAR, 19 | respondent_rssd VARCHAR); 20 | CREATE TEMPORARY TABLE panel_load 21 | (PANEL VARCHAR); -- LAR contains an entire LAR record 22 | COPY panel_load 23 | --Change to your local path 24 | FROM '{data_path}/data/panel/panel_2007.dat' 25 | ENCODING 'latin1'; 26 | COMMIT; 27 | INSERT INTO hmda_public.panel_2007 ( 28 | respondent_id, 29 | msa, 30 | agency_code, 31 | agency_group_code, 32 | respondent_name, 33 | respondent_city, 34 | respondent_state, 35 | respondent_state_fips_number, 36 | assets, 37 | other_lender_code, 38 | parent_respondent_id, 39 | parent_name, 40 | parent_city, 41 | parent_state, 42 | activity_year, 43 | respondent_rssd 44 | ) 45 | SELECT 46 | SUBSTRING(PANEL, 1, 10), 47 | SUBSTRING(PANEL, 11, 5), 48 | SUBSTRING(PANEL, 16, 1), 49 | SUBSTRING(PANEL, 17, 2), 50 | SUBSTRING(PANEL, 19, 30), 51 | SUBSTRING(PANEL, 49, 25), 52 | SUBSTRING(PANEL, 74, 2), 53 | SUBSTRING(PANEL, 76, 2), 54 | SUBSTRING(PANEL, 78, 10), 55 | SUBSTRING(PANEL, 88, 1), 56 | SUBSTRING(PANEL, 89, 10), 57 | SUBSTRING(PANEL, 99, 30), 58 | SUBSTRING(PANEL, 129, 25), 59 | SUBSTRING(PANEL, 154, 2), 60 | SUBSTRING(PANEL, 156, 4), 61 | SUBSTRING(PANEL, 160, 10) 62 | FROM panel_load; 63 | COMMIT; 64 | DROP TABLE IF EXISTS panel_load; 65 | COMMIT; 66 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_panel_2008.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.panel_2008; 2 | DROP TABLE IF EXISTS panel_load; 3 | CREATE TABLE hmda_public.panel_2008 ( 4 | respondent_id VARCHAR, 5 | msa VARCHAR, 6 | agency_code VARCHAR, 7 | agency_group_code VARCHAR, 8 | respondent_name VARCHAR, 9 | respondent_city VARCHAR, 10 | respondent_state VARCHAR, 11 | respondent_state_fips_number VARCHAR, 12 | assets VARCHAR, 13 | other_lender_code VARCHAR, 14 | parent_respondent_id VARCHAR, 15 | parent_name VARCHAR, 16 | parent_city VARCHAR, 17 | parent_state VARCHAR, 18 | activity_year VARCHAR, 19 | respondent_rssd VARCHAR); 20 | CREATE TEMPORARY TABLE panel_load 21 | (PANEL VARCHAR); -- LAR contains an entire LAR record 22 | COPY panel_load 23 | --Change to your local path 24 | FROM '{data_path}/data/panel/panel_2008.dat' 25 | ENCODING 'latin1'; 26 | COMMIT; 27 | INSERT INTO hmda_public.panel_2008 ( 28 | respondent_id, 29 | msa, 30 | agency_code, 31 | agency_group_code, 32 | respondent_name, 33 | respondent_city, 34 | respondent_state, 35 | respondent_state_fips_number, 36 | assets, 37 | other_lender_code, 38 | parent_respondent_id, 39 | parent_name, 40 | parent_city, 41 | parent_state, 42 | activity_year, 43 | respondent_rssd 44 | ) 45 | SELECT 46 | SUBSTRING(PANEL, 1, 10), 47 | SUBSTRING(PANEL, 11, 5), 48 | SUBSTRING(PANEL, 16, 1), 49 | SUBSTRING(PANEL, 17, 2), 50 | SUBSTRING(PANEL, 19, 30), 51 | SUBSTRING(PANEL, 49, 25), 52 | SUBSTRING(PANEL, 74, 2), 53 | SUBSTRING(PANEL, 76, 2), 54 | SUBSTRING(PANEL, 78, 10), 55 | SUBSTRING(PANEL, 88, 1), 56 | SUBSTRING(PANEL, 89, 10), 57 | SUBSTRING(PANEL, 99, 30), 58 | SUBSTRING(PANEL, 129, 25), 59 | SUBSTRING(PANEL, 154, 2), 60 | SUBSTRING(PANEL, 156, 4), 61 | SUBSTRING(PANEL, 160, 10) 62 | FROM panel_load; 63 | COMMIT; 64 | DROP TABLE IF EXISTS panel_load; 65 | COMMIT; 66 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_panel_2009.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.panel_2009; 2 | DROP TABLE IF EXISTS panel_load; 3 | CREATE TABLE hmda_public.panel_2009 ( 4 | respondent_id VARCHAR, 5 | msa VARCHAR, 6 | agency_code VARCHAR, 7 | agency_group_code VARCHAR, 8 | respondent_name VARCHAR, 9 | respondent_city VARCHAR, 10 | respondent_state VARCHAR, 11 | respondent_state_fips_number VARCHAR, 12 | assets VARCHAR, 13 | other_lender_code VARCHAR, 14 | parent_respondent_id VARCHAR, 15 | parent_name VARCHAR, 16 | parent_city VARCHAR, 17 | parent_state VARCHAR, 18 | activity_year VARCHAR, 19 | respondent_rssd VARCHAR); 20 | CREATE TEMPORARY TABLE panel_load 21 | (PANEL VARCHAR); -- LAR contains an entire LAR record 22 | COPY panel_load 23 | --Change to your local path 24 | FROM '{data_path}/data/panel/panel_2009.dat' 25 | ENCODING 'latin1'; 26 | COMMIT; 27 | INSERT INTO hmda_public.panel_2009 ( 28 | respondent_id, 29 | msa, 30 | agency_code, 31 | agency_group_code, 32 | respondent_name, 33 | respondent_city, 34 | respondent_state, 35 | respondent_state_fips_number, 36 | assets, 37 | other_lender_code, 38 | parent_respondent_id, 39 | parent_name, 40 | parent_city, 41 | parent_state, 42 | activity_year, 43 | respondent_rssd 44 | ) 45 | SELECT 46 | SUBSTRING(PANEL, 1, 10), 47 | SUBSTRING(PANEL, 11, 5), 48 | SUBSTRING(PANEL, 16, 1), 49 | SUBSTRING(PANEL, 17, 2), 50 | SUBSTRING(PANEL, 19, 30), 51 | SUBSTRING(PANEL, 49, 25), 52 | SUBSTRING(PANEL, 74, 2), 53 | SUBSTRING(PANEL, 76, 2), 54 | SUBSTRING(PANEL, 78, 10), 55 | SUBSTRING(PANEL, 88, 1), 56 | SUBSTRING(PANEL, 89, 10), 57 | SUBSTRING(PANEL, 99, 30), 58 | SUBSTRING(PANEL, 129, 25), 59 | SUBSTRING(PANEL, 154, 2), 60 | SUBSTRING(PANEL, 156, 4), 61 | SUBSTRING(PANEL, 160, 10) 62 | FROM panel_load; 63 | COMMIT; 64 | DROP TABLE IF EXISTS panel_load; 65 | COMMIT; 66 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_panel_2010.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.panel_2010; 2 | DROP TABLE IF EXISTS panel_load; 3 | CREATE TABLE hmda_public.panel_2010 ( 4 | respondent_id VARCHAR, 5 | msa VARCHAR, 6 | agency_code VARCHAR, 7 | agency_group_code VARCHAR, 8 | respondent_name VARCHAR, 9 | respondent_city VARCHAR, 10 | respondent_state VARCHAR, 11 | respondent_state_fips_number VARCHAR, 12 | assets VARCHAR, 13 | other_lender_code VARCHAR, 14 | parent_respondent_id VARCHAR, 15 | parent_name VARCHAR, 16 | parent_city VARCHAR, 17 | parent_state VARCHAR, 18 | activity_year VARCHAR, 19 | respondent_rssd VARCHAR 20 | ); 21 | CREATE TEMPORARY TABLE panel_load 22 | (PANEL VARCHAR); -- LAR contains an entire LAR record 23 | COPY panel_load 24 | --Change to your local path 25 | FROM '{data_path}/data/panel/panel_2010.dat' 26 | ENCODING 'latin1'; 27 | 28 | COMMIT; 29 | 30 | INSERT INTO hmda_public.panel_2010 ( 31 | respondent_id, 32 | msa, 33 | agency_code, 34 | agency_group_code, 35 | respondent_name, 36 | respondent_city, 37 | respondent_state, 38 | respondent_state_fips_number, 39 | assets, 40 | other_lender_code, 41 | parent_respondent_id, 42 | parent_name, 43 | parent_city, 44 | parent_state, 45 | activity_year, 46 | respondent_rssd 47 | ) 48 | SELECT 49 | SUBSTRING(PANEL, 1, 10), 50 | SUBSTRING(PANEL, 11, 5), 51 | SUBSTRING(PANEL, 16, 1), 52 | SUBSTRING(PANEL, 17, 2), 53 | SUBSTRING(PANEL, 19, 30), 54 | SUBSTRING(PANEL, 49, 25), 55 | SUBSTRING(PANEL, 74, 2), 56 | SUBSTRING(PANEL, 76, 2), 57 | SUBSTRING(PANEL, 78, 10), 58 | SUBSTRING(PANEL, 88, 1), 59 | SUBSTRING(PANEL, 89, 10), 60 | SUBSTRING(PANEL, 99, 30), 61 | SUBSTRING(PANEL, 129, 25), 62 | SUBSTRING(PANEL, 154, 2), 63 | SUBSTRING(PANEL, 156, 4), 64 | SUBSTRING(PANEL, 160, 10) 65 | FROM panel_load; 66 | COMMIT; 67 | DROP TABLE IF EXISTS panel_load; 68 | COMMIT; 69 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_panel_2011.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.panel_2011; 2 | DROP TABLE IF EXISTS panel_load; 3 | CREATE TABLE hmda_public.panel_2011 ( 4 | respondent_id VARCHAR, 5 | msa VARCHAR, 6 | agency_code VARCHAR, 7 | agency_group_code VARCHAR, 8 | respondent_name VARCHAR, 9 | respondent_city VARCHAR, 10 | respondent_state VARCHAR, 11 | respondent_state_fips_number VARCHAR, 12 | assets VARCHAR, 13 | other_lender_code VARCHAR, 14 | parent_respondent_id VARCHAR, 15 | parent_name VARCHAR, 16 | parent_city VARCHAR, 17 | parent_state VARCHAR, 18 | activity_year VARCHAR, 19 | respondent_rssd VARCHAR 20 | ); 21 | 22 | CREATE TEMPORARY TABLE panel_load 23 | (PANEL VARCHAR); -- LAR contains an entire LAR record 24 | COPY panel_load 25 | --Change to your local path 26 | FROM '{data_path}/data/panel/panel_2011.dat' 27 | ENCODING 'latin1'; 28 | 29 | COMMIT; 30 | 31 | INSERT INTO hmda_public.panel_2011 ( 32 | respondent_id, 33 | msa, 34 | agency_code, 35 | agency_group_code, 36 | respondent_name, 37 | respondent_city, 38 | respondent_state, 39 | respondent_state_fips_number, 40 | assets, 41 | other_lender_code, 42 | parent_respondent_id, 43 | parent_name, 44 | parent_city, 45 | parent_state, 46 | activity_year, 47 | respondent_rssd 48 | ) 49 | SELECT 50 | SUBSTRING(PANEL, 1, 10), 51 | SUBSTRING(PANEL, 11, 5), 52 | SUBSTRING(PANEL, 16, 1), 53 | SUBSTRING(PANEL, 17, 2), 54 | SUBSTRING(PANEL, 19, 30), 55 | SUBSTRING(PANEL, 49, 25), 56 | SUBSTRING(PANEL, 74, 2), 57 | SUBSTRING(PANEL, 76, 2), 58 | SUBSTRING(PANEL, 78, 10), 59 | SUBSTRING(PANEL, 88, 1), 60 | SUBSTRING(PANEL, 89, 10), 61 | SUBSTRING(PANEL, 99, 30), 62 | SUBSTRING(PANEL, 129, 25), 63 | SUBSTRING(PANEL, 154, 2), 64 | SUBSTRING(PANEL, 156, 4), 65 | SUBSTRING(PANEL, 160, 10) 66 | FROM panel_load; 67 | COMMIT; 68 | DROP TABLE IF EXISTS panel_load; 69 | COMMIT; 70 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_panel_2012.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.panel_2012; 2 | DROP TABLE IF EXISTS panel_load; 3 | CREATE TABLE hmda_public.panel_2012 ( 4 | respondent_id VARCHAR, 5 | msa VARCHAR, 6 | agency_code VARCHAR, 7 | agency_group_code VARCHAR, 8 | respondent_name VARCHAR, 9 | respondent_city VARCHAR, 10 | respondent_state VARCHAR, 11 | respondent_state_fips_number VARCHAR, 12 | assets VARCHAR, 13 | other_lender_code VARCHAR, 14 | parent_respondent_id VARCHAR, 15 | parent_name VARCHAR, 16 | parent_city VARCHAR, 17 | parent_state VARCHAR, 18 | activity_year VARCHAR, 19 | respondent_rssd VARCHAR 20 | ); 21 | CREATE TEMPORARY TABLE panel_load 22 | (PANEL VARCHAR); -- LAR contains an entire LAR record 23 | COPY panel_load 24 | --Change to your local path 25 | FROM '{data_path}/data/panel/panel_2012.dat' 26 | ENCODING 'latin1'; 27 | 28 | COMMIT; 29 | 30 | INSERT INTO hmda_public.panel_2012 ( 31 | respondent_id, 32 | msa, 33 | agency_code, 34 | agency_group_code, 35 | respondent_name, 36 | respondent_city, 37 | respondent_state, 38 | respondent_state_fips_number, 39 | assets, 40 | other_lender_code, 41 | parent_respondent_id, 42 | parent_name, 43 | parent_city, 44 | parent_state, 45 | activity_year, 46 | respondent_rssd 47 | ) 48 | SELECT 49 | SUBSTRING(PANEL, 1, 10), 50 | SUBSTRING(PANEL, 11, 5), 51 | SUBSTRING(PANEL, 16, 1), 52 | SUBSTRING(PANEL, 17, 2), 53 | SUBSTRING(PANEL, 19, 30), 54 | SUBSTRING(PANEL, 49, 25), 55 | SUBSTRING(PANEL, 74, 2), 56 | SUBSTRING(PANEL, 76, 2), 57 | SUBSTRING(PANEL, 78, 10), 58 | SUBSTRING(PANEL, 88, 1), 59 | SUBSTRING(PANEL, 89, 10), 60 | SUBSTRING(PANEL, 99, 30), 61 | SUBSTRING(PANEL, 129, 25), 62 | SUBSTRING(PANEL, 154, 2), 63 | SUBSTRING(PANEL, 156, 4), 64 | SUBSTRING(PANEL, 160, 10) 65 | FROM panel_load; 66 | COMMIT; 67 | DROP TABLE IF EXISTS panel_load; 68 | COMMIT; 69 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_panel_2013.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.panel_2013; 2 | DROP TABLE IF EXISTS panel_load; 3 | CREATE TABLE hmda_public.panel_2013 ( 4 | respondent_id VARCHAR, 5 | msa VARCHAR, 6 | agency_code VARCHAR, 7 | agency_group_code VARCHAR, 8 | respondent_name VARCHAR, 9 | respondent_city VARCHAR, 10 | respondent_state VARCHAR, 11 | respondent_state_fips_number VARCHAR, 12 | assets VARCHAR, 13 | other_lender_code VARCHAR, 14 | parent_respondent_id VARCHAR, 15 | parent_name VARCHAR, 16 | parent_city VARCHAR, 17 | parent_state VARCHAR, 18 | activity_year VARCHAR, 19 | respondent_rssd VARCHAR 20 | ); 21 | 22 | CREATE TEMPORARY TABLE panel_load 23 | (PANEL VARCHAR); -- LAR contains an entire LAR record 24 | COPY panel_load 25 | --Change to your local path 26 | FROM '{data_path}/data/panel/panel_2013.dat' 27 | ENCODING 'latin1'; 28 | 29 | COMMIT; 30 | 31 | INSERT INTO hmda_public.panel_2013 ( 32 | respondent_id, 33 | msa, 34 | agency_code, 35 | agency_group_code, 36 | respondent_name, 37 | respondent_city, 38 | respondent_state, 39 | respondent_state_fips_number, 40 | assets, 41 | other_lender_code, 42 | parent_respondent_id, 43 | parent_name, 44 | parent_city, 45 | parent_state, 46 | activity_year, 47 | respondent_rssd 48 | ) 49 | SELECT 50 | SUBSTRING(PANEL, 1, 10), 51 | SUBSTRING(PANEL, 11, 5), 52 | SUBSTRING(PANEL, 16, 1), 53 | SUBSTRING(PANEL, 17, 2), 54 | SUBSTRING(PANEL, 19, 30), 55 | SUBSTRING(PANEL, 49, 25), 56 | SUBSTRING(PANEL, 74, 2), 57 | SUBSTRING(PANEL, 76, 2), 58 | SUBSTRING(PANEL, 78, 10), 59 | SUBSTRING(PANEL, 88, 1), 60 | SUBSTRING(PANEL, 89, 10), 61 | SUBSTRING(PANEL, 99, 30), 62 | SUBSTRING(PANEL, 129, 25), 63 | SUBSTRING(PANEL, 154, 2), 64 | SUBSTRING(PANEL, 156, 4), 65 | SUBSTRING(PANEL, 160, 10) 66 | FROM panel_load; 67 | COMMIT; 68 | DROP TABLE IF EXISTS panel_load; 69 | COMMIT; 70 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_panel_2014.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.panel_2014; 2 | DROP TABLE IF EXISTS panel_load; 3 | CREATE TABLE hmda_public.panel_2014 ( 4 | activity_year VARCHAR, 5 | respondent_id VARCHAR, 6 | agency_code VARCHAR, 7 | parent_respondent_id VARCHAR, 8 | parent_name VARCHAR, 9 | filler VARCHAR, 10 | parent_city VARCHAR, 11 | parent_state VARCHAR, 12 | region VARCHAR, 13 | assets VARCHAR, 14 | other_lender_code VARCHAR, 15 | respondent_name VARCHAR, 16 | respondent_city VARCHAR, 17 | respondent_state VARCHAR, 18 | filler2 VARCHAR, 19 | filler3 VARCHAR, 20 | top_holder_rssd VARCHAR, 21 | top_holder_name VARCHAR, 22 | top_holder_city VARCHAR, 23 | top_holder_state VARCHAR, 24 | top_holder_country VARCHAR, 25 | respondent_rssd VARCHAR, 26 | parent_rssd VARCHAR, 27 | respondent_state_fips_number VARCHAR 28 | ); 29 | 30 | CREATE TEMPORARY TABLE panel_load 31 | (PANEL VARCHAR); -- LAR contains an entire LAR record 32 | COPY panel_load 33 | 34 | FROM '{data_path}/data/panel/panel_2014.dat' 35 | ENCODING 'latin1'; 36 | 37 | COMMIT; 38 | 39 | INSERT INTO hmda_public.panel_2014 ( 40 | activity_year, 41 | respondent_id, 42 | agency_code, 43 | parent_respondent_id, 44 | parent_name, 45 | filler, 46 | parent_city, 47 | parent_state, 48 | region, 49 | assets, 50 | other_lender_code, 51 | respondent_name, 52 | respondent_city, 53 | respondent_state, 54 | filler2, 55 | filler3, 56 | top_holder_rssd, 57 | top_holder_name, 58 | top_holder_city, 59 | top_holder_state, 60 | top_holder_country, 61 | respondent_rssd, 62 | parent_rssd, 63 | respondent_state_fips_number 64 | ) 65 | SELECT 66 | SUBSTRING(PANEL, 1, 4), 67 | SUBSTRING(PANEL, 5, 10), 68 | SUBSTRING(PANEL, 15, 1), 69 | SUBSTRING(PANEL, 16, 10), 70 | SUBSTRING(PANEL, 26, 30), 71 | SUBSTRING(PANEL, 56, 80), 72 | SUBSTRING(PANEL, 81, 2), 73 | SUBSTRING(PANEL, 83, 2), 74 | SUBSTRING(PANEL, 85, 10), 75 | SUBSTRING(PANEL, 95, 1), 76 | SUBSTRING(PANEL, 96, 30), 77 | SUBSTRING(PANEL, 126, 40), 78 | SUBSTRING(PANEL, 166, 25), 79 | SUBSTRING(PANEL, 191, 2), 80 | SUBSTRING(PANEL, 193, 10), 81 | SUBSTRING(PANEL, 203, 10), 82 | SUBSTRING(PANEL, 213, 10), 83 | SUBSTRING(PANEL, 223, 30), 84 | SUBSTRING(PANEL, 253, 25), 85 | SUBSTRING(PANEL, 278, 2), 86 | SUBSTRING(PANEL, 280, 40), 87 | SUBSTRING(PANEL, 320, 10), 88 | SUBSTRING(PANEL, 330, 10), 89 | SUBSTRING(PANEL, 340, 2) 90 | FROM panel_load; 91 | COMMIT; 92 | DROP TABLE IF EXISTS panel_load; 93 | COMMIT; 94 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_panel_2015.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.panel_2015; 2 | DROP TABLE IF EXISTS panel_load; 3 | CREATE TABLE hmda_public.panel_2015 ( 4 | activity_year VARCHAR, 5 | respondent_id VARCHAR, 6 | agency_code VARCHAR, 7 | parent_respondent_id VARCHAR, 8 | parent_name VARCHAR, 9 | filler VARCHAR, 10 | parent_city VARCHAR, 11 | parent_state VARCHAR, 12 | region VARCHAR, 13 | assets VARCHAR, 14 | other_lender_code VARCHAR, 15 | respondent_name VARCHAR, 16 | respondent_city VARCHAR, 17 | respondent_state VARCHAR, 18 | filler2 VARCHAR, 19 | filler3 VARCHAR, 20 | top_holder_rssd VARCHAR, 21 | top_holder_name VARCHAR, 22 | top_holder_city VARCHAR, 23 | top_holder_state VARCHAR, 24 | top_holder_country VARCHAR, 25 | respondent_rssd VARCHAR, 26 | parent_rssd VARCHAR, 27 | respondent_state_fips_number VARCHAR 28 | ); 29 | 30 | CREATE TEMPORARY TABLE panel_load 31 | (PANEL VARCHAR); -- LAR contains an entire LAR record 32 | COPY panel_load 33 | 34 | FROM '{data_path}/data/panel/panel_2015.txt' 35 | ENCODING 'latin1'; 36 | 37 | COMMIT; 38 | 39 | INSERT INTO hmda_public.panel_2015 ( 40 | activity_year, 41 | respondent_id, 42 | agency_code, 43 | parent_respondent_id, 44 | parent_name, 45 | filler, 46 | parent_city, 47 | parent_state, 48 | region, 49 | assets, 50 | other_lender_code, 51 | respondent_name, 52 | respondent_city, 53 | respondent_state, 54 | filler2, 55 | filler3, 56 | top_holder_rssd, 57 | top_holder_name, 58 | top_holder_city, 59 | top_holder_state, 60 | top_holder_country, 61 | respondent_rssd, 62 | parent_rssd, 63 | respondent_state_fips_number 64 | ) 65 | SELECT 66 | SUBSTRING(PANEL, 1, 4), 67 | SUBSTRING(PANEL, 5, 10), 68 | SUBSTRING(PANEL, 15, 1), 69 | SUBSTRING(PANEL, 16, 10), 70 | SUBSTRING(PANEL, 26, 30), 71 | SUBSTRING(PANEL, 56, 80), 72 | SUBSTRING(PANEL, 81, 2), 73 | SUBSTRING(PANEL, 83, 2), 74 | SUBSTRING(PANEL, 85, 10), 75 | SUBSTRING(PANEL, 95, 1), 76 | SUBSTRING(PANEL, 96, 30), 77 | SUBSTRING(PANEL, 126, 40), 78 | SUBSTRING(PANEL, 166, 25), 79 | SUBSTRING(PANEL, 191, 2), 80 | SUBSTRING(PANEL, 193, 10), 81 | SUBSTRING(PANEL, 203, 10), 82 | SUBSTRING(PANEL, 213, 10), 83 | SUBSTRING(PANEL, 223, 30), 84 | SUBSTRING(PANEL, 253, 25), 85 | SUBSTRING(PANEL, 278, 2), 86 | SUBSTRING(PANEL, 280, 40), 87 | SUBSTRING(PANEL, 320, 10), 88 | SUBSTRING(PANEL, 330, 10), 89 | SUBSTRING(PANEL, 340, 2) 90 | FROM panel_load; 91 | COMMIT; 92 | DROP TABLE IF EXISTS panel_load; 93 | COMMIT; 94 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_panel_2016.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.panel_2016; 2 | DROP TABLE IF EXISTS panel_load; 3 | CREATE TABLE hmda_public.panel_2016 ( 4 | activity_year VARCHAR, 5 | respondent_id VARCHAR, 6 | agency_code VARCHAR, 7 | parent_respondent_id VARCHAR, 8 | parent_name VARCHAR, 9 | filler VARCHAR, 10 | parent_city VARCHAR, 11 | parent_state VARCHAR, 12 | region VARCHAR, 13 | assets VARCHAR, 14 | other_lender_code VARCHAR, 15 | respondent_name VARCHAR, 16 | respondent_city VARCHAR, 17 | respondent_state VARCHAR, 18 | filler2 VARCHAR, 19 | filler3 VARCHAR, 20 | top_holder_rssd VARCHAR, 21 | top_holder_name VARCHAR, 22 | top_holder_city VARCHAR, 23 | top_holder_state VARCHAR, 24 | top_holder_country VARCHAR, 25 | respondent_rssd VARCHAR, 26 | parent_rssd VARCHAR, 27 | respondent_state_fips_number VARCHAR); 28 | 29 | CREATE TEMPORARY TABLE panel_load 30 | (PANEL VARCHAR); -- LAR contains an entire LAR record 31 | 32 | COPY panel_load 33 | FROM '{data_path}/data/panel/panel_2016.txt' 34 | ENCODING 'latin1'; 35 | COMMIT; 36 | INSERT INTO hmda_public.panel_2016 ( 37 | activity_year, 38 | respondent_id, 39 | agency_code, 40 | parent_respondent_id, 41 | parent_name, 42 | filler, 43 | parent_city, 44 | parent_state, 45 | region, 46 | assets, 47 | other_lender_code, 48 | respondent_name, 49 | respondent_city, 50 | respondent_state, 51 | filler2, 52 | filler3, 53 | top_holder_rssd, 54 | top_holder_name, 55 | top_holder_city, 56 | top_holder_state, 57 | top_holder_country, 58 | respondent_rssd, 59 | parent_rssd, 60 | respondent_state_fips_number 61 | ) 62 | SELECT 63 | SUBSTRING(PANEL, 1, 4), 64 | SUBSTRING(PANEL, 5, 10), 65 | SUBSTRING(PANEL, 15, 1), 66 | SUBSTRING(PANEL, 16, 10), 67 | SUBSTRING(PANEL, 26, 30), 68 | SUBSTRING(PANEL, 56, 80), 69 | SUBSTRING(PANEL, 81, 2), 70 | SUBSTRING(PANEL, 83, 2), 71 | SUBSTRING(PANEL, 85, 10), 72 | SUBSTRING(PANEL, 95, 1), 73 | SUBSTRING(PANEL, 96, 30), 74 | SUBSTRING(PANEL, 126, 40), 75 | SUBSTRING(PANEL, 166, 25), 76 | SUBSTRING(PANEL, 191, 2), 77 | SUBSTRING(PANEL, 193, 10), 78 | SUBSTRING(PANEL, 203, 10), 79 | SUBSTRING(PANEL, 213, 10), 80 | SUBSTRING(PANEL, 223, 30), 81 | SUBSTRING(PANEL, 253, 25), 82 | SUBSTRING(PANEL, 278, 2), 83 | SUBSTRING(PANEL, 280, 40), 84 | SUBSTRING(PANEL, 320, 10), 85 | SUBSTRING(PANEL, 330, 10), 86 | SUBSTRING(PANEL, 340, 2) 87 | FROM panel_load; 88 | COMMIT; 89 | DROP TABLE IF EXISTS panel_load; 90 | COMMIT; 91 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_panel_2017.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.panel_2017; 2 | CREATE TABLE hmda_public.panel_2017 ( 3 | activity_year VARCHAR, 4 | respondent_id VARCHAR, 5 | agency_code VARCHAR, 6 | parent_respondent_id VARCHAR, 7 | parent_name VARCHAR, 8 | parent_city VARCHAR, 9 | parent_state VARCHAR, 10 | region VARCHAR, 11 | assets VARCHAR, 12 | other_lender_code VARCHAR, 13 | respondent_name VARCHAR, 14 | respondent_city VARCHAR, 15 | respondent_state VARCHAR, 16 | top_holder_rssd VARCHAR, 17 | top_holder_name VARCHAR, 18 | top_holder_city VARCHAR, 19 | top_holder_state VARCHAR, 20 | top_holder_country VARCHAR, 21 | respondent_rssd VARCHAR, 22 | parent_rssd VARCHAR, 23 | respondent_state_fips_number VARCHAR); 24 | 25 | COPY hmda_public.panel_2017 26 | FROM '{data_path}/data/panel/panel_2017.txt' 27 | DELIMITER '|' ENCODING 'utf8'; 28 | COMMIT; 29 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_panel_2018.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.panel_2018; 2 | CREATE TABLE hmda_public.panel_2018 ( 3 | activity_year VARCHAR(4), 4 | lei VARCHAR(20), 5 | tax_id VARCHAR(11), 6 | agency_code VARCHAR(1), 7 | id_2017 VARCHAR(10), 8 | respondent_rssd VARCHAR, 9 | respondent_name VARCHAR, 10 | respondent_state VARCHAR, 11 | respondent_city VARCHAR, 12 | assets VARCHAR, 13 | other_lender_code VARCHAR, 14 | parent_rssd VARCHAR, 15 | parent_name VARCHAR, 16 | top_holder_rssd VARCHAR, 17 | top_holder_name VARCHAR 18 | ); 19 | 20 | COPY hmda_public.panel_2018 21 | FROM '{data_path}/data/panel/panel_2018.txt' 22 | DELIMITER '|' ENCODING 'utf8' CSV HEADER; 23 | COMMIT; 24 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_panel_2019.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.panel_2019; 2 | CREATE TABLE hmda_public.panel_2019 ( 3 | activity_year VARCHAR(4), 4 | lei VARCHAR(20), 5 | tax_id VARCHAR(10), 6 | agency_code VARCHAR(1), 7 | id_2017 VARCHAR(10), 8 | respondent_rssd VARCHAR, 9 | respondent_name VARCHAR, 10 | respondent_state VARCHAR, 11 | respondent_city VARCHAR, 12 | assets VARCHAR, 13 | other_lender_code VARCHAR, 14 | parent_rssd VARCHAR, 15 | parent_name VARCHAR, 16 | top_holder_rssd VARCHAR, 17 | top_holder_name VARCHAR 18 | ); 19 | 20 | COPY hmda_public.panel_2019 21 | FROM '{data_path}/data/panel/panel_2019.txt' 22 | DELIMITER '|' ENCODING 'utf8' CSV HEADER; 23 | COMMIT; 24 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_panel_2020.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.panel_2020; 2 | CREATE TABLE hmda_public.panel_2020 ( 3 | activity_year VARCHAR(4), 4 | lei VARCHAR(20), 5 | tax_id VARCHAR(10), 6 | agency_code VARCHAR(1), 7 | id_2017 VARCHAR(10), 8 | respondent_rssd VARCHAR, 9 | respondent_name VARCHAR, 10 | respondent_state VARCHAR, 11 | respondent_city VARCHAR, 12 | assets VARCHAR, 13 | other_lender_code VARCHAR, 14 | parent_rssd VARCHAR, 15 | parent_name VARCHAR, 16 | topholder_rssd VARCHAR, 17 | topholder_name VARCHAR 18 | ); 19 | 20 | COPY hmda_public.panel_2020 21 | FROM '{data_path}/data/panel/panel_2020.txt' 22 | DELIMITER '|' ENCODING 'utf8' CSV HEADER; 23 | COMMIT; 24 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_panel_2021.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.panel_2021; 2 | CREATE TABLE hmda_public.panel_2021 ( 3 | activity_year VARCHAR(4), 4 | lei VARCHAR(20), 5 | tax_id VARCHAR(10), 6 | agency_code VARCHAR(1), 7 | id_2017 VARCHAR(10), 8 | arid_2017 VARCHAR(11), 9 | respondent_rssd VARCHAR, 10 | respondent_name VARCHAR, 11 | respondent_state VARCHAR, 12 | respondent_city VARCHAR, 13 | assets VARCHAR, 14 | other_lender_code VARCHAR, 15 | parent_rssd VARCHAR, 16 | parent_name VARCHAR, 17 | top_holder_rssd VARCHAR, 18 | top_holder_name VARCHAR 19 | ); 20 | 21 | COPY hmda_public.panel_2021 22 | FROM '{data_path}/data/panel/panel_2021.txt' 23 | DELIMITER '|' ENCODING 'utf8' CSV HEADER; 24 | COMMIT; 25 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_panel_2022.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.panel_2022; 2 | CREATE TABLE hmda_public.panel_2022 ( 3 | activity_year VARCHAR(4), 4 | lei VARCHAR(20), 5 | tax_id VARCHAR(10), 6 | agency_code VARCHAR(1), 7 | id_2017 VARCHAR(10), 8 | respondent_rssd VARCHAR, 9 | respondent_name VARCHAR, 10 | respondent_state VARCHAR, 11 | respondent_city VARCHAR, 12 | assets VARCHAR, 13 | other_lender_code VARCHAR, 14 | parent_rssd VARCHAR, 15 | parent_name VARCHAR, 16 | top_holder_rssd VARCHAR, 17 | top_holder_name VARCHAR 18 | ); 19 | 20 | COPY hmda_public.panel_2022 21 | FROM '{data_path}/data/panel/panel_2022.txt' 22 | DELIMITER '|' ENCODING 'utf8' CSV HEADER; 23 | COMMIT; 24 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_ts_2004.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.ts_2004, ts_load; 2 | CREATE TABLE hmda_public.ts_2004 ( 3 | activity_year VARCHAR, 4 | agency_code VARCHAR, 5 | respondent_id VARCHAR, 6 | respondent_name VARCHAR, 7 | respondent_address VARCHAR, 8 | respondent_city VARCHAR, 9 | respondent_state VARCHAR, 10 | respondent_zip_code VARCHAR, 11 | parent_name VARCHAR, 12 | parent_address VARCHAR, 13 | parent_city VARCHAR, 14 | parent_state VARCHAR, 15 | parent_zip_code VARCHAR, 16 | edit_status VARCHAR, 17 | tax_id VARCHAR); 18 | 19 | CREATE TEMPORARY TABLE ts_load 20 | (TS VARCHAR) ; -- TS contains an entire TS record per row 21 | 22 | COPY ts_load 23 | -- Change to your local data path 24 | FROM '{data_path}/data/ts/ts_2004.dat' 25 | ENCODING 'latin1'; 26 | COMMIT; 27 | 28 | INSERT INTO hmda_public.ts_2004 ( 29 | activity_year, 30 | agency_code, 31 | respondent_id, 32 | respondent_name, 33 | respondent_address, 34 | respondent_city, 35 | respondent_state, 36 | respondent_zip_code, 37 | parent_name, 38 | parent_address, 39 | parent_city, 40 | parent_state, 41 | parent_zip_code, 42 | edit_status, 43 | tax_id 44 | ) 45 | 46 | SELECT 47 | SUBSTRING(TS, 1, 4), 48 | SUBSTRING(TS, 5, 1), 49 | SUBSTRING(TS, 6, 10), 50 | SUBSTRING(TS, 16, 30), 51 | SUBSTRING(TS, 46, 40), 52 | SUBSTRING(TS, 86, 25), 53 | SUBSTRING(TS, 111,2), 54 | SUBSTRING(TS, 113, 10), 55 | SUBSTRING(TS, 123, 30), 56 | SUBSTRING(TS, 153, 40), 57 | SUBSTRING(TS, 193, 25), 58 | SUBSTRING(TS, 218, 2), 59 | SUBSTRING(TS, 220, 10), 60 | SUBSTRING(TS, 230, 1), 61 | SUBSTRING(TS, 231, 10) 62 | 63 | FROM ts_load; 64 | COMMIT; 65 | DROP TABLE IF EXISTS ts_load; 66 | COMMIT; 67 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_ts_2005.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.ts_2005, ts_load; 2 | CREATE TABLE hmda_public.ts_2005 ( 3 | activity_year VARCHAR, 4 | agency_code VARCHAR, 5 | respondent_id VARCHAR, 6 | respondent_name VARCHAR, 7 | respondent_address VARCHAR, 8 | respondent_city VARCHAR, 9 | respondent_state VARCHAR, 10 | respondent_zip_code VARCHAR, 11 | parent_name VARCHAR, 12 | parent_address VARCHAR, 13 | parent_city VARCHAR, 14 | parent_state VARCHAR, 15 | parent_zip_code VARCHAR, 16 | edit_status VARCHAR, 17 | tax_id VARCHAR); 18 | 19 | CREATE TEMPORARY TABLE ts_load 20 | (TS VARCHAR) ; -- TS contains an entire TS record per row 21 | 22 | COPY ts_load 23 | -- Change to your local data path 24 | FROM '{data_path}/data/ts/ts_2005.dat' 25 | ENCODING 'latin1'; 26 | COMMIT; 27 | 28 | INSERT INTO hmda_public.ts_2005 ( 29 | activity_year, 30 | agency_code, 31 | respondent_id, 32 | respondent_name, 33 | respondent_address, 34 | respondent_city, 35 | respondent_state, 36 | respondent_zip_code, 37 | parent_name, 38 | parent_address, 39 | parent_city, 40 | parent_state, 41 | parent_zip_code, 42 | edit_status, 43 | tax_id 44 | ) 45 | 46 | SELECT 47 | SUBSTRING(TS, 1, 4), 48 | SUBSTRING(TS, 5, 1), 49 | SUBSTRING(TS, 6, 10), 50 | SUBSTRING(TS, 16, 30), 51 | SUBSTRING(TS, 46, 40), 52 | SUBSTRING(TS, 86,25), 53 | SUBSTRING(TS, 111,2), 54 | SUBSTRING(TS, 113, 10), 55 | SUBSTRING(TS, 123, 30), 56 | SUBSTRING(TS, 153, 40), 57 | SUBSTRING(TS, 193, 25), 58 | SUBSTRING(TS, 218, 2), 59 | SUBSTRING(TS, 220, 10), 60 | SUBSTRING(TS, 230, 1), 61 | SUBSTRING(TS, 231, 10) 62 | 63 | FROM ts_load; 64 | COMMIT; 65 | DROP TABLE IF EXISTS ts_load; 66 | COMMIT; 67 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_ts_2006.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.ts_2006, ts_load; 2 | CREATE TABLE hmda_public.ts_2006 ( 3 | activity_year VARCHAR, 4 | agency_code VARCHAR, 5 | respondent_id VARCHAR, 6 | respondent_name VARCHAR, 7 | respondent_address VARCHAR, 8 | respondent_city VARCHAR, 9 | respondent_state VARCHAR, 10 | respondent_zip_code VARCHAR, 11 | parent_name VARCHAR, 12 | parent_address VARCHAR, 13 | parent_city VARCHAR, 14 | parent_state VARCHAR, 15 | parent_zip_code VARCHAR, 16 | edit_status VARCHAR, 17 | tax_id VARCHAR); 18 | 19 | CREATE TEMPORARY TABLE ts_load 20 | (TS VARCHAR) ; -- TS contains an entire TS record per row 21 | 22 | COPY ts_load 23 | -- Change to your local data path 24 | FROM '{data_path}/data/ts/ts_2006.dat' 25 | ENCODING 'latin1'; 26 | COMMIT; 27 | 28 | INSERT INTO hmda_public.ts_2006 ( 29 | activity_year, 30 | agency_code, 31 | respondent_id, 32 | respondent_name, 33 | respondent_address, 34 | respondent_city, 35 | respondent_state, 36 | respondent_zip_code, 37 | parent_name, 38 | parent_address, 39 | parent_city, 40 | parent_state, 41 | parent_zip_code, 42 | edit_status, 43 | tax_id 44 | ) 45 | 46 | SELECT 47 | SUBSTRING(TS, 1, 4), 48 | SUBSTRING(TS, 5, 1), 49 | SUBSTRING(TS, 6, 10), 50 | SUBSTRING(TS, 16, 30), 51 | SUBSTRING(TS, 46, 40), 52 | SUBSTRING(TS, 86,25), 53 | SUBSTRING(TS, 111,2), 54 | SUBSTRING(TS, 113, 10), 55 | SUBSTRING(TS, 123, 30), 56 | SUBSTRING(TS, 153, 40), 57 | SUBSTRING(TS, 193, 25), 58 | SUBSTRING(TS, 218, 2), 59 | SUBSTRING(TS, 220, 10), 60 | SUBSTRING(TS, 230, 1), 61 | SUBSTRING(TS, 231, 10) 62 | 63 | FROM ts_load; 64 | COMMIT; 65 | DROP TABLE IF EXISTS ts_load; 66 | COMMIT; 67 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_ts_2007.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.ts_2007, ts_load; 2 | CREATE TABLE hmda_public.ts_2007 ( 3 | as_of_date VARCHAR, 4 | activity_year VARCHAR, 5 | agency_code VARCHAR, 6 | respondent_id VARCHAR, 7 | respondent_name VARCHAR, 8 | respondent_address VARCHAR, 9 | respondent_city VARCHAR, 10 | respondent_state VARCHAR, 11 | respondent_zip_code VARCHAR, 12 | parent_name VARCHAR, 13 | parent_address VARCHAR, 14 | parent_city VARCHAR, 15 | parent_state VARCHAR, 16 | parent_zip_code VARCHAR, 17 | edit_status VARCHAR, 18 | tax_id VARCHAR); 19 | 20 | CREATE TEMPORARY TABLE ts_load 21 | (TS VARCHAR) ; -- TS contains an entire TS record per row 22 | 23 | COPY ts_load 24 | -- Change to your local data path 25 | 26 | FROM '{data_path}/data/ts/ts_2007.dat' 27 | 28 | ENCODING 'latin1'; 29 | COMMIT; 30 | 31 | INSERT INTO hmda_public.ts_2007 ( 32 | activity_year, 33 | agency_code, 34 | respondent_id, 35 | respondent_name, 36 | respondent_address, 37 | respondent_city, 38 | respondent_state, 39 | respondent_zip_code, 40 | parent_name, 41 | parent_address, 42 | parent_city, 43 | parent_state, 44 | parent_zip_code, 45 | edit_status, 46 | tax_id 47 | ) 48 | 49 | SELECT 50 | SUBSTRING(TS, 1, 4), 51 | SUBSTRING(TS, 5, 1), 52 | SUBSTRING(TS, 6, 10), 53 | SUBSTRING(TS, 16, 30), 54 | SUBSTRING(TS, 46, 40), 55 | SUBSTRING(TS, 86,25), 56 | SUBSTRING(TS, 111,2), 57 | SUBSTRING(TS, 113, 10), 58 | SUBSTRING(TS, 123, 30), 59 | SUBSTRING(TS, 153, 40), 60 | SUBSTRING(TS, 193, 25), 61 | SUBSTRING(TS, 218, 2), 62 | SUBSTRING(TS, 220, 10), 63 | SUBSTRING(TS, 230, 1), 64 | SUBSTRING(TS, 231, 10) 65 | 66 | FROM ts_load; 67 | COMMIT; 68 | DROP TABLE IF EXISTS ts_load; 69 | COMMIT; 70 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_ts_2008.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.ts_2008, ts_load; 2 | CREATE TABLE hmda_public.ts_2008 ( 3 | activity_year VARCHAR, 4 | agency_code VARCHAR, 5 | respondent_id VARCHAR, 6 | respondent_name VARCHAR, 7 | respondent_address VARCHAR, 8 | respondent_city VARCHAR, 9 | respondent_state VARCHAR, 10 | respondent_zip_code VARCHAR, 11 | parent_name VARCHAR, 12 | parent_address VARCHAR, 13 | parent_city VARCHAR, 14 | parent_state VARCHAR, 15 | parent_zip_code VARCHAR, 16 | edit_status VARCHAR, 17 | tax_id VARCHAR); 18 | 19 | CREATE TEMPORARY TABLE ts_load 20 | (TS VARCHAR) ; -- TS contains an entire TS record per row 21 | 22 | COPY ts_load 23 | -- Change to your local data path 24 | FROM '{data_path}/data/ts/ts_2008.dat' 25 | ENCODING 'latin1'; 26 | COMMIT; 27 | 28 | INSERT INTO hmda_public.ts_2008 ( 29 | activity_year, 30 | agency_code, 31 | respondent_id, 32 | respondent_name, 33 | respondent_address, 34 | respondent_city, 35 | respondent_state, 36 | respondent_zip_code, 37 | parent_name, 38 | parent_address, 39 | parent_city, 40 | parent_state, 41 | parent_zip_code, 42 | edit_status, 43 | tax_id 44 | ) 45 | 46 | SELECT 47 | SUBSTRING(TS, 1, 4), 48 | SUBSTRING(TS, 5, 1), 49 | SUBSTRING(TS, 6, 10), 50 | SUBSTRING(TS, 16, 30), 51 | SUBSTRING(TS, 46, 40), 52 | SUBSTRING(TS, 86,25), 53 | SUBSTRING(TS, 111,2), 54 | SUBSTRING(TS, 113, 10), 55 | SUBSTRING(TS, 123, 30), 56 | SUBSTRING(TS, 153, 40), 57 | SUBSTRING(TS, 193, 25), 58 | SUBSTRING(TS, 218, 2), 59 | SUBSTRING(TS, 220, 10), 60 | SUBSTRING(TS, 230, 1), 61 | SUBSTRING(TS, 231, 10) 62 | 63 | FROM ts_load; 64 | COMMIT; 65 | DROP TABLE IF EXISTS ts_load; 66 | COMMIT; 67 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_ts_2009.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.ts_2009, ts_load; 2 | CREATE TABLE hmda_public.ts_2009 ( 3 | activity_year VARCHAR, 4 | agency_code VARCHAR, 5 | respondent_id VARCHAR, 6 | respondent_name VARCHAR, 7 | respondent_address VARCHAR, 8 | respondent_city VARCHAR, 9 | respondent_state VARCHAR, 10 | respondent_zip_code VARCHAR, 11 | parent_name VARCHAR, 12 | parent_address VARCHAR, 13 | parent_city VARCHAR, 14 | parent_state VARCHAR, 15 | parent_zip_code VARCHAR, 16 | edit_status VARCHAR, 17 | tax_id VARCHAR); 18 | 19 | CREATE TEMPORARY TABLE ts_load 20 | (TS VARCHAR) ; -- TS contains an entire TS record per row 21 | 22 | COPY ts_load 23 | -- Change to your local data path 24 | FROM '{data_path}/data/ts/ts_2009.dat' 25 | ENCODING 'latin1'; 26 | COMMIT; 27 | 28 | INSERT INTO hmda_public.ts_2009 ( 29 | activity_year, 30 | agency_code, 31 | respondent_id, 32 | respondent_name, 33 | respondent_address, 34 | respondent_city, 35 | respondent_state, 36 | respondent_zip_code, 37 | parent_name, 38 | parent_address, 39 | parent_city, 40 | parent_state, 41 | parent_zip_code, 42 | edit_status, 43 | tax_id 44 | ) 45 | 46 | SELECT 47 | SUBSTRING(TS, 1, 4), 48 | SUBSTRING(TS, 5, 1), 49 | SUBSTRING(TS, 6, 10), 50 | SUBSTRING(TS, 16, 30), 51 | SUBSTRING(TS, 46, 40), 52 | SUBSTRING(TS, 86,25), 53 | SUBSTRING(TS, 111,2), 54 | SUBSTRING(TS, 113, 10), 55 | SUBSTRING(TS, 123, 30), 56 | SUBSTRING(TS, 153, 40), 57 | SUBSTRING(TS, 193, 25), 58 | SUBSTRING(TS, 218, 2), 59 | SUBSTRING(TS, 220, 10), 60 | SUBSTRING(TS, 230, 1), 61 | SUBSTRING(TS, 231, 10) 62 | 63 | FROM ts_load; 64 | COMMIT; 65 | DROP TABLE IF EXISTS ts_load; 66 | COMMIT; 67 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_ts_2010.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.ts_2010, ts_load; 2 | CREATE TABLE hmda_public.ts_2010 ( 3 | activity_year VARCHAR, 4 | agency_code VARCHAR, 5 | respondent_id VARCHAR, 6 | respondent_name VARCHAR, 7 | respondent_address VARCHAR, 8 | respondent_city VARCHAR, 9 | respondent_state VARCHAR, 10 | respondent_zip_code VARCHAR, 11 | parent_name VARCHAR, 12 | parent_address VARCHAR, 13 | parent_city VARCHAR, 14 | parent_state VARCHAR, 15 | parent_zip_code VARCHAR, 16 | edit_status VARCHAR, 17 | tax_id VARCHAR); 18 | 19 | CREATE TEMPORARY TABLE ts_load 20 | (TS VARCHAR) ; -- TS contains an entire TS record per row 21 | 22 | COPY ts_load 23 | -- Change to your local data path 24 | FROM '{data_path}/data/ts/ts_2010.dat' 25 | ENCODING 'latin1'; 26 | COMMIT; 27 | 28 | INSERT INTO hmda_public.ts_2010 ( 29 | activity_year, 30 | agency_code, 31 | respondent_id, 32 | respondent_name, 33 | respondent_address, 34 | respondent_city, 35 | respondent_state, 36 | respondent_zip_code, 37 | parent_name, 38 | parent_address, 39 | parent_city, 40 | parent_state, 41 | parent_zip_code, 42 | edit_status, 43 | tax_id 44 | ) 45 | 46 | SELECT 47 | SUBSTRING(TS, 1, 4), 48 | SUBSTRING(TS, 5, 1), 49 | SUBSTRING(TS, 6, 10), 50 | SUBSTRING(TS, 16, 30), 51 | SUBSTRING(TS, 46, 40), 52 | SUBSTRING(TS, 86,25), 53 | SUBSTRING(TS, 111,2), 54 | SUBSTRING(TS, 113, 10), 55 | SUBSTRING(TS, 123, 30), 56 | SUBSTRING(TS, 153, 40), 57 | SUBSTRING(TS, 193, 25), 58 | SUBSTRING(TS, 218, 2), 59 | SUBSTRING(TS, 220, 10), 60 | SUBSTRING(TS, 230, 1), 61 | SUBSTRING(TS, 231, 10) 62 | 63 | FROM ts_load; 64 | COMMIT; 65 | DROP TABLE IF EXISTS ts_load; 66 | COMMIT; 67 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_ts_2011.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.ts_2011, ts_load; 2 | CREATE TABLE hmda_public.ts_2011 ( 3 | activity_year VARCHAR, 4 | agency_code VARCHAR, 5 | respondent_id VARCHAR, 6 | respondent_name VARCHAR, 7 | respondent_address VARCHAR, 8 | respondent_city VARCHAR, 9 | respondent_state VARCHAR, 10 | respondent_zip_code VARCHAR, 11 | parent_name VARCHAR, 12 | parent_address VARCHAR, 13 | parent_city VARCHAR, 14 | parent_state VARCHAR, 15 | parent_zip_code VARCHAR, 16 | edit_status VARCHAR, 17 | tax_id VARCHAR); 18 | 19 | CREATE TEMPORARY TABLE ts_load 20 | (TS VARCHAR) ; -- TS contains an entire TS record per row 21 | 22 | COPY ts_load 23 | -- Change to your local data path 24 | FROM '{data_path}/data/ts/ts_2011.dat' 25 | ENCODING 'latin1'; 26 | COMMIT; 27 | 28 | INSERT INTO hmda_public.ts_2011 ( 29 | activity_year, 30 | agency_code, 31 | respondent_id, 32 | respondent_name, 33 | respondent_address, 34 | respondent_city, 35 | respondent_state, 36 | respondent_zip_code, 37 | parent_name, 38 | parent_address, 39 | parent_city, 40 | parent_state, 41 | parent_zip_code, 42 | edit_status, 43 | tax_id 44 | ) 45 | 46 | SELECT 47 | SUBSTRING(TS, 1, 4), 48 | SUBSTRING(TS, 5, 1), 49 | SUBSTRING(TS, 6, 10), 50 | SUBSTRING(TS, 16, 30), 51 | SUBSTRING(TS, 46, 40), 52 | SUBSTRING(TS, 86,25), 53 | SUBSTRING(TS, 111,2), 54 | SUBSTRING(TS, 113, 10), 55 | SUBSTRING(TS, 123, 30), 56 | SUBSTRING(TS, 153, 40), 57 | SUBSTRING(TS, 193, 25), 58 | SUBSTRING(TS, 218, 2), 59 | SUBSTRING(TS, 220, 10), 60 | SUBSTRING(TS, 230, 1), 61 | SUBSTRING(TS, 231, 10) 62 | 63 | FROM ts_load; 64 | COMMIT; 65 | DROP TABLE IF EXISTS ts_load; 66 | COMMIT; 67 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_ts_2012.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.ts_2012, ts_load; 2 | CREATE TABLE hmda_public.ts_2012 ( 3 | activity_year VARCHAR, 4 | agency_code VARCHAR, 5 | respondent_id VARCHAR, 6 | respondent_name VARCHAR, 7 | respondent_address VARCHAR, 8 | respondent_city VARCHAR, 9 | respondent_state VARCHAR, 10 | respondent_zip_code VARCHAR, 11 | parent_name VARCHAR, 12 | parent_address VARCHAR, 13 | parent_city VARCHAR, 14 | parent_state VARCHAR, 15 | parent_zip_code VARCHAR, 16 | edit_status VARCHAR, 17 | tax_id VARCHAR); 18 | 19 | CREATE TEMPORARY TABLE ts_load 20 | (TS VARCHAR) ; -- TS contains an entire TS record per row 21 | 22 | COPY ts_load 23 | -- Change to your local data path 24 | FROM '{data_path}/data/ts/ts_2012.dat' 25 | ENCODING 'latin1'; 26 | COMMIT; 27 | 28 | INSERT INTO hmda_public.ts_2012 ( 29 | activity_year, 30 | agency_code, 31 | respondent_id, 32 | respondent_name, 33 | respondent_address, 34 | respondent_city, 35 | respondent_state, 36 | respondent_zip_code, 37 | parent_name, 38 | parent_address, 39 | parent_city, 40 | parent_state, 41 | parent_zip_code, 42 | edit_status, 43 | tax_id 44 | ) 45 | 46 | SELECT 47 | SUBSTRING(TS, 1, 4), 48 | SUBSTRING(TS, 5, 1), 49 | SUBSTRING(TS, 6, 10), 50 | SUBSTRING(TS, 16, 30), 51 | SUBSTRING(TS, 46, 40), 52 | SUBSTRING(TS, 86,25), 53 | SUBSTRING(TS, 111,2), 54 | SUBSTRING(TS, 113, 10), 55 | SUBSTRING(TS, 123, 30), 56 | SUBSTRING(TS, 153, 40), 57 | SUBSTRING(TS, 193, 25), 58 | SUBSTRING(TS, 218, 2), 59 | SUBSTRING(TS, 220, 10), 60 | SUBSTRING(TS, 230, 1), 61 | SUBSTRING(TS, 231, 10) 62 | 63 | FROM ts_load; 64 | COMMIT; 65 | DROP TABLE IF EXISTS ts_load; 66 | COMMIT; 67 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_ts_2013.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.ts_2013, ts_load; 2 | CREATE TABLE hmda_public.ts_2013 ( 3 | activity_year VARCHAR, 4 | agency_code VARCHAR, 5 | respondent_id VARCHAR, 6 | respondent_name VARCHAR, 7 | respondent_address VARCHAR, 8 | respondent_city VARCHAR, 9 | respondent_state VARCHAR, 10 | respondent_zip_code VARCHAR, 11 | parent_name VARCHAR, 12 | parent_address VARCHAR, 13 | parent_city VARCHAR, 14 | parent_state VARCHAR, 15 | parent_zip_code VARCHAR, 16 | edit_status VARCHAR, 17 | tax_id VARCHAR); 18 | 19 | CREATE TEMPORARY TABLE ts_load 20 | (TS VARCHAR) ; -- TS contains an entire TS record per row 21 | 22 | COPY ts_load 23 | -- Change to your local data path 24 | FROM '{data_path}/data/ts/ts_2013.dat' 25 | ENCODING 'latin1'; 26 | COMMIT; 27 | 28 | INSERT INTO hmda_public.ts_2013 ( 29 | activity_year, 30 | agency_code, 31 | respondent_id, 32 | respondent_name, 33 | respondent_address, 34 | respondent_city, 35 | respondent_state, 36 | respondent_zip_code, 37 | parent_name, 38 | parent_address, 39 | parent_city, 40 | parent_state, 41 | parent_zip_code, 42 | edit_status, 43 | tax_id 44 | ) 45 | 46 | SELECT 47 | SUBSTRING(TS, 1, 4), 48 | SUBSTRING(TS, 5, 1), 49 | SUBSTRING(TS, 6, 10), 50 | SUBSTRING(TS, 16, 30), 51 | SUBSTRING(TS, 46, 40), 52 | SUBSTRING(TS, 86,25), 53 | SUBSTRING(TS, 111,2), 54 | SUBSTRING(TS, 113, 10), 55 | SUBSTRING(TS, 123, 30), 56 | SUBSTRING(TS, 153, 40), 57 | SUBSTRING(TS, 193, 25), 58 | SUBSTRING(TS, 218, 2), 59 | SUBSTRING(TS, 220, 10), 60 | SUBSTRING(TS, 230, 1), 61 | SUBSTRING(TS, 231, 10) 62 | 63 | FROM ts_load; 64 | COMMIT; 65 | DROP TABLE IF EXISTS ts_load; 66 | COMMIT; 67 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_ts_2014.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.ts_2014, ts_load; 2 | CREATE TABLE hmda_public.ts_2014( 3 | activity_year VARCHAR, 4 | agency_code VARCHAR, 5 | respondent_id VARCHAR, 6 | respondent_name VARCHAR, 7 | respondent_address VARCHAR, 8 | respondent_city VARCHAR, 9 | respondent_state VARCHAR, 10 | respondent_zip_code VARCHAR, 11 | parent_name VARCHAR, 12 | parent_address VARCHAR, 13 | parent_city VARCHAR, 14 | parent_state VARCHAR, 15 | parent_zip_code VARCHAR, 16 | edit_status VARCHAR, 17 | tax_id VARCHAR 18 | ); 19 | 20 | CREATE TEMPORARY TABLE ts_load 21 | (TS VARCHAR) ; -- TS contains an entire TS record per row 22 | 23 | COPY ts_load 24 | -- Change to your local data path 25 | FROM '{data_path}/data/ts/ts_2014' -- Exclude the .txt extension, as 2014 TS is weird. 26 | ENCODING 'latin1'; 27 | COMMIT; 28 | 29 | INSERT INTO hmda_public.ts_2014 ( 30 | activity_year, 31 | agency_code, 32 | respondent_id, 33 | respondent_name, 34 | respondent_address, 35 | respondent_city, 36 | respondent_state, 37 | respondent_zip_code, 38 | parent_name, 39 | parent_address, 40 | parent_city, 41 | parent_state, 42 | parent_zip_code, 43 | edit_status, 44 | tax_id 45 | ) 46 | 47 | SELECT 48 | SUBSTRING(TS, 1, 4), 49 | SUBSTRING(TS, 5, 1), 50 | SUBSTRING(TS, 6, 10), 51 | SUBSTRING(TS, 16, 30), 52 | SUBSTRING(TS, 46, 40), 53 | SUBSTRING(TS, 86,25), 54 | SUBSTRING(TS, 111,2), 55 | SUBSTRING(TS, 113, 10), 56 | SUBSTRING(TS, 123, 30), 57 | SUBSTRING(TS, 153, 40), 58 | SUBSTRING(TS, 193, 25), 59 | SUBSTRING(TS, 218, 2), 60 | SUBSTRING(TS, 220, 10), 61 | SUBSTRING(TS, 230, 1), 62 | SUBSTRING(TS, 231, 10) 63 | 64 | FROM ts_load; 65 | COMMIT; 66 | DROP TABLE IF EXISTS ts_load; 67 | COMMIT; 68 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_ts_2015.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.ts_2015; 2 | CREATE TABLE hmda_public.ts_2015( 3 | activity_year VARCHAR, 4 | respondent_id VARCHAR, 5 | agency_code VARCHAR, 6 | tax_id VARCHAR, 7 | respondent_name VARCHAR, 8 | respondent_address VARCHAR, 9 | respondent_city VARCHAR, 10 | respondent_state VARCHAR, 11 | respondent_zip_code VARCHAR, 12 | parent_name VARCHAR, 13 | parent_address VARCHAR, 14 | parent_city VARCHAR, 15 | parent_state VARCHAR, 16 | parent_zip_code VARCHAR, 17 | respondent_name_panel VARCHAR, 18 | respondent_city_panel VARCHAR, 19 | respondent_state_panel VARCHAR, 20 | assets_panel VARCHAR, 21 | olc_panel VARCHAR, 22 | region_code_panel VARCHAR, 23 | lar_count VARCHAR, 24 | edit_status VARCHAR 25 | ); 26 | 27 | COPY hmda_public.ts_2015 28 | -- Change to your local data path 29 | FROM '{data_path}/data/ts/ts_2015.txt' 30 | DELIMITER E'\t' ENCODING 'latin1'; 31 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_ts_2016.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.ts_2016; 2 | CREATE TABLE hmda_public.ts_2016( 3 | activity_year VARCHAR, 4 | respondent_id VARCHAR, 5 | agency_code VARCHAR, 6 | tax_id VARCHAR, 7 | respondent_name VARCHAR, 8 | respondent_address VARCHAR, 9 | respondent_city VARCHAR, 10 | respondent_state VARCHAR, 11 | respondent_zip_code VARCHAR, 12 | parent_name VARCHAR, 13 | parent_address VARCHAR, 14 | parent_city VARCHAR, 15 | parent_state VARCHAR, 16 | parent_zip_code VARCHAR, 17 | respondent_name_panel VARCHAR, 18 | respondent_city_panel VARCHAR, 19 | respondent_state_panel VARCHAR, 20 | assets_panel VARCHAR, 21 | olc_panel VARCHAR, 22 | region_code_panel VARCHAR, 23 | lar_count VARCHAR, 24 | edit_status VARCHAR 25 | ); 26 | 27 | COPY hmda_public.ts_2016 28 | -- Change to your local data path 29 | FROM '{data_path}/data/ts/ts_2016_fixed.txt' 30 | DELIMITER E'\t' ENCODING 'latin1'; 31 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_ts_2017.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.ts_2017; 2 | CREATE TABLE hmda_public.ts_2017 ( 3 | activity_year VARCHAR, 4 | respondent_id VARCHAR, 5 | agency_code VARCHAR, 6 | tax_id VARCHAR, 7 | respondent_name VARCHAR, 8 | respondent_address VARCHAR, 9 | respondent_city VARCHAR, 10 | respondent_state VARCHAR, 11 | respondent_zip_code VARCHAR, 12 | parent_name VARCHAR, 13 | parent_address VARCHAR, 14 | parent_city VARCHAR, 15 | parent_state VARCHAR, 16 | parent_zip_code VARCHAR, 17 | lar_count VARCHAR 18 | ); 19 | 20 | COPY hmda_public.ts_2017 21 | FROM 22 | '{data_path}/data/ts/ts_2017.txt' 23 | DELIMITER '|' ENCODING 'latin1'; 24 | COMMIT; 25 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_ts_2018.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.ts_2018; 2 | CREATE TABLE hmda_public.ts_2018 ( 3 | activity_year VARCHAR(4), 4 | calendar_quarter VARCHAR(1), 5 | lei VARCHAR(20), 6 | tax_id VARCHAR(10), 7 | agency_code VARCHAR(1), 8 | respondent_name VARCHAR, 9 | respondent_state VARCHAR(2), 10 | respondent_city VARCHAR, 11 | respondent_zip_code VARCHAR, 12 | lar_count VARCHAR 13 | ); 14 | 15 | COPY hmda_public.ts_2018 16 | FROM 17 | '{data_path}/data/ts/ts_2018.txt' 18 | DELIMITER '|' ENCODING 'utf8' CSV HEADER; 19 | COMMIT; 20 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_ts_2019.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.ts_2019; 2 | CREATE TABLE hmda_public.ts_2019 ( 3 | activity_year VARCHAR(4), 4 | calendar_quarter VARCHAR(1), 5 | lei VARCHAR(20), 6 | tax_id VARCHAR(10), 7 | agency_code VARCHAR(1), 8 | respondent_name VARCHAR, 9 | respondent_state VARCHAR(2), 10 | respondent_city VARCHAR, 11 | respondent_zip_code VARCHAR, 12 | lar_count VARCHAR 13 | ); 14 | 15 | COPY hmda_public.ts_2019 16 | FROM 17 | '{data_path}/data/ts/ts_2019.txt' 18 | DELIMITER '|' ENCODING 'utf8' CSV HEADER; 19 | COMMIT; 20 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_ts_2020.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.ts_2020; 2 | CREATE TABLE hmda_public.ts_2020 ( 3 | activity_year VARCHAR(4), 4 | calendar_quarter VARCHAR(1), 5 | lei VARCHAR(20), 6 | tax_id VARCHAR(10), 7 | agency_code VARCHAR(1), 8 | respondent_name VARCHAR, 9 | respondent_state VARCHAR(2), 10 | respondent_city VARCHAR, 11 | respondent_zip_code VARCHAR, 12 | lar_count VARCHAR 13 | ); 14 | 15 | COPY hmda_public.ts_2020 16 | FROM 17 | '{data_path}/data/ts/ts_2020.txt' 18 | DELIMITER '|' ENCODING 'utf8' CSV HEADER; 19 | COMMIT; 20 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_ts_2021.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.ts_2021; 2 | CREATE TABLE hmda_public.ts_2021 ( 3 | activity_year VARCHAR(4), 4 | calendar_quarter VARCHAR(1), 5 | lei VARCHAR(20), 6 | tax_id VARCHAR(10), 7 | agency_code VARCHAR(1), 8 | respondent_name VARCHAR, 9 | respondent_state VARCHAR(2), 10 | respondent_city VARCHAR, 11 | respondent_zip_code VARCHAR, 12 | lar_count VARCHAR 13 | ); 14 | 15 | COPY hmda_public.ts_2021 16 | FROM 17 | '{data_path}/data/ts/ts_2021.txt' 18 | DELIMITER '|' ENCODING 'utf8' CSV HEADER; 19 | COMMIT; 20 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_and_load_ts_2022.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS hmda_public.ts_2022; 2 | CREATE TABLE hmda_public.ts_2022 ( 3 | activity_year VARCHAR(4), 4 | calendar_quarter VARCHAR, 5 | lei VARCHAR(20), 6 | tax_id VARCHAR(10), 7 | agency_code VARCHAR(1), 8 | respondent_name VARCHAR, 9 | respondent_state VARCHAR(2), 10 | respondent_city VARCHAR, 11 | respondent_zip_code VARCHAR, 12 | lar_count VARCHAR 13 | ); 14 | 15 | COPY hmda_public.ts_2022 16 | FROM 17 | '{data_path}/data/ts/ts_2022.txt' 18 | DELIMITER '|' ENCODING 'utf8' CSV HEADER; 19 | COMMIT; 20 | -------------------------------------------------------------------------------- /load_scripts/SQL/create_hmda_db.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cfpb/HMDA_Data_Science_Kit/b053b1a6ebc4c264cc12ca3888d955cee4be4857/load_scripts/SQL/create_hmda_db.sql -------------------------------------------------------------------------------- /load_scripts/create_and_load_lar_2004_2017.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #Set default PSQL connection variables 3 | USER="postgres" #the postgres username for the connection 4 | PASSWORD="" #user's password for the connection 5 | DATABASE="postgres" #Sets connection database 6 | HOST="localhost" #host address of the database 7 | PORT="5432" #connection port of the database 8 | 9 | if [ $# -gt 0 ]; then 10 | #while [ "${1}" != "" ]; 11 | for var in "$@" 12 | do #iterate over passed command line arguments as long as one remains 13 | case $1 in 14 | -u | --user ) shift 15 | USER=$1 16 | echo "user: $USER" 17 | ;; 18 | 19 | -p | --password ) shift 20 | PASSWORD=$1 21 | ;; 22 | 23 | -d | --database ) shift 24 | DATABASE=$1 25 | echo "database: $DATABASE" 26 | ;; 27 | 28 | -h | --host ) shift 29 | HOST=$1 30 | echo "HOST: $HOST" 31 | ;; 32 | 33 | -o | --port ) shift 34 | PORT=$1 35 | ;; 36 | 37 | --help ) echo 38 | echo "*****************" 39 | echo "This section explains the command line arguments available in this script" 40 | echo "" 41 | echo "-u or --user sets the username for connection to database" 42 | echo "-p or --password sets the password for the connection" 43 | echo "-d or --database sets the database for connection" 44 | echo "-h or --host sets the connection address for the database" 45 | echo "-o or --port sets the connection port for the database" 46 | echo "--help calls this output" 47 | exit 48 | ;; 49 | esac 50 | done 51 | fi 52 | 53 | # Set these environmental variables to override them, 54 | # but they have safe defaults. 55 | export PGUSER="${PGUSER-${USER}}" #postgres user for the connection 56 | export PGPASSWORD="${PGPASSWORD-${PASSWORD}}" #PG password for the user 57 | export PGDATABASE="${PGDATABASE-${DATABASE}}" #Sets connection database 58 | export PGHOST="${PGHOST-${HOST}}" #host address of the database 59 | export PGPORT="${PGPORT-${PORT}}" #connection port of the database 60 | 61 | echo "setting data paths for SQL load" 62 | bash load_scripts/write_data_paths.sh 63 | 64 | echo "creating HMDA database and hmda_public schema" 65 | psql $PGDATABASE $PGUSER << EOF 66 | CREATE DATABASE hmda; 67 | \c hmda 68 | CREATE SCHEMA hmda_public; 69 | EOF 70 | 71 | echo "creating HMDA LAR tables for 2004-2017" 72 | psql $PGDATABASE $PGUSER << EOF 73 | \c hmda; 74 | set schema 'hmda_public'; 75 | \i 'load_scripts/SQL/create_and_load_lar_2004.sql' 76 | \i 'load_scripts/SQL/create_and_load_lar_2005.sql' 77 | \i 'load_scripts/SQL/create_and_load_lar_2006.sql' 78 | \i 'load_scripts/SQL/create_and_load_lar_2007.sql' 79 | \i 'load_scripts/SQL/create_and_load_lar_2008.sql' 80 | \i 'load_scripts/SQL/create_and_load_lar_2009.sql' 81 | \i 'load_scripts/SQL/create_and_load_lar_2010.sql' 82 | \i 'load_scripts/SQL/create_and_load_lar_2011.sql' 83 | \i 'load_scripts/SQL/create_and_load_lar_2012.sql' 84 | \i 'load_scripts/SQL/create_and_load_lar_2013.sql' 85 | \i 'load_scripts/SQL/create_and_load_lar_2014.sql' 86 | \i 'load_scripts/SQL/create_and_load_lar_2015.sql' 87 | \i 'load_scripts/SQL/create_and_load_lar_2016.sql' 88 | \i 'load_scripts/SQL/create_and_load_lar_2017.sql' 89 | EOF -------------------------------------------------------------------------------- /load_scripts/create_and_load_lar_2004_2019.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #Set default PSQL connection variables 3 | USER="postgres" #the postgres username for the connection 4 | PASSWORD="" #user's password for the connection 5 | DATABASE="postgres" #Sets connection database 6 | HOST="localhost" #host address of the database 7 | PORT="5432" #connection port of the database 8 | 9 | if [ $# -gt 0 ]; then 10 | #while [ "${1}" != "" ]; 11 | for var in "$@" 12 | do #iterate over passed command line arguments as long as one remains 13 | case $1 in 14 | -u | --user ) shift 15 | USER=$1 16 | echo "user: $USER" 17 | ;; 18 | 19 | -p | --password ) shift 20 | PASSWORD=$1 21 | ;; 22 | 23 | -d | --database ) shift 24 | DATABASE=$1 25 | echo "database: $DATABASE" 26 | ;; 27 | 28 | -h | --host ) shift 29 | HOST=$1 30 | echo "HOST: $HOST" 31 | ;; 32 | 33 | -o | --port ) shift 34 | PORT=$1 35 | ;; 36 | 37 | --help ) echo 38 | echo "*****************" 39 | echo "This section explains the command line arguments available in this script" 40 | echo "" 41 | echo "-u or --user sets the username for connection to database" 42 | echo "-p or --password sets the password for the connection" 43 | echo "-d or --database sets the database for connection" 44 | echo "-h or --host sets the connection address for the database" 45 | echo "-o or --port sets the connection port for the database" 46 | echo "--help calls this output" 47 | exit 48 | ;; 49 | esac 50 | done 51 | fi 52 | 53 | # Set these environmental variables to override them, 54 | # but they have safe defaults. 55 | export PGUSER="${PGUSER-${USER}}" #postgres user for the connection 56 | export PGPASSWORD="${PGPASSWORD-${PASSWORD}}" #PG password for the user 57 | export PGDATABASE="${PGDATABASE-${DATABASE}}" #Sets connection database 58 | export PGHOST="${PGHOST-${HOST}}" #host address of the database 59 | export PGPORT="${PGPORT-${PORT}}" #connection port of the database 60 | 61 | echo "setting data paths for SQL load" 62 | bash load_scripts/write_data_paths.sh 63 | 64 | echo "creating HMDA database and hmda_public schema" 65 | psql $PGDATABASE $PGUSER << EOF 66 | CREATE DATABASE hmda; 67 | \c hmda 68 | CREATE SCHEMA hmda_public; 69 | EOF 70 | 71 | echo "creating HMDA LAR tables for 2004-2019" 72 | psql $PGDATABASE $PGUSER << EOF 73 | \c hmda; 74 | set schema 'hmda_public'; 75 | \i 'load_scripts/SQL/create_and_load_lar_2004.sql' 76 | \i 'load_scripts/SQL/create_and_load_lar_2005.sql' 77 | \i 'load_scripts/SQL/create_and_load_lar_2006.sql' 78 | \i 'load_scripts/SQL/create_and_load_lar_2007.sql' 79 | \i 'load_scripts/SQL/create_and_load_lar_2008.sql' 80 | \i 'load_scripts/SQL/create_and_load_lar_2009.sql' 81 | \i 'load_scripts/SQL/create_and_load_lar_2010.sql' 82 | \i 'load_scripts/SQL/create_and_load_lar_2011.sql' 83 | \i 'load_scripts/SQL/create_and_load_lar_2012.sql' 84 | \i 'load_scripts/SQL/create_and_load_lar_2013.sql' 85 | \i 'load_scripts/SQL/create_and_load_lar_2014.sql' 86 | \i 'load_scripts/SQL/create_and_load_lar_2015.sql' 87 | \i 'load_scripts/SQL/create_and_load_lar_2016.sql' 88 | \i 'load_scripts/SQL/create_and_load_lar_2017.sql' 89 | \i 'load_scripts/SQL/create_and_load_lar_2018.sql' 90 | \i 'load_scripts/SQL/create_and_load_lar_2019.sql' 91 | EOF -------------------------------------------------------------------------------- /load_scripts/create_and_load_lar_2004_2020.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #Set default PSQL connection variables 3 | USER="postgres" #the postgres username for the connection 4 | PASSWORD="" #user's password for the connection 5 | DATABASE="postgres" #Sets connection database 6 | HOST="localhost" #host address of the database 7 | PORT="5432" #connection port of the database 8 | 9 | if [ $# -gt 0 ]; then 10 | #while [ "${1}" != "" ]; 11 | for var in "$@" 12 | do #iterate over passed command line arguments as long as one remains 13 | case $1 in 14 | -u | --user ) shift 15 | USER=$1 16 | echo "user: $USER" 17 | ;; 18 | 19 | -p | --password ) shift 20 | PASSWORD=$1 21 | ;; 22 | 23 | -d | --database ) shift 24 | DATABASE=$1 25 | echo "database: $DATABASE" 26 | ;; 27 | 28 | -h | --host ) shift 29 | HOST=$1 30 | echo "HOST: $HOST" 31 | ;; 32 | 33 | -o | --port ) shift 34 | PORT=$1 35 | ;; 36 | 37 | --help ) echo 38 | echo "*****************" 39 | echo "This section explains the command line arguments available in this script" 40 | echo "" 41 | echo "-u or --user sets the username for connection to database" 42 | echo "-p or --password sets the password for the connection" 43 | echo "-d or --database sets the database for connection" 44 | echo "-h or --host sets the connection address for the database" 45 | echo "-o or --port sets the connection port for the database" 46 | echo "--help calls this output" 47 | exit 48 | ;; 49 | esac 50 | done 51 | fi 52 | 53 | # Set these environmental variables to override them, 54 | # but they have safe defaults. 55 | export PGUSER="${PGUSER-${USER}}" #postgres user for the connection 56 | export PGPASSWORD="${PGPASSWORD-${PASSWORD}}" #PG password for the user 57 | export PGDATABASE="${PGDATABASE-${DATABASE}}" #Sets connection database 58 | export PGHOST="${PGHOST-${HOST}}" #host address of the database 59 | export PGPORT="${PGPORT-${PORT}}" #connection port of the database 60 | 61 | echo "setting data paths for SQL load" 62 | bash load_scripts/write_data_paths.sh 63 | 64 | echo "creating HMDA database and hmda_public schema" 65 | psql $PGDATABASE $PGUSER << EOF 66 | CREATE DATABASE hmda; 67 | \c hmda 68 | CREATE SCHEMA hmda_public; 69 | EOF 70 | 71 | echo "creating HMDA LAR tables for 2004-2020" 72 | psql $PGDATABASE $PGUSER << EOF 73 | \c hmda; 74 | set schema 'hmda_public'; 75 | \i 'load_scripts/SQL/create_and_load_lar_2004.sql' 76 | \i 'load_scripts/SQL/create_and_load_lar_2005.sql' 77 | \i 'load_scripts/SQL/create_and_load_lar_2006.sql' 78 | \i 'load_scripts/SQL/create_and_load_lar_2007.sql' 79 | \i 'load_scripts/SQL/create_and_load_lar_2008.sql' 80 | \i 'load_scripts/SQL/create_and_load_lar_2009.sql' 81 | \i 'load_scripts/SQL/create_and_load_lar_2010.sql' 82 | \i 'load_scripts/SQL/create_and_load_lar_2011.sql' 83 | \i 'load_scripts/SQL/create_and_load_lar_2012.sql' 84 | \i 'load_scripts/SQL/create_and_load_lar_2013.sql' 85 | \i 'load_scripts/SQL/create_and_load_lar_2014.sql' 86 | \i 'load_scripts/SQL/create_and_load_lar_2015.sql' 87 | \i 'load_scripts/SQL/create_and_load_lar_2016.sql' 88 | \i 'load_scripts/SQL/create_and_load_lar_2017.sql' 89 | \i 'load_scripts/SQL/create_and_load_lar_2018.sql' 90 | \i 'load_scripts/SQL/create_and_load_lar_2019.sql' 91 | \i 'load_scripts/SQL/create_and_load_lar_2020.sql' 92 | EOF -------------------------------------------------------------------------------- /load_scripts/create_and_load_panel_2004_2017.sh: -------------------------------------------------------------------------------- 1 | 2 | #!/bin/bash 3 | #Set default PSQL connection variables 4 | USER="postgres" #the postgres username for the connection 5 | PASSWORD="" #user's password for the connection 6 | DATABASE="postgres" #Sets connection database 7 | HOST="localhost" #host address of the database 8 | PORT="5432" #connection port of the database 9 | 10 | if [ $# -gt 0 ]; then 11 | #while [ "${1}" != "" ]; 12 | for var in "$@" 13 | do #iterate over passed command line arguments as long as one remains 14 | case $1 in 15 | -u | --user ) shift 16 | USER=$1 17 | echo "user: $USER" 18 | ;; 19 | 20 | -p | --password ) shift 21 | PASSWORD=$1 22 | ;; 23 | 24 | -d | --database ) shift 25 | DATABASE=$1 26 | echo "database: $DATABASE" 27 | ;; 28 | 29 | -h | --host ) shift 30 | HOST=$1 31 | echo "HOST: $HOST" 32 | ;; 33 | 34 | -o | --port ) shift 35 | PORT=$1 36 | ;; 37 | 38 | --help ) echo 39 | echo "*****************" 40 | echo "This section explains the command line arguments available in this script" 41 | echo "" 42 | echo "-u or --user sets the username for connection to database" 43 | echo "-p or --password sets the password for the connection" 44 | echo "-d or --database sets the database for connection" 45 | echo "-h or --host sets the connection address for the database" 46 | echo "-o or --port sets the connection port for the database" 47 | echo "--help calls this output" 48 | exit 49 | ;; 50 | esac 51 | done 52 | fi 53 | 54 | # Set these environmental variables to override them, 55 | # but they have safe defaults. 56 | export PGUSER="${PGUSER-${USER}}" #postgres user for the connection 57 | export PGPASSWORD="${PGPASSWORD-${PASSWORD}}" #PG password for the user 58 | export PGDATABASE="${PGDATABASE-${DATABASE}}" #Sets connection database 59 | export PGHOST="${PGHOST-${HOST}}" #host address of the database 60 | export PGPORT="${PGPORT-${PORT}}" #connection port of the database 61 | 62 | echo "setting data paths for SQL load" 63 | bash load_scripts/write_data_paths.sh 64 | 65 | echo "creating HMDA database and hmda_public schema" 66 | psql $PGDATABASE $PGUSER << EOF 67 | CREATE DATABASE hmda; 68 | \c hmda 69 | CREATE SCHEMA hmda_public; 70 | EOF 71 | 72 | echo "creating HMDA tables for 2004-2017" 73 | psql $PGDATABASE $PGUSER << EOF 74 | \c hmda; 75 | set schema 'hmda_public'; 76 | \i 'load_scripts/SQL/create_and_load_panel_2004.sql' 77 | \i 'load_scripts/SQL/create_and_load_panel_2005.sql' 78 | \i 'load_scripts/SQL/create_and_load_panel_2006.sql' 79 | \i 'load_scripts/SQL/create_and_load_panel_2007.sql' 80 | \i 'load_scripts/SQL/create_and_load_panel_2008.sql' 81 | \i 'load_scripts/SQL/create_and_load_panel_2009.sql' 82 | \i 'load_scripts/SQL/create_and_load_panel_2010.sql' 83 | \i 'load_scripts/SQL/create_and_load_panel_2011.sql' 84 | \i 'load_scripts/SQL/create_and_load_panel_2012.sql' 85 | \i 'load_scripts/SQL/create_and_load_panel_2013.sql' 86 | \i 'load_scripts/SQL/create_and_load_panel_2014.sql' 87 | \i 'load_scripts/SQL/create_and_load_panel_2015.sql' 88 | \i 'load_scripts/SQL/create_and_load_panel_2016.sql' 89 | \i 'load_scripts/SQL/create_and_load_panel_2017.sql' 90 | EOF -------------------------------------------------------------------------------- /load_scripts/create_and_load_panel_2004_2019.sh: -------------------------------------------------------------------------------- 1 | 2 | #!/bin/bash 3 | #Set default PSQL connection variables 4 | USER="postgres" #the postgres username for the connection 5 | PASSWORD="" #user's password for the connection 6 | DATABASE="postgres" #Sets connection database 7 | HOST="localhost" #host address of the database 8 | PORT="5432" #connection port of the database 9 | 10 | if [ $# -gt 0 ]; then 11 | #while [ "${1}" != "" ]; 12 | for var in "$@" 13 | do #iterate over passed command line arguments as long as one remains 14 | case $1 in 15 | -u | --user ) shift 16 | USER=$1 17 | echo "user: $USER" 18 | ;; 19 | 20 | -p | --password ) shift 21 | PASSWORD=$1 22 | ;; 23 | 24 | -d | --database ) shift 25 | DATABASE=$1 26 | echo "database: $DATABASE" 27 | ;; 28 | 29 | -h | --host ) shift 30 | HOST=$1 31 | echo "HOST: $HOST" 32 | ;; 33 | 34 | -o | --port ) shift 35 | PORT=$1 36 | ;; 37 | 38 | --help ) echo 39 | echo "*****************" 40 | echo "This section explains the command line arguments available in this script" 41 | echo "" 42 | echo "-u or --user sets the username for connection to database" 43 | echo "-p or --password sets the password for the connection" 44 | echo "-d or --database sets the database for connection" 45 | echo "-h or --host sets the connection address for the database" 46 | echo "-o or --port sets the connection port for the database" 47 | echo "--help calls this output" 48 | exit 49 | ;; 50 | esac 51 | done 52 | fi 53 | 54 | # Set these environmental variables to override them, 55 | # but they have safe defaults. 56 | export PGUSER="${PGUSER-${USER}}" #postgres user for the connection 57 | export PGPASSWORD="${PGPASSWORD-${PASSWORD}}" #PG password for the user 58 | export PGDATABASE="${PGDATABASE-${DATABASE}}" #Sets connection database 59 | export PGHOST="${PGHOST-${HOST}}" #host address of the database 60 | export PGPORT="${PGPORT-${PORT}}" #connection port of the database 61 | 62 | echo "setting data paths for SQL load" 63 | bash load_scripts/write_data_paths.sh 64 | 65 | echo "creating HMDA database and hmda_public schema" 66 | psql $PGDATABASE $PGUSER << EOF 67 | CREATE DATABASE hmda; 68 | \c hmda 69 | CREATE SCHEMA hmda_public; 70 | EOF 71 | 72 | echo "creating HMDA tables for 2004-2019" 73 | psql $PGDATABASE $PGUSER << EOF 74 | \c hmda; 75 | set schema 'hmda_public'; 76 | \i 'load_scripts/SQL/create_and_load_panel_2004.sql' 77 | \i 'load_scripts/SQL/create_and_load_panel_2005.sql' 78 | \i 'load_scripts/SQL/create_and_load_panel_2006.sql' 79 | \i 'load_scripts/SQL/create_and_load_panel_2007.sql' 80 | \i 'load_scripts/SQL/create_and_load_panel_2008.sql' 81 | \i 'load_scripts/SQL/create_and_load_panel_2009.sql' 82 | \i 'load_scripts/SQL/create_and_load_panel_2010.sql' 83 | \i 'load_scripts/SQL/create_and_load_panel_2011.sql' 84 | \i 'load_scripts/SQL/create_and_load_panel_2012.sql' 85 | \i 'load_scripts/SQL/create_and_load_panel_2013.sql' 86 | \i 'load_scripts/SQL/create_and_load_panel_2014.sql' 87 | \i 'load_scripts/SQL/create_and_load_panel_2015.sql' 88 | \i 'load_scripts/SQL/create_and_load_panel_2016.sql' 89 | \i 'load_scripts/SQL/create_and_load_panel_2017.sql' 90 | \i 'load_scripts/SQL/create_and_load_panel_2018.sql' 91 | \i 'load_scripts/SQL/create_and_load_panel_2019.sql' 92 | EOF -------------------------------------------------------------------------------- /load_scripts/create_and_load_panel_2004_2020.sh: -------------------------------------------------------------------------------- 1 | 2 | #!/bin/bash 3 | #Set default PSQL connection variables 4 | USER="postgres" #the postgres username for the connection 5 | PASSWORD="" #user's password for the connection 6 | DATABASE="postgres" #Sets connection database 7 | HOST="localhost" #host address of the database 8 | PORT="5432" #connection port of the database 9 | 10 | if [ $# -gt 0 ]; then 11 | #while [ "${1}" != "" ]; 12 | for var in "$@" 13 | do #iterate over passed command line arguments as long as one remains 14 | case $1 in 15 | -u | --user ) shift 16 | USER=$1 17 | echo "user: $USER" 18 | ;; 19 | 20 | -p | --password ) shift 21 | PASSWORD=$1 22 | ;; 23 | 24 | -d | --database ) shift 25 | DATABASE=$1 26 | echo "database: $DATABASE" 27 | ;; 28 | 29 | -h | --host ) shift 30 | HOST=$1 31 | echo "HOST: $HOST" 32 | ;; 33 | 34 | -o | --port ) shift 35 | PORT=$1 36 | ;; 37 | 38 | --help ) echo 39 | echo "*****************" 40 | echo "This section explains the command line arguments available in this script" 41 | echo "" 42 | echo "-u or --user sets the username for connection to database" 43 | echo "-p or --password sets the password for the connection" 44 | echo "-d or --database sets the database for connection" 45 | echo "-h or --host sets the connection address for the database" 46 | echo "-o or --port sets the connection port for the database" 47 | echo "--help calls this output" 48 | exit 49 | ;; 50 | esac 51 | done 52 | fi 53 | 54 | # Set these environmental variables to override them, 55 | # but they have safe defaults. 56 | export PGUSER="${PGUSER-${USER}}" #postgres user for the connection 57 | export PGPASSWORD="${PGPASSWORD-${PASSWORD}}" #PG password for the user 58 | export PGDATABASE="${PGDATABASE-${DATABASE}}" #Sets connection database 59 | export PGHOST="${PGHOST-${HOST}}" #host address of the database 60 | export PGPORT="${PGPORT-${PORT}}" #connection port of the database 61 | 62 | echo "setting data paths for SQL load" 63 | bash load_scripts/write_data_paths.sh 64 | 65 | echo "creating HMDA database and hmda_public schema" 66 | psql $PGDATABASE $PGUSER << EOF 67 | CREATE DATABASE hmda; 68 | \c hmda 69 | CREATE SCHEMA hmda_public; 70 | EOF 71 | 72 | echo "creating HMDA tables for 2004-2020" 73 | psql $PGDATABASE $PGUSER << EOF 74 | \c hmda; 75 | set schema 'hmda_public'; 76 | \i 'load_scripts/SQL/create_and_load_panel_2004.sql' 77 | \i 'load_scripts/SQL/create_and_load_panel_2005.sql' 78 | \i 'load_scripts/SQL/create_and_load_panel_2006.sql' 79 | \i 'load_scripts/SQL/create_and_load_panel_2007.sql' 80 | \i 'load_scripts/SQL/create_and_load_panel_2008.sql' 81 | \i 'load_scripts/SQL/create_and_load_panel_2009.sql' 82 | \i 'load_scripts/SQL/create_and_load_panel_2010.sql' 83 | \i 'load_scripts/SQL/create_and_load_panel_2011.sql' 84 | \i 'load_scripts/SQL/create_and_load_panel_2012.sql' 85 | \i 'load_scripts/SQL/create_and_load_panel_2013.sql' 86 | \i 'load_scripts/SQL/create_and_load_panel_2014.sql' 87 | \i 'load_scripts/SQL/create_and_load_panel_2015.sql' 88 | \i 'load_scripts/SQL/create_and_load_panel_2016.sql' 89 | \i 'load_scripts/SQL/create_and_load_panel_2017.sql' 90 | \i 'load_scripts/SQL/create_and_load_panel_2018.sql' 91 | \i 'load_scripts/SQL/create_and_load_panel_2019.sql' 92 | \i 'load_scripts/SQL/create_and_load_panel_2020.sql' 93 | EOF -------------------------------------------------------------------------------- /load_scripts/create_and_load_ts_2004_2017.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #Set default PSQL connection variables 4 | USER="postgres" #the postgres username for the connection 5 | PASSWORD="" #user's password for the connection 6 | DATABASE="postgres" #Sets connection database 7 | HOST="localhost" #host address of the database 8 | PORT="5432" #connection port of the database 9 | 10 | if [ $# -gt 0 ]; then 11 | #while [ "${1}" != "" ]; 12 | for var in "$@" 13 | do #iterate over passed command line arguments as long as one remains 14 | case $1 in 15 | -u | --user ) shift 16 | USER=$1 17 | echo "user: $USER" 18 | ;; 19 | 20 | -p | --password ) shift 21 | PASSWORD=$1 22 | ;; 23 | 24 | -d | --database ) shift 25 | DATABASE=$1 26 | echo "database: $DATABASE" 27 | ;; 28 | 29 | -h | --host ) shift 30 | HOST=$1 31 | echo "HOST: $HOST" 32 | ;; 33 | 34 | -o | --port ) shift 35 | PORT=$1 36 | ;; 37 | 38 | --help ) echo 39 | echo "*****************" 40 | echo "This section explains the command line arguments available in this script" 41 | echo "" 42 | echo "-u or --user sets the username for connection to database" 43 | echo "-p or --password sets the password for the connection" 44 | echo "-d or --database sets the database for connection" 45 | echo "-h or --host sets the connection address for the database" 46 | echo "-o or --port sets the connection port for the database" 47 | echo "--help calls this output" 48 | exit 49 | ;; 50 | esac 51 | done 52 | fi 53 | 54 | # Set these environmental variables to override them, 55 | # but they have safe defaults. 56 | export PGUSER="${PGUSER-${USER}}" #postgres user for the connection 57 | export PGPASSWORD="${PGPASSWORD-${PASSWORD}}" #PG password for the user 58 | export PGDATABASE="${PGDATABASE-${DATABASE}}" #Sets connection database 59 | export PGHOST="${PGHOST-${HOST}}" #host address of the database 60 | export PGPORT="${PGPORT-${PORT}}" #connection port of the database 61 | 62 | echo "setting data paths for SQL load" 63 | bash load_scripts/write_data_paths.sh 64 | 65 | echo "creating HMDA database and hmda_public schema" 66 | psql $PGDATABASE $PGUSER << EOF 67 | CREATE DATABASE hmda; 68 | \c hmda 69 | CREATE SCHEMA hmda_public; 70 | EOF 71 | 72 | echo "removing extra tab in ts 2016" 73 | python3 load_scripts/remove_tab_ts_2016.py 74 | 75 | echo "creating HMDA tables for 2004-2017" 76 | psql $PGDATABASE $PGUSER << EOF 77 | \c hmda; 78 | set schema 'hmda_public'; 79 | \i 'load_scripts/SQL/create_and_load_ts_2004.sql' 80 | \i 'load_scripts/SQL/create_and_load_ts_2005.sql' 81 | \i 'load_scripts/SQL/create_and_load_ts_2006.sql' 82 | \i 'load_scripts/SQL/create_and_load_ts_2007.sql' 83 | \i 'load_scripts/SQL/create_and_load_ts_2008.sql' 84 | \i 'load_scripts/SQL/create_and_load_ts_2009.sql' 85 | \i 'load_scripts/SQL/create_and_load_ts_2010.sql' 86 | \i 'load_scripts/SQL/create_and_load_ts_2011.sql' 87 | \i 'load_scripts/SQL/create_and_load_ts_2012.sql' 88 | \i 'load_scripts/SQL/create_and_load_ts_2013.sql' 89 | \i 'load_scripts/SQL/create_and_load_ts_2014.sql' 90 | \i 'load_scripts/SQL/create_and_load_ts_2015.sql' 91 | \i 'load_scripts/SQL/create_and_load_ts_2016.sql' 92 | \i 'load_scripts/SQL/create_and_load_ts_2017.sql' 93 | EOF 94 | -------------------------------------------------------------------------------- /load_scripts/create_and_load_ts_2004_2019.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #Set default PSQL connection variables 4 | USER="postgres" #the postgres username for the connection 5 | PASSWORD="" #user's password for the connection 6 | DATABASE="postgres" #Sets connection database 7 | HOST="localhost" #host address of the database 8 | PORT="5432" #connection port of the database 9 | 10 | if [ $# -gt 0 ]; then 11 | #while [ "${1}" != "" ]; 12 | for var in "$@" 13 | do #iterate over passed command line arguments as long as one remains 14 | case $1 in 15 | -u | --user ) shift 16 | USER=$1 17 | echo "user: $USER" 18 | ;; 19 | 20 | -p | --password ) shift 21 | PASSWORD=$1 22 | ;; 23 | 24 | -d | --database ) shift 25 | DATABASE=$1 26 | echo "database: $DATABASE" 27 | ;; 28 | 29 | -h | --host ) shift 30 | HOST=$1 31 | echo "HOST: $HOST" 32 | ;; 33 | 34 | -o | --port ) shift 35 | PORT=$1 36 | ;; 37 | 38 | --help ) echo 39 | echo "*****************" 40 | echo "This section explains the command line arguments available in this script" 41 | echo "" 42 | echo "-u or --user sets the username for connection to database" 43 | echo "-p or --password sets the password for the connection" 44 | echo "-d or --database sets the database for connection" 45 | echo "-h or --host sets the connection address for the database" 46 | echo "-o or --port sets the connection port for the database" 47 | echo "--help calls this output" 48 | exit 49 | ;; 50 | esac 51 | done 52 | fi 53 | 54 | # Set these environmental variables to override them, 55 | # but they have safe defaults. 56 | export PGUSER="${PGUSER-${USER}}" #postgres user for the connection 57 | export PGPASSWORD="${PGPASSWORD-${PASSWORD}}" #PG password for the user 58 | export PGDATABASE="${PGDATABASE-${DATABASE}}" #Sets connection database 59 | export PGHOST="${PGHOST-${HOST}}" #host address of the database 60 | export PGPORT="${PGPORT-${PORT}}" #connection port of the database 61 | 62 | echo "setting data paths for SQL load" 63 | bash load_scripts/write_data_paths.sh 64 | 65 | echo "creating HMDA database and hmda_public schema" 66 | psql $PGDATABASE $PGUSER << EOF 67 | CREATE DATABASE hmda; 68 | \c hmda 69 | CREATE SCHEMA hmda_public; 70 | EOF 71 | 72 | echo "removing extra tab in ts 2016" 73 | python3 load_scripts/remove_tab_ts_2016.py 74 | 75 | echo "creating HMDA tables for 2004-2019" 76 | psql $PGDATABASE $PGUSER << EOF 77 | \c hmda; 78 | set schema 'hmda_public'; 79 | \i 'load_scripts/SQL/create_and_load_ts_2004.sql' 80 | \i 'load_scripts/SQL/create_and_load_ts_2005.sql' 81 | \i 'load_scripts/SQL/create_and_load_ts_2006.sql' 82 | \i 'load_scripts/SQL/create_and_load_ts_2007.sql' 83 | \i 'load_scripts/SQL/create_and_load_ts_2008.sql' 84 | \i 'load_scripts/SQL/create_and_load_ts_2009.sql' 85 | \i 'load_scripts/SQL/create_and_load_ts_2010.sql' 86 | \i 'load_scripts/SQL/create_and_load_ts_2011.sql' 87 | \i 'load_scripts/SQL/create_and_load_ts_2012.sql' 88 | \i 'load_scripts/SQL/create_and_load_ts_2013.sql' 89 | \i 'load_scripts/SQL/create_and_load_ts_2014.sql' 90 | \i 'load_scripts/SQL/create_and_load_ts_2015.sql' 91 | \i 'load_scripts/SQL/create_and_load_ts_2016.sql' 92 | \i 'load_scripts/SQL/create_and_load_ts_2017.sql' 93 | \i 'load_scripts/SQL/create_and_load_ts_2018.sql' 94 | \i 'load_scripts/SQL/create_and_load_ts_2019.sql' 95 | EOF 96 | -------------------------------------------------------------------------------- /load_scripts/create_hmda_db.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #This script is designed to quickly set up a local database 4 | #Caution: using this script to develop applications or set up a database on a network will create security vulnerabilities. 5 | 6 | #this script should: 7 | # create a hmda database 8 | # create a hmda_public schema 9 | 10 | #Set default PSQL connection variables 11 | USER="postgres" #the postgres username for the connection 12 | PASSWORD="" #user's password for the connection 13 | DATABASE="postgres" #Sets connection database 14 | HOST="localhost" #host address of the database 15 | PORT="5432" #connection port of the database 16 | 17 | if [ $# -gt 0 ]; then 18 | #while [ "${1}" != "" ]; 19 | for var in "$@" 20 | do #iterate over passed command line arguments as long as one remains 21 | case $1 in 22 | -u | --user ) shift 23 | USER=$1 24 | echo "user: $USER" 25 | ;; 26 | 27 | -p | --password ) shift 28 | PASSWORD=$1 29 | ;; 30 | 31 | -d | --database ) shift 32 | DATABASE=$1 33 | echo "database: $DATABASE" 34 | ;; 35 | 36 | -h | --host ) shift 37 | HOST=$1 38 | echo "HOST: $HOST" 39 | ;; 40 | 41 | -o | --port ) shift 42 | PORT=$1 43 | ;; 44 | 45 | --help ) echo 46 | echo "*****************" 47 | echo "This section explains the command line arguments available in this script" 48 | echo "" 49 | echo "-u or --user sets the username for connection to database" 50 | echo "-p or --password sets the password for the connection" 51 | echo "-d or --database sets the database for connection" 52 | echo "-h or --host sets the connection address for the database" 53 | echo "-o or --port sets the connection port for the database" 54 | echo "--help calls this output" 55 | exit 56 | ;; 57 | esac 58 | done 59 | fi 60 | 61 | # Set these environmental variables to override them, 62 | # but they have safe defaults. 63 | export PGUSER="${PGUSER-${USER}}" #postgres user for the connection 64 | export PGPASSWORD="${PGPASSWORD-${PASSWORD}}" #PG password for the user 65 | export PGDATABASE="${PGDATABASE-${DATABASE}}" #Sets connection database 66 | export PGHOST="${PGHOST-${HOST}}" #host address of the database 67 | export PGPORT="${PGPORT-${PORT}}" #connection port of the database 68 | 69 | echo "creating HMDA database and hmda_public schema" 70 | #format is psql dbnam username 71 | psql $PGDATABASE $PGUSER << EOF 72 | CREATE DATABASE hmda; 73 | \c hmda 74 | CREATE SCHEMA hmda_public; 75 | EOF -------------------------------------------------------------------------------- /load_scripts/remove_tab_ts_2016.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | with open("data/ts/ts_2016.txt", "r") as infile: 4 | with open("data/ts/ts_2016_fixed.txt", "w") as outfile: 5 | count = 0 6 | 7 | lines = infile.readlines() 8 | print(count) 9 | for line in lines: 10 | count +=1 11 | if count == 2674: 12 | print(line) 13 | line = line.replace("7940 SOUTH HARLEM AVENUE ", "7940 SOUTH HARLEM AVENUE") 14 | outfile.write(line) 15 | 16 | -------------------------------------------------------------------------------- /load_scripts/reset_path.py: -------------------------------------------------------------------------------- 1 | import fileinput 2 | import os 3 | from os import listdir 4 | from os.path import isfile, join 5 | 6 | dir_path = os.path.dirname(os.path.realpath(__file__)) 7 | # Back up to the base HMDA_Data_Science_Kit directory 8 | dir_path = os.path.realpath(os.path.join(dir_path, "..")) 9 | # dir_path = dir_path[:-34] 10 | print(dir_path) 11 | text_to_search = dir_path 12 | replacement_text = "{data_path}" 13 | 14 | mypath = os.path.join(dir_path, "load_scripts/SQL/") 15 | print(mypath) 16 | 17 | onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))] 18 | ts_files = [mypath + f for f in onlyfiles if f[-11:-9] == "ts"] 19 | lar_files = [mypath + f for f in onlyfiles if f[-12:-9] == "lar"] 20 | panel_files = [mypath + f for f in onlyfiles if f[-14:-9] == "panel"] 21 | 22 | print("resetting TS file paths") 23 | with fileinput.FileInput(ts_files, inplace=True, backup="") as sql_file: 24 | for line in sql_file: 25 | print(line.replace(text_to_search, replacement_text), end="") 26 | 27 | print("resetting LAR file paths") 28 | with fileinput.FileInput(lar_files, inplace=True, backup="") as sql_file: 29 | for line in sql_file: 30 | print(line.replace(text_to_search, replacement_text), end="") 31 | 32 | print("resetting Panel file paths") 33 | with fileinput.FileInput(panel_files, inplace=True, backup="") as sql_file: 34 | for line in sql_file: 35 | print(line.replace(text_to_search, replacement_text), end="") 36 | -------------------------------------------------------------------------------- /load_scripts/write_data_paths.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import fileinput 3 | import os 4 | from os import listdir 5 | from os.path import isfile, join 6 | 7 | 8 | dir_path = os.path.dirname(os.path.realpath(__file__)) 9 | dir_path = dir_path[:-34] 10 | 11 | mypath = dir_path+"HMDA_Data_Science_Kit/load_scripts/SQL/" 12 | onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))] 13 | ts_files = [mypath+f for f in onlyfiles if f[-11:-9]=="ts"] 14 | lar_files = [mypath+f for f in onlyfiles if f[-12:-9]=="lar"] 15 | panel_files = [mypath+f for f in onlyfiles if f[-14:-9]=="panel"] 16 | 17 | 18 | ts_replacement_text = dir_path 19 | lar_replacement_text = dir_path 20 | panel_replacement_text = dir_path 21 | text_to_search = '{data_path}' 22 | 23 | 24 | with fileinput.FileInput(ts_files, inplace=True, backup='') as sql_file: 25 | for line in sql_file: 26 | print(line.replace(text_to_search, ts_replacement_text), end='') 27 | 28 | with fileinput.FileInput(lar_files, inplace=True, backup='') as sql_file: 29 | for line in sql_file: 30 | print(line.replace(text_to_search, lar_replacement_text), end='') 31 | 32 | with fileinput.FileInput(panel_files, inplace=True, backup='') as sql_file: 33 | for line in sql_file: 34 | print(line.replace(text_to_search, panel_replacement_text), end='') 35 | -------------------------------------------------------------------------------- /load_scripts/write_data_paths.sh: -------------------------------------------------------------------------------- 1 | #!bin/sh 2 | 3 | #get current directory path 4 | data_path=$PWD 5 | 6 | #iterate over files in load_scripts/SQL and change {data_path} to the HMDA_Data_Science_Kit base directory 7 | for filename in load_scripts/SQL/*.sql; do 8 | #get year of file for filtering files 9 | substr=${filename:${#filename} - 8} 10 | year=${substr:0:4} 11 | re='^[0-9]+$' 12 | 13 | #modify data path 14 | if [[ $year =~ $re ]] ; then 15 | echo "Modifying data path for $filename" 16 | sed -i '' "s+{data_path}+${data_path}+g" $filename 17 | fi 18 | 19 | 20 | done 21 | 22 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | appnope==0.1.0 2 | argon2-cffi==20.1.0 3 | attrs==20.1.0 4 | backcall==0.2.0 5 | bleach==3.1.5 6 | cffi==1.14.2 7 | decorator==4.4.2 8 | defusedxml==0.6.0 9 | entrypoints==0.3 10 | importlib-metadata==1.7.0 11 | ipykernel==5.3.4 12 | ipython==7.16.1 13 | ipython-genutils==0.2.0 14 | ipywidgets==7.5.1 15 | jedi==0.17.2 16 | Jinja2==2.11.2 17 | jsonschema==3.2.0 18 | jupyter==1.0.0 19 | jupyter-client==6.1.6 20 | jupyter-console==6.1.0 21 | jupyter-core==4.6.3 22 | MarkupSafe==1.1.1 23 | mistune==0.8.4 24 | nbconvert==6.5.1 25 | nbformat==5.0.7 26 | notebook==6.1.5 27 | numpy==1.19.1 28 | packaging==20.4 29 | pandas==1.1.1 30 | pandocfilters==1.4.2 31 | parso==0.7.1 32 | pexpect==4.8.0 33 | pickleshare==0.7.5 34 | prometheus-client==0.8.0 35 | prompt-toolkit==3.0.6 36 | psycopg2==2.8.5 37 | ptyprocess==0.6.0 38 | pycparser==2.20 39 | Pygments==2.6.1 40 | pyparsing==2.4.7 41 | pyrsistent==0.16.0 42 | python-dateutil==2.8.1 43 | pytz==2020.1 44 | pyzmq==19.0.2 45 | qtconsole==4.7.6 46 | QtPy==1.9.0 47 | Send2Trash==1.5.0 48 | six==1.15.0 49 | terminado==0.8.3 50 | testpath==0.4.4 51 | tornado==6.0.4 52 | traitlets==4.3.3 53 | wcwidth==0.2.5 54 | webencodings==0.5.1 55 | widgetsnbextension==3.5.1 56 | zipp==3.1.0 57 | --------------------------------------------------------------------------------