├── .gitattributes ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── pull_request_template.md ├── .gitignore ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.txt ├── README.md ├── api ├── .mvn │ └── wrapper │ │ ├── MavenWrapperDownloader.java │ │ └── maven-wrapper.properties ├── Dockerfile ├── Makefile ├── README.md ├── edit ├── mvnw ├── mvnw.cmd ├── pom.xml ├── src │ └── main │ │ ├── java │ │ └── com │ │ │ └── homeaway │ │ │ └── datapullclient │ │ │ ├── api │ │ │ └── DataPullClientApi.java │ │ │ ├── config │ │ │ ├── DataPullClientConfig.java │ │ │ ├── DataPullContext.java │ │ │ ├── DataPullContextHolder.java │ │ │ ├── DataPullProperties.java │ │ │ ├── EMRProperties.java │ │ │ ├── OktaAccessTokenFilter.java │ │ │ └── SwaggerConfig.java │ │ │ ├── data │ │ │ ├── ClusterStepCollection.java │ │ │ ├── JobStatus.java │ │ │ ├── ResponseEntity.java │ │ │ └── SimpleResponseEntity.java │ │ │ ├── exception │ │ │ ├── InputException.java │ │ │ ├── InvalidPointedJsonException.java │ │ │ └── ProcessingException.java │ │ │ ├── handlers │ │ │ └── DataPullRequestHandler.java │ │ │ ├── input │ │ │ ├── ClusterProperties.java │ │ │ ├── Destination.java │ │ │ ├── JsonInputFile.java │ │ │ ├── Migration.java │ │ │ ├── Source.java │ │ │ └── SparkOptions.java │ │ │ ├── process │ │ │ ├── DataPullRequestProcessor.java │ │ │ └── DataPullTask.java │ │ │ ├── service │ │ │ ├── DataPullClientService.java │ │ │ └── SimpleDataPullClientService.java │ │ │ └── start │ │ │ └── DatapullclientApplication.java │ │ └── resources │ │ ├── application-dev.yml │ │ ├── application-prod.yml │ │ ├── application-stage.yml │ │ ├── application-test.yml │ │ ├── input_json_schema.json │ │ ├── logback-spring.xml │ │ ├── overwrite_config.sc │ │ └── read_application_config.sc └── terraform │ ├── datapull_iam │ ├── create_user_and_roles.bat │ ├── create_user_and_roles.sh │ ├── datapull_user_and_roles.tf │ ├── uninstall_user_and_roles.sh │ └── variables.tf │ └── datapull_task │ ├── datapull_ecs.tf │ ├── ecs_deploy.sh │ ├── ecs_deploy_uninstall.sh │ └── variables.tf ├── core ├── Makefile ├── client-server.jks ├── docker_spark_server │ ├── Dockerfile │ └── LICENSE ├── mvnw ├── mvnw.cmd ├── pom.xml └── src │ ├── main │ ├── resources │ │ ├── SampleData │ │ │ ├── HelloWorld │ │ │ │ └── HelloWorld.csv │ │ │ ├── foodassociation.csv │ │ │ ├── foods.csv │ │ │ └── foodtypes.csv │ │ ├── Samples │ │ │ ├── Input_Json_Specification.json │ │ │ ├── Input_Sample_Cassandra-to-Cassandra.json │ │ │ ├── Input_Sample_CassandraWithSSL-to-S3.json │ │ │ ├── Input_Sample_CustomSQLQuery.json │ │ │ ├── Input_Sample_ElasticSearch.json │ │ │ ├── Input_Sample_ElasticSearchOverride.json │ │ │ ├── Input_Sample_ElasticSearch_Postmigrate.json │ │ │ ├── Input_Sample_ElasticSearch_Premigrate.json │ │ │ ├── Input_Sample_ElasticSearch_withquery.json │ │ │ ├── Input_Sample_Join_Hetrogeneous.json │ │ │ ├── Input_Sample_Join_Homogeneous_sources.json │ │ │ ├── Input_Sample_Kafka-to-S3.json │ │ │ ├── Input_Sample_Mongo-to_mongo-with-multiSchema-collections.json │ │ │ ├── Input_Sample_Mongo_to_Mongo.json │ │ │ ├── Input_Sample_Mongo_to_Mongo_prepost.json │ │ │ ├── Input_Sample_MySql_to_S3.json │ │ │ ├── Input_Sample_Oracle_PostPreMigration.json │ │ │ ├── Input_Sample_Oracle_to_S3.json │ │ │ ├── Input_Sample_Parallel_migrations.json │ │ │ ├── Input_Sample_Postgres_to_s3.json │ │ │ ├── Input_Sample_Postgresql.json │ │ │ ├── Input_Sample_S3_to_Mongo.json │ │ │ ├── Input_Sample_SQLServer-to-Cassandra.json │ │ │ ├── Input_Sample_SQLServer_to_Cassandra_with_windows_authentication.json │ │ │ ├── Input_Sample_Teradata-to-S3.json │ │ │ ├── Input_Sample_auroraMysql_to_S3.json │ │ │ ├── Input_Sample_aurorapostgres_to_s3.json │ │ │ ├── Input_Sample_filesystem-to-Email.json │ │ │ ├── Input_Sample_filesystem-to-filesystem-avro.json │ │ │ ├── Input_Sample_filesystem-to-filesystem-withAlerts.json │ │ │ ├── Input_Sample_filesystem-to-filesystem.json │ │ │ ├── Input_Sample_filesystem-to-mongo.json │ │ │ ├── Input_Sample_filesystem-to-sftp.json │ │ │ ├── Input_Sample_hive_to_s3.json │ │ │ ├── Input_Sample_influxdb-to-filesystem.json │ │ │ ├── Input_Sample_s3-to-hive.json │ │ │ ├── Input_Sample_s3Compaction.json │ │ │ ├── Input_Sample_s3_SFTP.json │ │ │ ├── Input_Sample_s3_to_kafka.json │ │ │ ├── Input_Sample_sftp-to-filesystem.json │ │ │ ├── Input_Sample_snowflake-to-filesystem-to-snowflake.json │ │ │ └── Input_Sample_teradata-to-filesystem.json │ │ ├── application-dev.yml │ │ ├── application-prod.yml │ │ ├── application-stage.yml │ │ └── application-test.yml │ └── scala │ │ ├── config │ │ └── AppConfig.scala │ │ ├── core │ │ ├── Controller.scala │ │ ├── DataFrameFromTo.scala │ │ ├── DataPull.scala │ │ └── Migration.scala │ │ ├── helper │ │ ├── Consul.scala │ │ └── Helper.scala │ │ ├── logging │ │ ├── DataPullLog.scala │ │ └── Logger.scala │ │ └── security │ │ ├── Alert.scala │ │ ├── SecretService.scala │ │ ├── SecretStore.scala │ │ ├── SecretsManager.scala │ │ └── Vault.scala │ └── test │ └── scala │ ├── DataFrameFromToFileSystem.scala │ └── DataFrameFromToTest.scala ├── docs ├── docs │ ├── access_secrets.md │ ├── architecture.md │ ├── aws_account_setup.md │ ├── custom_emr_ec2_role.md │ ├── custom_role_creation.md │ ├── emr_runbook.md │ ├── extra.css │ ├── faq.md │ ├── hive.md │ ├── index.md │ ├── install_on_aws.md │ ├── media │ │ ├── architecture_diagram.png │ │ ├── datapull_installation.png │ │ ├── emr_cluster.jpg │ │ ├── emr_console.jpg │ │ ├── emr_master_node.jpg │ │ ├── logo.png │ │ ├── logo_transparent.png │ │ ├── spark_ui.jpg │ │ └── yarn_manager.jpg │ ├── monitor_spark_ui.md │ ├── oracle_teradata_support.md │ ├── resources │ │ ├── sample_custom_emr_ec2_role.tf │ │ └── sample_instanceprofile.tf │ ├── sftp.md │ ├── transformation.md │ └── uninstall_on_aws.md └── mkdocs.yml ├── functional-test ├── .gitignore ├── Jenkinsfile ├── README.md ├── pom.xml ├── src │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── homeaway │ │ │ │ ├── constants │ │ │ │ ├── Environment.java │ │ │ │ └── FilePath.java │ │ │ │ ├── dto │ │ │ │ ├── Cluster.java │ │ │ │ ├── InputJson.java │ │ │ │ ├── JsonInputFile.java │ │ │ │ ├── Migration.java │ │ │ │ ├── Parquet.java │ │ │ │ ├── RDSSpecification.java │ │ │ │ ├── Response.java │ │ │ │ ├── UsersList.java │ │ │ │ └── migration │ │ │ │ │ ├── Destination.java │ │ │ │ │ ├── Node1.java │ │ │ │ │ ├── PostMigrateCommand.java │ │ │ │ │ ├── PreMigrateCommand.java │ │ │ │ │ ├── QueryS3SqlFile.java │ │ │ │ │ ├── Relation.java │ │ │ │ │ ├── Source.java │ │ │ │ │ ├── Sparkoptions.java │ │ │ │ │ └── Sql.java │ │ │ │ ├── utils │ │ │ │ ├── DataPull.java │ │ │ │ ├── DataTypeHandler.java │ │ │ │ ├── ExtentManager.java │ │ │ │ ├── GlobalVariables.java │ │ │ │ ├── ParquetReaderUtil.java │ │ │ │ ├── SQLScriptRunner.java │ │ │ │ ├── Wait.java │ │ │ │ ├── aws │ │ │ │ │ ├── AwsS3.java │ │ │ │ │ ├── DataPullApi.java │ │ │ │ │ ├── EmrCluster.java │ │ │ │ │ └── RDSInstance.java │ │ │ │ ├── db │ │ │ │ │ ├── Cassandra.java │ │ │ │ │ ├── ElasticSearch.java │ │ │ │ │ ├── GlobalRDSInstance.java │ │ │ │ │ ├── JdbcDb.java │ │ │ │ │ ├── KafkaConsumerCustom.java │ │ │ │ │ ├── KafkaProducerCustom.java │ │ │ │ │ ├── MongoDB.java │ │ │ │ │ ├── MySQL.java │ │ │ │ │ ├── Oracle.java │ │ │ │ │ ├── PostgreSQL.java │ │ │ │ │ └── SqlServer.java │ │ │ │ └── jenkins │ │ │ │ │ └── JenkinsDataPullJob.java │ │ │ │ └── validator │ │ │ │ └── Validator.java │ │ └── resources │ │ │ ├── AWSTags.json │ │ │ ├── aws_rds │ │ │ ├── MsSQLServer.properties │ │ │ ├── MySQL.properties │ │ │ └── PostgreSQL.properties │ │ │ ├── dev_global-config.json │ │ │ ├── environment.properties │ │ │ ├── input_files │ │ │ ├── Cassandra-to-MongoDB.json │ │ │ ├── Close-EMR-Cluster.json │ │ │ ├── ElasticSearch-to-SQLServer.json │ │ │ ├── ElasticSearchMappingIdAndSaveMode.json │ │ │ ├── JsonPointingToAnotherInputJson.json │ │ │ ├── Kafka-to-Postgres.json │ │ │ ├── ParallelMigrations.json │ │ │ ├── Postgres-to-ElasticSearch.json │ │ │ ├── S3-to-Cassandra-With-AccessKey.json │ │ │ ├── S3-to-Cassandra-with-post-migrate-command.json │ │ │ ├── S3-to-Cassandra.json │ │ │ ├── S3-to-s3-migration-threshold.json │ │ │ ├── SQLServer-to-S3-CSV-FileFormat.json │ │ │ ├── SQLServer-to-S3.json │ │ │ ├── parquet-to-csv-format.json │ │ │ └── tab-delimited-csv-to-parquet-format.json │ │ │ ├── log4j.properties │ │ │ ├── test_data │ │ │ ├── ElasticSearch-Data.json │ │ │ ├── KafkaData.json │ │ │ ├── MongoDB-Data.json │ │ │ ├── SQL_Files │ │ │ │ ├── mysql-destination-table-schema.sql │ │ │ │ ├── mysql-source-table-schema-and-data.sql │ │ │ │ ├── oracle-data.sql │ │ │ │ ├── oracle-schema.sql │ │ │ │ ├── postgres-destination-table-schema.sql │ │ │ │ ├── postgres-source-table-insert-update.sql │ │ │ │ ├── postgres-source-table-schema-and-data.sql │ │ │ │ ├── postgres-source-table-update.sql │ │ │ │ ├── sqlserver-destination-table-schema.sql │ │ │ │ └── sqlserver-source-table-schema-and-data.sql │ │ │ ├── cassandra_cql_files │ │ │ │ ├── cassandra_destination_table_schema.cql │ │ │ │ ├── cassandra_source_table_data.cql │ │ │ │ └── cassandra_source_table_schema.cql │ │ │ ├── s3-data.json │ │ │ ├── s3_csv_data │ │ │ │ └── users_data.csv │ │ │ ├── s3_parquet_file │ │ │ │ ├── part1.parquet │ │ │ │ └── part2.parquet │ │ │ ├── s3_partitioned_data │ │ │ │ ├── PasswordSalt=UNUSED │ │ │ │ │ └── users_data_part1.csv │ │ │ │ └── PasswordSalt=USED │ │ │ │ │ └── users_data_part2.csv │ │ │ └── s3_tab_delimited_data │ │ │ │ └── tab_delimited_data.csv │ │ │ └── test_global-config.json │ └── test │ │ └── java │ │ └── com │ │ └── homeaway │ │ ├── BaseTest.java │ │ └── DataPull │ │ ├── CronExpressionTest.java │ │ ├── DataPullCassandraToMongoDbTest.java │ │ ├── DataPullCustomJarTest.java │ │ ├── DataPullElasticSearchToSQLServerTest.java │ │ ├── DataPullKafkaToPostgresTest.java │ │ ├── DataPullMongoDBToOracleTest.java │ │ ├── DataPullOracleToMySQLTest.java │ │ ├── DataPullParallelMigrationsTest.java │ │ ├── DataPullPostgresToElasticSearchTest.java │ │ ├── DataPullS3ToCassandraTest.java │ │ ├── DataPullS3ToS3Test.java │ │ ├── DataPullSqlServerToS3Test.java │ │ ├── MandatoryFieldsTest.java │ │ └── MigrationThresholdTest.java ├── testng.xml └── updating_test_data.gif ├── manual-tests ├── README.md ├── filesystem_dataset_to_elasticsearch_to_filesystem │ ├── README.md │ ├── datapull_input.json │ └── docker_elasticsearch_server │ │ └── docker-compose.yml ├── filesystem_dataset_to_kafka │ ├── README.md │ ├── datapull_input.json │ └── docker_kafka_server │ │ ├── Dockerfile-confluenthub │ │ └── docker-compose.yml ├── filesystem_dataset_with_uuids_to_mongodb │ ├── README.md │ └── datapull_input.json ├── filesystem_stream_to_kafka │ ├── README.md │ ├── datapull_input.json │ └── docker_kafka_server └── mssql_dataset_to_mysql_to_mssql │ ├── README.md │ ├── datapull_input.json │ └── docker_rdbms_servers │ └── docker-compose.yml ├── master_application_config-dev.yml ├── master_application_config-prod.yml ├── master_application_config-stage.yml └── master_application_config-test.yml /.gitattributes: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/.gitattributes -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/.github/ISSUE_TEMPLATE/bug_report.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/.github/ISSUE_TEMPLATE/feature_request.md -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/.github/pull_request_template.md -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/.gitignore -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/CHANGELOG.md -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/LICENSE.txt -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/README.md -------------------------------------------------------------------------------- /api/.mvn/wrapper/MavenWrapperDownloader.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/.mvn/wrapper/MavenWrapperDownloader.java -------------------------------------------------------------------------------- /api/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/.mvn/wrapper/maven-wrapper.properties -------------------------------------------------------------------------------- /api/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/Dockerfile -------------------------------------------------------------------------------- /api/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/Makefile -------------------------------------------------------------------------------- /api/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/README.md -------------------------------------------------------------------------------- /api/edit: -------------------------------------------------------------------------------- 1 | export.zip 2 | -------------------------------------------------------------------------------- /api/mvnw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/mvnw -------------------------------------------------------------------------------- /api/mvnw.cmd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/mvnw.cmd -------------------------------------------------------------------------------- /api/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/pom.xml -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/api/DataPullClientApi.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/api/DataPullClientApi.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/config/DataPullClientConfig.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/config/DataPullClientConfig.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/config/DataPullContext.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/config/DataPullContext.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/config/DataPullContextHolder.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/config/DataPullContextHolder.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/config/DataPullProperties.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/config/DataPullProperties.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/config/EMRProperties.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/config/EMRProperties.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/config/OktaAccessTokenFilter.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/config/OktaAccessTokenFilter.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/config/SwaggerConfig.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/config/SwaggerConfig.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/data/ClusterStepCollection.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/data/ClusterStepCollection.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/data/JobStatus.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/data/JobStatus.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/data/ResponseEntity.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/data/ResponseEntity.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/data/SimpleResponseEntity.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/data/SimpleResponseEntity.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/exception/InputException.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/exception/InputException.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/exception/InvalidPointedJsonException.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/exception/InvalidPointedJsonException.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/exception/ProcessingException.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/exception/ProcessingException.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/handlers/DataPullRequestHandler.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/handlers/DataPullRequestHandler.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/input/ClusterProperties.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/input/ClusterProperties.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/input/Destination.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/input/Destination.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/input/JsonInputFile.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/input/JsonInputFile.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/input/Migration.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/input/Migration.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/input/Source.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/input/Source.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/input/SparkOptions.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/input/SparkOptions.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/process/DataPullRequestProcessor.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/process/DataPullRequestProcessor.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/process/DataPullTask.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/process/DataPullTask.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/service/DataPullClientService.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/service/DataPullClientService.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/service/SimpleDataPullClientService.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/service/SimpleDataPullClientService.java -------------------------------------------------------------------------------- /api/src/main/java/com/homeaway/datapullclient/start/DatapullclientApplication.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/java/com/homeaway/datapullclient/start/DatapullclientApplication.java -------------------------------------------------------------------------------- /api/src/main/resources/application-dev.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/resources/application-dev.yml -------------------------------------------------------------------------------- /api/src/main/resources/application-prod.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/resources/application-prod.yml -------------------------------------------------------------------------------- /api/src/main/resources/application-stage.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/resources/application-stage.yml -------------------------------------------------------------------------------- /api/src/main/resources/application-test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/resources/application-test.yml -------------------------------------------------------------------------------- /api/src/main/resources/input_json_schema.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/resources/input_json_schema.json -------------------------------------------------------------------------------- /api/src/main/resources/logback-spring.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/resources/logback-spring.xml -------------------------------------------------------------------------------- /api/src/main/resources/overwrite_config.sc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/resources/overwrite_config.sc -------------------------------------------------------------------------------- /api/src/main/resources/read_application_config.sc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/src/main/resources/read_application_config.sc -------------------------------------------------------------------------------- /api/terraform/datapull_iam/create_user_and_roles.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/terraform/datapull_iam/create_user_and_roles.bat -------------------------------------------------------------------------------- /api/terraform/datapull_iam/create_user_and_roles.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/terraform/datapull_iam/create_user_and_roles.sh -------------------------------------------------------------------------------- /api/terraform/datapull_iam/datapull_user_and_roles.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/terraform/datapull_iam/datapull_user_and_roles.tf -------------------------------------------------------------------------------- /api/terraform/datapull_iam/uninstall_user_and_roles.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/terraform/datapull_iam/uninstall_user_and_roles.sh -------------------------------------------------------------------------------- /api/terraform/datapull_iam/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/terraform/datapull_iam/variables.tf -------------------------------------------------------------------------------- /api/terraform/datapull_task/datapull_ecs.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/terraform/datapull_task/datapull_ecs.tf -------------------------------------------------------------------------------- /api/terraform/datapull_task/ecs_deploy.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/terraform/datapull_task/ecs_deploy.sh -------------------------------------------------------------------------------- /api/terraform/datapull_task/ecs_deploy_uninstall.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/terraform/datapull_task/ecs_deploy_uninstall.sh -------------------------------------------------------------------------------- /api/terraform/datapull_task/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/api/terraform/datapull_task/variables.tf -------------------------------------------------------------------------------- /core/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/Makefile -------------------------------------------------------------------------------- /core/client-server.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/client-server.jks -------------------------------------------------------------------------------- /core/docker_spark_server/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/docker_spark_server/Dockerfile -------------------------------------------------------------------------------- /core/docker_spark_server/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/docker_spark_server/LICENSE -------------------------------------------------------------------------------- /core/mvnw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/mvnw -------------------------------------------------------------------------------- /core/mvnw.cmd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/mvnw.cmd -------------------------------------------------------------------------------- /core/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/pom.xml -------------------------------------------------------------------------------- /core/src/main/resources/SampleData/HelloWorld/HelloWorld.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/SampleData/HelloWorld/HelloWorld.csv -------------------------------------------------------------------------------- /core/src/main/resources/SampleData/foodassociation.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/SampleData/foodassociation.csv -------------------------------------------------------------------------------- /core/src/main/resources/SampleData/foods.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/SampleData/foods.csv -------------------------------------------------------------------------------- /core/src/main/resources/SampleData/foodtypes.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/SampleData/foodtypes.csv -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Json_Specification.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Json_Specification.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_Cassandra-to-Cassandra.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_Cassandra-to-Cassandra.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_CassandraWithSSL-to-S3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_CassandraWithSSL-to-S3.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_CustomSQLQuery.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_CustomSQLQuery.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_ElasticSearch.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_ElasticSearch.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_ElasticSearchOverride.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_ElasticSearchOverride.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_ElasticSearch_Postmigrate.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_ElasticSearch_Postmigrate.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_ElasticSearch_Premigrate.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_ElasticSearch_Premigrate.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_ElasticSearch_withquery.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_ElasticSearch_withquery.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_Join_Hetrogeneous.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_Join_Hetrogeneous.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_Join_Homogeneous_sources.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_Join_Homogeneous_sources.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_Kafka-to-S3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_Kafka-to-S3.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_Mongo-to_mongo-with-multiSchema-collections.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_Mongo-to_mongo-with-multiSchema-collections.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_Mongo_to_Mongo.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_Mongo_to_Mongo.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_Mongo_to_Mongo_prepost.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_Mongo_to_Mongo_prepost.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_MySql_to_S3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_MySql_to_S3.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_Oracle_PostPreMigration.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_Oracle_PostPreMigration.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_Oracle_to_S3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_Oracle_to_S3.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_Parallel_migrations.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_Parallel_migrations.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_Postgres_to_s3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_Postgres_to_s3.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_Postgresql.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_Postgresql.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_S3_to_Mongo.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_S3_to_Mongo.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_SQLServer-to-Cassandra.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_SQLServer-to-Cassandra.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_SQLServer_to_Cassandra_with_windows_authentication.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_SQLServer_to_Cassandra_with_windows_authentication.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_Teradata-to-S3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_Teradata-to-S3.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_auroraMysql_to_S3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_auroraMysql_to_S3.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_aurorapostgres_to_s3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_aurorapostgres_to_s3.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_filesystem-to-Email.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_filesystem-to-Email.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_filesystem-to-filesystem-avro.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_filesystem-to-filesystem-avro.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_filesystem-to-filesystem-withAlerts.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_filesystem-to-filesystem-withAlerts.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_filesystem-to-filesystem.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_filesystem-to-filesystem.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_filesystem-to-mongo.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_filesystem-to-mongo.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_filesystem-to-sftp.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_filesystem-to-sftp.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_hive_to_s3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_hive_to_s3.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_influxdb-to-filesystem.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_influxdb-to-filesystem.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_s3-to-hive.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_s3-to-hive.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_s3Compaction.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_s3Compaction.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_s3_SFTP.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_s3_SFTP.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_s3_to_kafka.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_s3_to_kafka.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_sftp-to-filesystem.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_sftp-to-filesystem.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_snowflake-to-filesystem-to-snowflake.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_snowflake-to-filesystem-to-snowflake.json -------------------------------------------------------------------------------- /core/src/main/resources/Samples/Input_Sample_teradata-to-filesystem.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/Samples/Input_Sample_teradata-to-filesystem.json -------------------------------------------------------------------------------- /core/src/main/resources/application-dev.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/application-dev.yml -------------------------------------------------------------------------------- /core/src/main/resources/application-prod.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/application-prod.yml -------------------------------------------------------------------------------- /core/src/main/resources/application-stage.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/application-stage.yml -------------------------------------------------------------------------------- /core/src/main/resources/application-test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/resources/application-test.yml -------------------------------------------------------------------------------- /core/src/main/scala/config/AppConfig.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/scala/config/AppConfig.scala -------------------------------------------------------------------------------- /core/src/main/scala/core/Controller.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/scala/core/Controller.scala -------------------------------------------------------------------------------- /core/src/main/scala/core/DataFrameFromTo.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/scala/core/DataFrameFromTo.scala -------------------------------------------------------------------------------- /core/src/main/scala/core/DataPull.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/scala/core/DataPull.scala -------------------------------------------------------------------------------- /core/src/main/scala/core/Migration.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/scala/core/Migration.scala -------------------------------------------------------------------------------- /core/src/main/scala/helper/Consul.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/scala/helper/Consul.scala -------------------------------------------------------------------------------- /core/src/main/scala/helper/Helper.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/scala/helper/Helper.scala -------------------------------------------------------------------------------- /core/src/main/scala/logging/DataPullLog.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/scala/logging/DataPullLog.scala -------------------------------------------------------------------------------- /core/src/main/scala/logging/Logger.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/scala/logging/Logger.scala -------------------------------------------------------------------------------- /core/src/main/scala/security/Alert.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/scala/security/Alert.scala -------------------------------------------------------------------------------- /core/src/main/scala/security/SecretService.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/scala/security/SecretService.scala -------------------------------------------------------------------------------- /core/src/main/scala/security/SecretStore.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/scala/security/SecretStore.scala -------------------------------------------------------------------------------- /core/src/main/scala/security/SecretsManager.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/scala/security/SecretsManager.scala -------------------------------------------------------------------------------- /core/src/main/scala/security/Vault.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/main/scala/security/Vault.scala -------------------------------------------------------------------------------- /core/src/test/scala/DataFrameFromToFileSystem.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/test/scala/DataFrameFromToFileSystem.scala -------------------------------------------------------------------------------- /core/src/test/scala/DataFrameFromToTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/core/src/test/scala/DataFrameFromToTest.scala -------------------------------------------------------------------------------- /docs/docs/access_secrets.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/access_secrets.md -------------------------------------------------------------------------------- /docs/docs/architecture.md: -------------------------------------------------------------------------------- 1 | ![Screenshot](media/architecture_diagram.png) -------------------------------------------------------------------------------- /docs/docs/aws_account_setup.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/aws_account_setup.md -------------------------------------------------------------------------------- /docs/docs/custom_emr_ec2_role.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/custom_emr_ec2_role.md -------------------------------------------------------------------------------- /docs/docs/custom_role_creation.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/custom_role_creation.md -------------------------------------------------------------------------------- /docs/docs/emr_runbook.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/emr_runbook.md -------------------------------------------------------------------------------- /docs/docs/extra.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/extra.css -------------------------------------------------------------------------------- /docs/docs/faq.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/faq.md -------------------------------------------------------------------------------- /docs/docs/hive.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/hive.md -------------------------------------------------------------------------------- /docs/docs/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/index.md -------------------------------------------------------------------------------- /docs/docs/install_on_aws.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/install_on_aws.md -------------------------------------------------------------------------------- /docs/docs/media/architecture_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/media/architecture_diagram.png -------------------------------------------------------------------------------- /docs/docs/media/datapull_installation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/media/datapull_installation.png -------------------------------------------------------------------------------- /docs/docs/media/emr_cluster.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/media/emr_cluster.jpg -------------------------------------------------------------------------------- /docs/docs/media/emr_console.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/media/emr_console.jpg -------------------------------------------------------------------------------- /docs/docs/media/emr_master_node.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/media/emr_master_node.jpg -------------------------------------------------------------------------------- /docs/docs/media/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/media/logo.png -------------------------------------------------------------------------------- /docs/docs/media/logo_transparent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/media/logo_transparent.png -------------------------------------------------------------------------------- /docs/docs/media/spark_ui.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/media/spark_ui.jpg -------------------------------------------------------------------------------- /docs/docs/media/yarn_manager.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/media/yarn_manager.jpg -------------------------------------------------------------------------------- /docs/docs/monitor_spark_ui.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/monitor_spark_ui.md -------------------------------------------------------------------------------- /docs/docs/oracle_teradata_support.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/oracle_teradata_support.md -------------------------------------------------------------------------------- /docs/docs/resources/sample_custom_emr_ec2_role.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/resources/sample_custom_emr_ec2_role.tf -------------------------------------------------------------------------------- /docs/docs/resources/sample_instanceprofile.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/resources/sample_instanceprofile.tf -------------------------------------------------------------------------------- /docs/docs/sftp.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/sftp.md -------------------------------------------------------------------------------- /docs/docs/transformation.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/transformation.md -------------------------------------------------------------------------------- /docs/docs/uninstall_on_aws.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/docs/uninstall_on_aws.md -------------------------------------------------------------------------------- /docs/mkdocs.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/docs/mkdocs.yml -------------------------------------------------------------------------------- /functional-test/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/.gitignore -------------------------------------------------------------------------------- /functional-test/Jenkinsfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/Jenkinsfile -------------------------------------------------------------------------------- /functional-test/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/README.md -------------------------------------------------------------------------------- /functional-test/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/pom.xml -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/constants/Environment.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/constants/Environment.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/constants/FilePath.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/constants/FilePath.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/dto/Cluster.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/dto/Cluster.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/dto/InputJson.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/dto/InputJson.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/dto/JsonInputFile.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/dto/JsonInputFile.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/dto/Migration.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/dto/Migration.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/dto/Parquet.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/dto/Parquet.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/dto/RDSSpecification.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/dto/RDSSpecification.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/dto/Response.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/dto/Response.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/dto/UsersList.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/dto/UsersList.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/dto/migration/Destination.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/dto/migration/Destination.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/dto/migration/Node1.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/dto/migration/Node1.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/dto/migration/PostMigrateCommand.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/dto/migration/PostMigrateCommand.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/dto/migration/PreMigrateCommand.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/dto/migration/PreMigrateCommand.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/dto/migration/QueryS3SqlFile.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/dto/migration/QueryS3SqlFile.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/dto/migration/Relation.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/dto/migration/Relation.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/dto/migration/Source.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/dto/migration/Source.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/dto/migration/Sparkoptions.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/dto/migration/Sparkoptions.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/dto/migration/Sql.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/dto/migration/Sql.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/DataPull.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/DataPull.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/DataTypeHandler.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/DataTypeHandler.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/ExtentManager.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/ExtentManager.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/GlobalVariables.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/GlobalVariables.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/ParquetReaderUtil.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/ParquetReaderUtil.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/SQLScriptRunner.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/SQLScriptRunner.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/Wait.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/Wait.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/aws/AwsS3.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/aws/AwsS3.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/aws/DataPullApi.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/aws/DataPullApi.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/aws/EmrCluster.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/aws/EmrCluster.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/aws/RDSInstance.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/aws/RDSInstance.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/db/Cassandra.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/db/Cassandra.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/db/ElasticSearch.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/db/ElasticSearch.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/db/GlobalRDSInstance.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/db/GlobalRDSInstance.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/db/JdbcDb.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/db/JdbcDb.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/db/KafkaConsumerCustom.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/db/KafkaConsumerCustom.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/db/KafkaProducerCustom.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/db/KafkaProducerCustom.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/db/MongoDB.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/db/MongoDB.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/db/MySQL.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/db/MySQL.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/db/Oracle.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/db/Oracle.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/db/PostgreSQL.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/db/PostgreSQL.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/db/SqlServer.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/db/SqlServer.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/utils/jenkins/JenkinsDataPullJob.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/utils/jenkins/JenkinsDataPullJob.java -------------------------------------------------------------------------------- /functional-test/src/main/java/com/homeaway/validator/Validator.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/java/com/homeaway/validator/Validator.java -------------------------------------------------------------------------------- /functional-test/src/main/resources/AWSTags.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/AWSTags.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/aws_rds/MsSQLServer.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/aws_rds/MsSQLServer.properties -------------------------------------------------------------------------------- /functional-test/src/main/resources/aws_rds/MySQL.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/aws_rds/MySQL.properties -------------------------------------------------------------------------------- /functional-test/src/main/resources/aws_rds/PostgreSQL.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/aws_rds/PostgreSQL.properties -------------------------------------------------------------------------------- /functional-test/src/main/resources/dev_global-config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/dev_global-config.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/environment.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/environment.properties -------------------------------------------------------------------------------- /functional-test/src/main/resources/input_files/Cassandra-to-MongoDB.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/input_files/Cassandra-to-MongoDB.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/input_files/Close-EMR-Cluster.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/input_files/Close-EMR-Cluster.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/input_files/ElasticSearch-to-SQLServer.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/input_files/ElasticSearch-to-SQLServer.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/input_files/ElasticSearchMappingIdAndSaveMode.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/input_files/ElasticSearchMappingIdAndSaveMode.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/input_files/JsonPointingToAnotherInputJson.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/input_files/JsonPointingToAnotherInputJson.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/input_files/Kafka-to-Postgres.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/input_files/Kafka-to-Postgres.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/input_files/ParallelMigrations.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/input_files/ParallelMigrations.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/input_files/Postgres-to-ElasticSearch.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/input_files/Postgres-to-ElasticSearch.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/input_files/S3-to-Cassandra-With-AccessKey.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/input_files/S3-to-Cassandra-With-AccessKey.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/input_files/S3-to-Cassandra-with-post-migrate-command.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/input_files/S3-to-Cassandra-with-post-migrate-command.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/input_files/S3-to-Cassandra.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/input_files/S3-to-Cassandra.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/input_files/S3-to-s3-migration-threshold.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/input_files/S3-to-s3-migration-threshold.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/input_files/SQLServer-to-S3-CSV-FileFormat.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/input_files/SQLServer-to-S3-CSV-FileFormat.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/input_files/SQLServer-to-S3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/input_files/SQLServer-to-S3.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/input_files/parquet-to-csv-format.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/input_files/parquet-to-csv-format.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/input_files/tab-delimited-csv-to-parquet-format.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/input_files/tab-delimited-csv-to-parquet-format.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/log4j.properties -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/ElasticSearch-Data.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/ElasticSearch-Data.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/KafkaData.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/KafkaData.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/MongoDB-Data.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/MongoDB-Data.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/SQL_Files/mysql-destination-table-schema.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/SQL_Files/mysql-destination-table-schema.sql -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/SQL_Files/mysql-source-table-schema-and-data.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/SQL_Files/mysql-source-table-schema-and-data.sql -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/SQL_Files/oracle-data.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/SQL_Files/oracle-data.sql -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/SQL_Files/oracle-schema.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/SQL_Files/oracle-schema.sql -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/SQL_Files/postgres-destination-table-schema.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/SQL_Files/postgres-destination-table-schema.sql -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/SQL_Files/postgres-source-table-insert-update.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/SQL_Files/postgres-source-table-insert-update.sql -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/SQL_Files/postgres-source-table-schema-and-data.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/SQL_Files/postgres-source-table-schema-and-data.sql -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/SQL_Files/postgres-source-table-update.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/SQL_Files/postgres-source-table-update.sql -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/SQL_Files/sqlserver-destination-table-schema.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/SQL_Files/sqlserver-destination-table-schema.sql -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/SQL_Files/sqlserver-source-table-schema-and-data.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/SQL_Files/sqlserver-source-table-schema-and-data.sql -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/cassandra_cql_files/cassandra_destination_table_schema.cql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/cassandra_cql_files/cassandra_destination_table_schema.cql -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/cassandra_cql_files/cassandra_source_table_data.cql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/cassandra_cql_files/cassandra_source_table_data.cql -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/cassandra_cql_files/cassandra_source_table_schema.cql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/cassandra_cql_files/cassandra_source_table_schema.cql -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/s3-data.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/s3-data.json -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/s3_csv_data/users_data.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/s3_csv_data/users_data.csv -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/s3_parquet_file/part1.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/s3_parquet_file/part1.parquet -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/s3_parquet_file/part2.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/s3_parquet_file/part2.parquet -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/s3_partitioned_data/PasswordSalt=UNUSED/users_data_part1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/s3_partitioned_data/PasswordSalt=UNUSED/users_data_part1.csv -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/s3_partitioned_data/PasswordSalt=USED/users_data_part2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/s3_partitioned_data/PasswordSalt=USED/users_data_part2.csv -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_data/s3_tab_delimited_data/tab_delimited_data.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_data/s3_tab_delimited_data/tab_delimited_data.csv -------------------------------------------------------------------------------- /functional-test/src/main/resources/test_global-config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/main/resources/test_global-config.json -------------------------------------------------------------------------------- /functional-test/src/test/java/com/homeaway/BaseTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/test/java/com/homeaway/BaseTest.java -------------------------------------------------------------------------------- /functional-test/src/test/java/com/homeaway/DataPull/CronExpressionTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/test/java/com/homeaway/DataPull/CronExpressionTest.java -------------------------------------------------------------------------------- /functional-test/src/test/java/com/homeaway/DataPull/DataPullCassandraToMongoDbTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/test/java/com/homeaway/DataPull/DataPullCassandraToMongoDbTest.java -------------------------------------------------------------------------------- /functional-test/src/test/java/com/homeaway/DataPull/DataPullCustomJarTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/test/java/com/homeaway/DataPull/DataPullCustomJarTest.java -------------------------------------------------------------------------------- /functional-test/src/test/java/com/homeaway/DataPull/DataPullElasticSearchToSQLServerTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/test/java/com/homeaway/DataPull/DataPullElasticSearchToSQLServerTest.java -------------------------------------------------------------------------------- /functional-test/src/test/java/com/homeaway/DataPull/DataPullKafkaToPostgresTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/test/java/com/homeaway/DataPull/DataPullKafkaToPostgresTest.java -------------------------------------------------------------------------------- /functional-test/src/test/java/com/homeaway/DataPull/DataPullMongoDBToOracleTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/test/java/com/homeaway/DataPull/DataPullMongoDBToOracleTest.java -------------------------------------------------------------------------------- /functional-test/src/test/java/com/homeaway/DataPull/DataPullOracleToMySQLTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/test/java/com/homeaway/DataPull/DataPullOracleToMySQLTest.java -------------------------------------------------------------------------------- /functional-test/src/test/java/com/homeaway/DataPull/DataPullParallelMigrationsTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/test/java/com/homeaway/DataPull/DataPullParallelMigrationsTest.java -------------------------------------------------------------------------------- /functional-test/src/test/java/com/homeaway/DataPull/DataPullPostgresToElasticSearchTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/test/java/com/homeaway/DataPull/DataPullPostgresToElasticSearchTest.java -------------------------------------------------------------------------------- /functional-test/src/test/java/com/homeaway/DataPull/DataPullS3ToCassandraTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/test/java/com/homeaway/DataPull/DataPullS3ToCassandraTest.java -------------------------------------------------------------------------------- /functional-test/src/test/java/com/homeaway/DataPull/DataPullS3ToS3Test.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/test/java/com/homeaway/DataPull/DataPullS3ToS3Test.java -------------------------------------------------------------------------------- /functional-test/src/test/java/com/homeaway/DataPull/DataPullSqlServerToS3Test.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/test/java/com/homeaway/DataPull/DataPullSqlServerToS3Test.java -------------------------------------------------------------------------------- /functional-test/src/test/java/com/homeaway/DataPull/MandatoryFieldsTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/test/java/com/homeaway/DataPull/MandatoryFieldsTest.java -------------------------------------------------------------------------------- /functional-test/src/test/java/com/homeaway/DataPull/MigrationThresholdTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/src/test/java/com/homeaway/DataPull/MigrationThresholdTest.java -------------------------------------------------------------------------------- /functional-test/testng.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/testng.xml -------------------------------------------------------------------------------- /functional-test/updating_test_data.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/functional-test/updating_test_data.gif -------------------------------------------------------------------------------- /manual-tests/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/manual-tests/README.md -------------------------------------------------------------------------------- /manual-tests/filesystem_dataset_to_elasticsearch_to_filesystem/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/manual-tests/filesystem_dataset_to_elasticsearch_to_filesystem/README.md -------------------------------------------------------------------------------- /manual-tests/filesystem_dataset_to_elasticsearch_to_filesystem/datapull_input.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/manual-tests/filesystem_dataset_to_elasticsearch_to_filesystem/datapull_input.json -------------------------------------------------------------------------------- /manual-tests/filesystem_dataset_to_elasticsearch_to_filesystem/docker_elasticsearch_server/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/manual-tests/filesystem_dataset_to_elasticsearch_to_filesystem/docker_elasticsearch_server/docker-compose.yml -------------------------------------------------------------------------------- /manual-tests/filesystem_dataset_to_kafka/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/manual-tests/filesystem_dataset_to_kafka/README.md -------------------------------------------------------------------------------- /manual-tests/filesystem_dataset_to_kafka/datapull_input.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/manual-tests/filesystem_dataset_to_kafka/datapull_input.json -------------------------------------------------------------------------------- /manual-tests/filesystem_dataset_to_kafka/docker_kafka_server/Dockerfile-confluenthub: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/manual-tests/filesystem_dataset_to_kafka/docker_kafka_server/Dockerfile-confluenthub -------------------------------------------------------------------------------- /manual-tests/filesystem_dataset_to_kafka/docker_kafka_server/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/manual-tests/filesystem_dataset_to_kafka/docker_kafka_server/docker-compose.yml -------------------------------------------------------------------------------- /manual-tests/filesystem_dataset_with_uuids_to_mongodb/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/manual-tests/filesystem_dataset_with_uuids_to_mongodb/README.md -------------------------------------------------------------------------------- /manual-tests/filesystem_dataset_with_uuids_to_mongodb/datapull_input.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/manual-tests/filesystem_dataset_with_uuids_to_mongodb/datapull_input.json -------------------------------------------------------------------------------- /manual-tests/filesystem_stream_to_kafka/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/manual-tests/filesystem_stream_to_kafka/README.md -------------------------------------------------------------------------------- /manual-tests/filesystem_stream_to_kafka/datapull_input.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/manual-tests/filesystem_stream_to_kafka/datapull_input.json -------------------------------------------------------------------------------- /manual-tests/filesystem_stream_to_kafka/docker_kafka_server: -------------------------------------------------------------------------------- 1 | ../filesystem_dataset_to_kafka/docker_kafka_server -------------------------------------------------------------------------------- /manual-tests/mssql_dataset_to_mysql_to_mssql/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/manual-tests/mssql_dataset_to_mysql_to_mssql/README.md -------------------------------------------------------------------------------- /manual-tests/mssql_dataset_to_mysql_to_mssql/datapull_input.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/manual-tests/mssql_dataset_to_mysql_to_mssql/datapull_input.json -------------------------------------------------------------------------------- /manual-tests/mssql_dataset_to_mysql_to_mssql/docker_rdbms_servers/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/manual-tests/mssql_dataset_to_mysql_to_mssql/docker_rdbms_servers/docker-compose.yml -------------------------------------------------------------------------------- /master_application_config-dev.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/master_application_config-dev.yml -------------------------------------------------------------------------------- /master_application_config-prod.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/master_application_config-prod.yml -------------------------------------------------------------------------------- /master_application_config-stage.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/master_application_config-stage.yml -------------------------------------------------------------------------------- /master_application_config-test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/homeaway/datapull/HEAD/master_application_config-test.yml --------------------------------------------------------------------------------