├── .github ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── PULL_REQUEST_TEMPLATE │ ├── bug_fix.md │ └── feature.md ├── .gitignore ├── .travis.yml ├── Backend ├── README.md ├── __init__.py ├── common-resource-files │ ├── role.yml │ └── serverless.yml ├── docker │ └── serverless-deploy │ │ ├── Dockerfile │ │ └── src │ │ ├── post_harden.sh │ │ └── requirements.txt ├── emr-autoscaling │ ├── README.md │ ├── __init__.py │ ├── serverless.yml │ ├── serverless │ │ ├── cfn │ │ │ └── resource.yml │ │ ├── lambdas │ │ │ ├── add_autoscaling.yml │ │ │ ├── delete_autoscaling.yml │ │ │ └── list_autoscaling.yml │ │ └── models │ │ │ ├── EMRAutoScalingRequest.json │ │ │ ├── EMRAutoScalingResponse.json │ │ │ └── models.yml │ ├── src │ │ ├── __init__.py │ │ ├── conf │ │ │ └── autoscaling-config.json │ │ ├── scripts │ │ │ ├── __init__.py │ │ │ ├── emr_add_autoscaling.py │ │ │ ├── emr_delete_autoscaling.py │ │ │ └── emr_list_autoscaling.py │ │ └── util │ │ │ ├── __init__.py │ │ │ ├── commlib.py │ │ │ ├── emrlib.py │ │ │ ├── exceptions.py │ │ │ └── log.py │ └── tests │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── pytest.ini │ │ └── test_emr_asg_add.py ├── emr-cluster-ops │ ├── README.md │ ├── __init__.py │ ├── ba_step_scripts │ │ ├── install-dr-elephant.sh │ │ ├── install-gradle-bootstrap.sh │ │ ├── install-nodejs.sh │ │ └── syslog-setup.sh │ ├── serverless.yml │ ├── serverless │ │ ├── cfn │ │ │ └── resource.yml │ │ ├── lambdas │ │ │ ├── cost_usage.yml │ │ │ ├── create.yml │ │ │ ├── custom_add_steps.yml │ │ │ ├── dns_ops.yml │ │ │ ├── precheck.yml │ │ │ ├── protect.yml │ │ │ ├── rm_proxy.yml │ │ │ ├── status.yml │ │ │ ├── steps.yml │ │ │ ├── terminate.yml │ │ │ ├── validate.yml │ │ │ └── validate_steps.yml │ │ └── models │ │ │ ├── mapping_templates │ │ │ ├── EMRClusterAddCustomStepRequest │ │ │ ├── EMRClusterAddCustomStepResponse │ │ │ ├── EMRStepValidateRequest │ │ │ └── EMRStepValidateResponse │ │ │ ├── models.yml │ │ │ └── schemas │ │ │ ├── 500ErrorResponse.json │ │ │ ├── CreateEMRClusterRequest.json │ │ │ ├── CreateEMRClusterResponse.json │ │ │ ├── EMRClusterAddCustomStepRequest.json │ │ │ ├── EMRClusterAddCustomStepResponse.json │ │ │ ├── EMRClusterAddStepRequest.json │ │ │ ├── EMRClusterAddStepResponse.json │ │ │ ├── EMRDNSOperation.json │ │ │ ├── EMRRMProxyRequest.json │ │ │ ├── EMRRMProxyResponse.json │ │ │ ├── EMRStepValidateRequest.json │ │ │ ├── EMRStepValidateResponse.json │ │ │ ├── ProtectEMRClusterRequest.json │ │ │ ├── ProtectEMRClusterResponse.json │ │ │ ├── TerminateEMRClusterRequest.json │ │ │ ├── TerminateEMRClusterResponse.json │ │ │ ├── ValidateEMRClusterRequest.json │ │ │ └── ValidateEMRClusterResponse.json │ ├── src │ │ ├── __init__.py │ │ ├── conf │ │ │ ├── common │ │ │ │ ├── autoscaling-config.json │ │ │ │ ├── common-tags.json │ │ │ │ ├── configurations.json │ │ │ │ └── configurations_r5.json │ │ │ └── example_account │ │ │ │ ├── emr-metadata.json │ │ │ │ └── emr-nonkerb-config.json │ │ ├── scripts │ │ │ ├── __init__.py │ │ │ ├── emr_add_custom_steps.py │ │ │ ├── emr_add_steps.py │ │ │ ├── emr_cost_usage.py │ │ │ ├── emr_create.py │ │ │ ├── emr_create_nonkerb.py │ │ │ ├── emr_dns_ops.py │ │ │ ├── emr_precheck.py │ │ │ ├── emr_protect.py │ │ │ ├── emr_rm_proxy.py │ │ │ ├── emr_status.py │ │ │ ├── emr_terminate.py │ │ │ ├── emr_validate.py │ │ │ └── emr_validate_step.py │ │ └── util │ │ │ ├── __init__.py │ │ │ ├── commlib.py │ │ │ ├── constants.py │ │ │ ├── dnslib.py │ │ │ ├── emrlib.py │ │ │ ├── exceptions.py │ │ │ ├── log.py │ │ │ └── rmlib.py │ └── tests │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── pytest.ini │ │ ├── test_add_step.py │ │ ├── test_custom_step.py │ │ ├── test_emr_creation.py │ │ ├── test_emr_precheck.py │ │ ├── test_emr_protect.py │ │ ├── test_emr_status.py │ │ ├── test_emr_terminate.py │ │ ├── test_emr_validate.py │ │ └── test_validate_step.py ├── emr-testsuites │ ├── README.md │ ├── serverless.yml │ ├── serverless │ │ ├── cfn │ │ │ └── resource.yml │ │ ├── lambdas │ │ │ ├── test_autoscaling.yml │ │ │ ├── test_bootstrap.yml │ │ │ ├── test_emr_app.yml │ │ │ └── test_steps.yml │ │ └── models │ │ │ ├── EMRClusterTestAppRequest.json │ │ │ ├── EMRClusterTestAppResponse.json │ │ │ ├── EMRTestSuiteRequest.json │ │ │ ├── EMRTestSuiteResponse.json │ │ │ └── models.yml │ ├── src │ │ ├── __init__.py │ │ ├── scripts │ │ │ ├── __init__.py │ │ │ ├── emr_test_app.py │ │ │ ├── emr_test_autoscaling.py │ │ │ ├── emr_test_bootstrap.py │ │ │ └── emr_test_steps.py │ │ └── util │ │ │ ├── __init__.py │ │ │ ├── constants.py │ │ │ ├── emrlib.py │ │ │ ├── exceptions.py │ │ │ └── log.py │ └── tests │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── pytest.ini │ │ ├── test_emr_app.py │ │ ├── test_emr_asg.py │ │ ├── test_emr_ba.py │ │ └── test_emr_steps.py ├── pytest.ini ├── run_test.sh ├── serverless-deploy │ ├── README.md │ ├── logger.py │ ├── plugin_install.py │ ├── plugin_metadata.json │ └── quickfabric_setup.py ├── setup.cfg └── setup.py ├── DB ├── .gitignore ├── Dockerfile ├── QuickFabric_ER_Diagram.png ├── QuickFabric_ER_model.mwb ├── dcl.sql ├── ddl │ ├── account_configurations.sql │ ├── application_configurations.sql │ ├── aws_account_profile.sql │ ├── cluster_metrics.sql │ ├── cluster_metrics_history.sql │ ├── cluster_step_request.sql │ ├── configuration_data_types.sql │ ├── configuration_definitions.sql │ ├── configuration_types.sql │ ├── db_patch.sql │ ├── emr_billing_component_cost.sql │ ├── emr_billing_component_cost_hist.sql │ ├── emr_cluster_metadata.sql │ ├── emr_functional_testsuites.sql │ ├── emr_functional_testsuites_status.sql │ ├── emr_functional_testsuites_status_history.sql │ ├── report_subscriptions.sql │ ├── reports.sql │ ├── roles.sql │ ├── segments.sql │ ├── services.sql │ ├── user.sql │ ├── user_account_segment_mapping.sql │ ├── user_account_segment_role_mapping.sql │ └── workflow.sql ├── dml │ ├── application_configurations.sql │ ├── aws_account_profile.sql │ ├── cluster_metrics.sql │ ├── cluster_metrics_history.sql │ ├── configuration_data_types.sql │ ├── configuration_definitions.sql │ ├── configuration_types.sql │ ├── emr_billing_component_cost.sql │ ├── emr_billing_component_cost_hist.sql │ ├── emr_cluster_metadata.sql │ ├── reports.sql │ ├── roles.sql │ ├── segments.sql │ ├── services.sql │ ├── user.sql │ ├── user_account_segment_mapping.sql │ ├── user_account_segment_role_mapping.sql │ └── workflow.sql ├── patch │ ├── patch.sh │ ├── v1.sql │ └── v2.sql └── quickfabric_bootstrap.sql ├── Frontend ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── babel.config.js ├── build.sh ├── default.conf ├── nginx.conf ├── package.json ├── public │ ├── Hive.png │ ├── Vertica.png │ ├── favicon.ico │ ├── index.html │ └── manifest.json └── src │ ├── __mocks__ │ ├── fileMock.js │ └── styleMock.js │ ├── __tests__ │ ├── components │ │ ├── admin │ │ │ ├── __snapshots__ │ │ │ │ └── admin.test.jsx.snap │ │ │ └── admin.test.jsx │ │ ├── emrcost │ │ │ ├── EMRCost.test.jsx │ │ │ └── __snapshots__ │ │ │ │ └── EMRCost.test.jsx.snap │ │ ├── emrmanagement │ │ │ ├── CreateCluster.test.jsx │ │ │ ├── EMRManagement.test.jsx │ │ │ └── __snapshots__ │ │ │ │ ├── CreateCluster.test.jsx.snap │ │ │ │ └── EMRManagement.test.jsx.snap │ │ └── utils │ │ │ └── components │ │ │ ├── ToggleSlider.test.jsx │ │ │ └── __snapshots__ │ │ │ └── ToggleSlider.test.jsx.snap │ ├── containers │ │ ├── App.test.jsx │ │ └── __snapshots__ │ │ │ └── App.test.jsx.snap │ ├── reducers │ │ ├── Admin.test.js │ │ ├── EMRCost.test.js │ │ ├── Profile.test.js │ │ └── User.test.js │ └── setup │ │ └── setupEnzyme.js │ ├── actions │ ├── actionTypes │ │ ├── AdminActionTypes.js │ │ ├── EMRCostActionTypes.js │ │ ├── EMRHealthActionTypes.js │ │ ├── EMRManagementActionTypes.js │ │ ├── ProfileActionTypes.js │ │ └── UserActionTypes.js │ ├── admin.js │ ├── emrCost.js │ ├── emrHealth.js │ ├── emrManagement.js │ ├── profile.js │ └── user.js │ ├── api-config.js │ ├── assets │ ├── help │ │ ├── AccountConfig.png │ │ ├── AccountSetup.gif │ │ ├── Admin.png │ │ ├── AppConfig.png │ │ ├── ClusterCost.png │ │ ├── Cluster_Details_1.png │ │ ├── Cluster_Details_2.png │ │ ├── Cluster_Operations.gif │ │ ├── CreateClusterWorkflow.png │ │ ├── Create_Cluster.gif │ │ ├── EMRCost.png │ │ ├── EMR_Graph.png │ │ ├── Expert_Advice.png │ │ ├── QuickFabric_ER_Diagram.png │ │ ├── UserProfile.png │ │ ├── ViewWorkflow.png │ │ ├── addSteps.gif │ │ ├── architecture.png │ │ ├── autoPilot.gif │ │ ├── cloneCluster.gif │ │ ├── cluster_details.gif │ │ ├── dnsFlip.gif │ │ ├── drussotto.jpg │ │ ├── gbagdi.jpg │ │ ├── gdoon.jpg │ │ ├── ktran7.jpg │ │ ├── kverma.jpg │ │ ├── nkk.jpg │ │ ├── rotateAMI.gif │ │ ├── suttamchandani.jpg │ │ ├── terminateCluster.gif │ │ └── vsood.jpg │ ├── login │ │ ├── data.jpg │ │ ├── gitLogo.png │ │ └── qf_black.svg │ └── navbar │ │ ├── qdlogo.png │ │ └── qf_white.svg │ ├── components │ ├── Admin │ │ ├── ConfigCustomCell.jsx │ │ ├── ConfigDialogContainer.jsx │ │ ├── ConfigManagement.jsx │ │ ├── admin.scss │ │ ├── detailGrids.js │ │ └── index.jsx │ ├── EMRCost │ │ ├── emrCost.scss │ │ └── index.jsx │ ├── EMRHealth │ │ ├── ToggleText.jsx │ │ ├── emrHealth.css │ │ ├── emrHealth1.scss │ │ ├── emrHealthCell.js │ │ ├── index.jsx │ │ └── observabilityGraph │ │ │ ├── Legends.jsx │ │ │ ├── ObservabilityGraph.jsx │ │ │ ├── RowData.jsx │ │ │ ├── VisOptions.js │ │ │ └── toggle.scss │ ├── EMRManagement │ │ ├── AMIRotationDays.jsx │ │ ├── AddSteps.jsx │ │ ├── CreateCluster.jsx │ │ ├── CreateClusterConstants.js │ │ ├── CreateClusterWorkflow.jsx │ │ ├── CustomAlert.jsx │ │ ├── DNSFlip.jsx │ │ ├── Modal │ │ │ ├── ModalCreateClusterDetail.jsx │ │ │ └── ModalStepsStatusDetail.jsx │ │ ├── RotateAMI.jsx │ │ ├── RotateAMIWorkflow.jsx │ │ ├── SubmitActionButtonCell.jsx │ │ ├── TerminateCluster.jsx │ │ ├── TestSuite.jsx │ │ ├── emrManagement.css │ │ ├── emrStatusCell.jsx │ │ ├── emrStepStatusCell.jsx │ │ └── index.jsx │ ├── Help │ │ ├── Help.scss │ │ └── index.jsx │ ├── NavBar │ │ ├── index.jsx │ │ └── navBar.scss │ ├── Profile │ │ ├── index.jsx │ │ └── profile.scss │ ├── SignIn │ │ ├── SignIn.css │ │ └── index.jsx │ └── index.js │ ├── containers │ ├── AdminContainer.jsx │ ├── AppContainer.jsx │ ├── EMRCostContainer.jsx │ ├── EMRHealthContainer.jsx │ ├── EMRManagementContainer.jsx │ ├── HelpContainer.jsx │ ├── ProfileContainer.jsx │ └── SignInContainer.jsx │ ├── id_rsa-cert.pub │ ├── index.js │ ├── main.css │ ├── reducers │ ├── admin.js │ ├── emrcost.js │ ├── emrhealth.js │ ├── emrmanagement.js │ ├── index.js │ ├── profile.js │ └── user.js │ ├── store │ └── index.js │ └── utils │ ├── components │ ├── CheckJiraEnabled.js │ ├── MyCommandCell.jsx │ ├── ToggleSlider.jsx │ └── with-state.jsx │ ├── styles │ ├── shared.css │ └── toggle.scss │ └── utils │ ├── action.js │ └── convertStrToBoolean.js ├── LICENSE ├── Middleware ├── .gitignore ├── README.md ├── commons │ ├── .gitignore │ ├── .project │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── intuit │ │ └── quickfabric │ │ └── commons │ │ ├── constants │ │ ├── ApiUrls.java │ │ ├── ApplicationConstant.java │ │ ├── HealthCheckConstants.java │ │ ├── Roles.java │ │ └── WorkflowConstants.java │ │ ├── dao │ │ ├── ConfigDao.java │ │ ├── ConfigDaoImpl.java │ │ ├── LoginRolesDao.java │ │ └── LoginRolesDaoImpl.java │ │ ├── domain │ │ ├── QuickFabricAuthenticationToken.java │ │ ├── Role.java │ │ └── UserAccess.java │ │ ├── exceptions │ │ ├── QuickFabricBaseException.java │ │ ├── QuickFabricClientException.java │ │ ├── QuickFabricJsonException.java │ │ ├── QuickFabricRestHandlerException.java │ │ ├── QuickFabricSQLException.java │ │ ├── QuickFabricServerException.java │ │ ├── QuickFabricUnauthenticatedException.java │ │ ├── QuickFabricUnauthorizedException.java │ │ └── ServiceException.java │ │ ├── helper │ │ ├── ConfigHelper.java │ │ └── LoginRolesHelper.java │ │ ├── mapper │ │ ├── AccountConfigMapper.java │ │ ├── ApplicationConfigMapper.java │ │ ├── ConfigDefinitionMapper.java │ │ ├── LoginRolesMapper.java │ │ └── UserAccessMapper.java │ │ ├── model │ │ └── LoginRolesModel.java │ │ ├── security │ │ ├── AccessControl.java │ │ ├── JWTTokenProvider.java │ │ ├── JwtAuthenticationSuccessHandler.java │ │ ├── JwtTokenFilter.java │ │ ├── JwtTokenFilterConfigurer.java │ │ ├── RestAuthenticationEntryPoint.java │ │ └── WebSecurityConfig.java │ │ ├── service │ │ ├── ConfigService.java │ │ └── LoginService.java │ │ ├── utils │ │ ├── AWSEmailUtil.java │ │ ├── ApiKeyRequestInterceptor.java │ │ ├── AppConfig.java │ │ ├── AuthorizationUtils.java │ │ ├── CommonUtils.java │ │ ├── EnableMethod.java │ │ ├── EnableMethodAspect.java │ │ ├── LoggingRequestInterceptor.java │ │ ├── ReportBuilder.java │ │ ├── RestTemplateResponseErrorHandler.java │ │ └── SwaggerConfig.java │ │ └── vo │ │ ├── AccountSetupModel.java │ │ ├── ApiErrorVO.java │ │ ├── AppSeverity.java │ │ ├── AppStatus.java │ │ ├── AutoScalingTestResponse.java │ │ ├── AwsAccountProfile.java │ │ ├── BootstrapActionVO.java │ │ ├── BootstrapParametersRequest.java │ │ ├── BootstrapTestResponse.java │ │ ├── ChangeRoleModel.java │ │ ├── ChangeRoleVO.java │ │ ├── ClusterError.java │ │ ├── ClusterHealthCheckRequest.java │ │ ├── ClusterHealthCheckStatusType.java │ │ ├── ClusterHealthCheckStatusUpdate.java │ │ ├── ClusterHealthStatus.java │ │ ├── ClusterMessageResponseVO.java │ │ ├── ClusterRequest.java │ │ ├── ClusterStatus.java │ │ ├── ClusterStep.java │ │ ├── ClusterStepRequest.java │ │ ├── ClusterTestSuitesDefinitionVO.java │ │ ├── ClusterType.java │ │ ├── ClusterVO.java │ │ ├── ConfigDataType.java │ │ ├── ConfigDefinitionVO.java │ │ ├── ConfigType.java │ │ ├── ConfigVO.java │ │ ├── ConnectivityTestRequest.java │ │ ├── ConnectivityTestResponse.java │ │ ├── EMRAppVO.java │ │ ├── EMRClusterHealthTestCase.java │ │ ├── EMRClusterMetricsVO.java │ │ ├── EMRGroupCostVO.java │ │ ├── EMRTimeSeriesReportVO.java │ │ ├── HadoopJarStep.java │ │ ├── JobPerformanceAdviceVO.java │ │ ├── JobSchedulingAdviceVO.java │ │ ├── LoginRequest.java │ │ ├── LoginRolesVO.java │ │ ├── MonthlyCostVO.java │ │ ├── ResetPasswordRequest.java │ │ ├── SSODetailsVO.java │ │ ├── SegmentVO.java │ │ ├── ServiceType.java │ │ ├── ServiceVO.java │ │ ├── StepResponseVO.java │ │ ├── StepStatus.java │ │ ├── StepsValidateRequest.java │ │ ├── SubscriptionVO.java │ │ ├── UserAccountSegmentMapping.java │ │ ├── UserRole.java │ │ ├── UserVO.java │ │ ├── Workflow.java │ │ ├── WorkflowStatus.java │ │ ├── WorkflowStep.java │ │ ├── WorkflowType.java │ │ └── YarnAppHeuristicVO.java ├── emr │ ├── .project │ ├── Dockerfile │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── intuit │ │ │ │ └── quickfabric │ │ │ │ └── emr │ │ │ │ ├── EMRMainApplication.java │ │ │ │ ├── dao │ │ │ │ ├── AdminDao.java │ │ │ │ ├── AdminDaoImpl.java │ │ │ │ ├── EMRClusterCostDao.java │ │ │ │ ├── EMRClusterCostDaoImpl.java │ │ │ │ ├── EMRClusterHealthCheckDao.java │ │ │ │ ├── EMRClusterHealthCheckDaoImpl.java │ │ │ │ ├── EMRClusterMetadataDao.java │ │ │ │ ├── EMRClusterMetadataDaoImpl.java │ │ │ │ ├── EMRClusterMetricsDao.java │ │ │ │ ├── EMRClusterMetricsDaoImpl.java │ │ │ │ ├── EMRClusterStepsDao.java │ │ │ │ ├── EMRClusterStepsDaoImpl.java │ │ │ │ ├── EmailReportSubscriptionsDao.java │ │ │ │ ├── EmailReportSubscriptionsDaoImpl.java │ │ │ │ ├── WorkflowDao.java │ │ │ │ └── WorkflowDaoImpl.java │ │ │ │ ├── helper │ │ │ │ ├── AdminHelper.java │ │ │ │ ├── EMRAWSServiceCallerHelper.java │ │ │ │ ├── EMRClusterAdviceHelper.java │ │ │ │ ├── EMRClusterCostHelper.java │ │ │ │ ├── EMRClusterHealthHelper.java │ │ │ │ ├── EMRClusterManagementHelper.java │ │ │ │ ├── EMRClusterMetadataHelper.java │ │ │ │ ├── EMRClusterMetricsHelper.java │ │ │ │ ├── EmailReportSubscriptionsHelper.java │ │ │ │ ├── TicketValidationHelper.java │ │ │ │ ├── ValidationHelper.java │ │ │ │ └── WorkflowHelper.java │ │ │ │ ├── mapper │ │ │ │ ├── AMIRotationClusterMetadataMapper.java │ │ │ │ ├── AWSAccountProfileMapper.java │ │ │ │ ├── BootstrapActionMapper.java │ │ │ │ ├── ClusterStepMapper.java │ │ │ │ ├── ClusterStepResponseMapper.java │ │ │ │ ├── EMRClusterHealthHistoryMapper.java │ │ │ │ ├── EMRClusterHealthStatusMapper.java │ │ │ │ ├── EMRClusterMetadataMapper.java │ │ │ │ ├── EMRClusterMetadataRowMapper.java │ │ │ │ ├── EMRClusterMetricsAggregateMapper.java │ │ │ │ ├── EMRClusterMetricsMapper.java │ │ │ │ ├── EMRClusterMetricsMultipleDaysHourlyMapper.java │ │ │ │ ├── EMRClusterMetricsRmUrlMapper.java │ │ │ │ ├── EMRClusterMetricsTimeSeriesMapper.java │ │ │ │ ├── EMRClusterValidationMapper.java │ │ │ │ ├── EMRCostTimeSeriesMapper.java │ │ │ │ ├── SegmentsMapper.java │ │ │ │ ├── TestSuiteDefinitionMapper.java │ │ │ │ ├── UserAccountSegmentMappingMapper.java │ │ │ │ └── WorkflowMapper.java │ │ │ │ ├── model │ │ │ │ ├── EMRAppsModel.java │ │ │ │ ├── EMRClusterAdviceModel.java │ │ │ │ ├── EMRClusterCostModel.java │ │ │ │ ├── EMRClusterMetadataModel.java │ │ │ │ ├── EMRClusterMetricsModel.java │ │ │ │ └── UIPopulationModel.java │ │ │ │ └── service │ │ │ │ ├── AdminService.java │ │ │ │ ├── EMRClusterAdviceService.java │ │ │ │ ├── EMRClusterCostService.java │ │ │ │ ├── EMRClusterHealthCheckService.java │ │ │ │ ├── EMRClusterManagementService.java │ │ │ │ ├── EMRClusterMetadataService.java │ │ │ │ ├── EMRClusterMetricsService.java │ │ │ │ ├── EmailReportSubscriptionService.java │ │ │ │ └── WorkflowService.java │ │ └── resources │ │ │ ├── application.properties │ │ │ └── log4j2.xml │ │ └── test │ │ └── java │ │ ├── com │ │ └── intuit │ │ │ └── quickfabric │ │ │ └── emr │ │ │ └── tests │ │ │ ├── helper │ │ │ └── EMRClusterHealthHelperTests.java │ │ │ └── service │ │ │ └── EMRClusterHealthCheckServiceTests.java │ │ └── resources │ │ └── application.properties ├── pom.xml └── schedulers │ ├── .gitignore │ ├── .project │ ├── Dockerfile │ ├── pom.xml │ └── src │ └── main │ ├── java │ └── com │ │ └── intuit │ │ └── quickfabric │ │ └── schedulers │ │ ├── SchedulersMainApplication.java │ │ ├── dao │ │ ├── ClusterHealthCheckDao.java │ │ ├── ClusterHealthCheckDaoImpl.java │ │ ├── EMRClusterMetadataDao.java │ │ ├── EMRClusterMetadataDaoImpl.java │ │ ├── EMRClusterMetricsDao.java │ │ ├── EMRClusterMetricsDaoImpl.java │ │ ├── EMRClusterStepsDao.java │ │ └── EMRClusterStepsDaoImpl.java │ │ ├── functions │ │ ├── ClusterHealthCheckScheduler.java │ │ ├── EMRClusterLifeCycleSchedulers.java │ │ └── EMRClusterMetricsSchedulers.java │ │ ├── helpers │ │ ├── AutoScalingHealthCheckHelper.java │ │ ├── BootstrapHealthCheckHelper.java │ │ ├── ConnectivityHealthCheckHelper.java │ │ ├── EMRAWSServiceCallerHelper.java │ │ ├── EMRClusterManagementHelper.java │ │ ├── EMRClusterMetadataHelper.java │ │ ├── EMRClusterMetricsHelper.java │ │ ├── HealthChecksHelper.java │ │ └── ValidationHelper.java │ │ └── mappers │ │ ├── AMIRotationClusterMetadataMapper.java │ │ ├── AddUserRequestMapper.java │ │ ├── BootstrapActionMapper.java │ │ ├── ClusterStepMapper.java │ │ ├── EMRClusterDetailsMapper.java │ │ ├── EMRClusterFetchMetricsMapper.java │ │ ├── EMRClusterHealthStatusMapper.java │ │ ├── EMRClusterMetadataRowMapper.java │ │ ├── EMRClusterMetricsAMIReportMapper.java │ │ ├── EMRClusterMetricsHourlyReportMapper.java │ │ ├── EMRClusterMetricsMapper.java │ │ ├── EMRClusterValidationMapper.java │ │ └── TestSuiteDefinitionMapper.java │ └── resources │ ├── application.properties │ └── log4j2.xml ├── README.md ├── docker-compose-rdsversion.yml ├── docker-compose.yml ├── terraform ├── .gitignore ├── LICENSE ├── input.tfvars.advanced.example ├── input.tfvars.basic.example ├── main.tf ├── main_qf.tf ├── modules │ ├── container │ │ ├── main.tf │ │ └── variables.tf │ ├── ec2 │ │ ├── files │ │ │ ├── instance_policy.txt │ │ │ └── instance_role.txt │ │ ├── main.tf │ │ ├── output.tf │ │ └── variables.tf │ ├── igw │ │ ├── main.tf │ │ └── variables.tf │ ├── nat │ │ ├── main.tf │ │ └── variables.tf │ ├── r53 │ │ ├── main.tf │ │ ├── output.tf │ │ └── variables.tf │ ├── s3 │ │ ├── main.tf │ │ ├── output.tf │ │ └── variables.tf │ ├── s3_uploads │ │ ├── main.tf │ │ └── variables.tf │ ├── sg │ │ ├── get_ip.sh │ │ ├── main.tf │ │ ├── output.tf │ │ └── variables.tf │ ├── subnet │ │ ├── main.tf │ │ ├── output.tf │ │ └── variables.tf │ └── vpc │ │ ├── main.tf │ │ ├── output.tf │ │ └── variables.tf ├── modules_backend │ └── s3_backend │ │ ├── main.tf │ │ └── variables.tf ├── output.tf ├── provider.tf ├── scripts │ └── env.py ├── secrets.tfvars.example ├── templates │ ├── qf │ │ └── userdata.tpl │ └── serverless │ │ ├── emr-config.json │ │ └── output.conf.tpl ├── terraform_wrapper.py └── variables.tf └── terraform_local ├── LICENSE ├── README.md ├── main.tf ├── modules └── container │ ├── main.tf │ └── variables.tf ├── output.tf ├── provider.tf ├── secrets.tfvars.example └── variables.tf /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # To add a code owner, give a file pattern followed by the usernames or emails of the owner. 2 | * @khilawar4 3 | /DB/ @khilawar4 @gdrossi46 @drussotto @soodvarun 4 | /Backend/ @gdrossi46 @sandipnahak 5 | /Middleware/ @khilawar4 @gdrossi46 @drussotto @soodvarun 6 | /Frontend/ @nishakk 7 | /terraform/ @khilawar4 @sanoojm 8 | /terraform_local/ @khilawar4 @sanoojm 9 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | 5 | --- 6 | 7 | **Describe the bug** 8 | A clear and concise description of what the bug is. 9 | 10 | **To Reproduce** 11 | Steps to reproduce the behavior. For example: 12 | 1. Launch using command '...' 13 | 2. Using this configuration '...' (i.e. your application.conf) 14 | 3. Send a request like '...' 15 | 4. See error 16 | 17 | **Expected behavior** 18 | A clear and concise description of what you expected to happen. 19 | 20 | **Screenshots** 21 | If applicable, add screenshots to help explain your problem. 22 | 23 | **Desktop (please complete the following information):** 24 | - OS: [e.g. MacOS] 25 | - Browser [e.g. chrome, safari] 26 | - Version [e.g. 22] 27 | 28 | **Additional context** 29 | Add any other context about the problem here. 30 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | 5 | --- 6 | 7 | **Is your feature request related to a problem? Please describe.** 8 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 9 | 10 | **Describe the solution you'd like** 11 | A clear and concise description of what you want to happen. 12 | 13 | **Describe alternatives you've considered** 14 | A clear and concise description of any alternative solutions or features you've considered. 15 | 16 | **Additional context** 17 | Add any other context or screenshots about the feature request here. 18 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/bug_fix.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug fix 3 | about: Fix an error or unexpected behavior 4 | 5 | --- 6 | 7 | **Describe the bug** 8 | Tell us what behavior you're fixing. If there's an open issue 9 | for it, feel free to link it here. 10 | 11 | **Example output before and after** 12 | Show us what was going wrong before, and what it does now that it's fixed. 13 | This might be a snippet from a console or a screenshot. 14 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/feature.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature 3 | about: Implement a new feature for the project 4 | 5 | --- 6 | 7 | **Feature description or context** 8 | What problems does this feature solve? Feel free to link issues or other 9 | discussions about the feature. 10 | 11 | **How does this feature impact users?** 12 | Tell us any changes to the user interface this might have. This might 13 | include end-users (e.g. a UI feature), or it might impact programmatic 14 | clients or API consumers. 15 | 16 | **Code highlights or explanations** 17 | If there are any pieces of code you'd like to draw our attention to, 18 | point us to them and tell us about how they work. 19 | 20 | **New tests or modified tests** 21 | Tell us exactly how your code behaves by providing (or updating) some 22 | tests. 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.classpath 2 | *.lock.info 3 | *.tfstate 4 | *.tfstate.* 5 | *__pycache_* 6 | .DS_Store 7 | .idea 8 | .idea/ 9 | .idea/** 10 | .idea/*.xml 11 | .idea/*configs.zip 12 | .settings/ 13 | *.coverage* 14 | *id_rsa* 15 | Middleware/commons/bin/ 16 | Middleware/emr/bin/ 17 | Middleware/schedulers/bin/ 18 | 19 | crash.log 20 | *package-lock.json* 21 | *cfn-stack-output.json* 22 | input.tfvars 23 | *dist/* 24 | *.pytest_cache* 25 | *.egg* 26 | 27 | .terraform 28 | terraform.tfstate 29 | terraform.tfstate.backup 30 | terraform.tfstate.d 31 | terraform_wrapper_* 32 | secrets.tfvars 33 | 34 | Backend/emr-autoscaling/node_modules 35 | Backend/serverless-deploy/config.yml 36 | Backend/emr-cluster-ops/.serverless/ 37 | Backend/emr-testsuites/.serverless/ 38 | Backend/emr-autoscaling/.serverless/ 39 | Backend/emr-cluster-ops/.pytest_cache 40 | /Backend/tests/build/ 41 | /Backend/tests/dist/ 42 | /Backend/tests/quickfabric_test.egg-info/ 43 | -------------------------------------------------------------------------------- /Backend/README.md: -------------------------------------------------------------------------------- 1 | # quickfabric -------------------------------------------------------------------------------- /Backend/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Backend/__init__.py -------------------------------------------------------------------------------- /Backend/docker/serverless-deploy/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:lts-alpine 2 | 3 | LABEL Description="Serverless Docker image for deploying quickfabric" 4 | LABEL MAINTAINER="opensource@intuit.com" 5 | 6 | COPY src ./ 7 | ENV SERVERLESS_VERSION=1.60.4 8 | RUN apk upgrade --no-cache && apk add python3 9 | 10 | RUN pip3 install --upgrade pip && \ 11 | pip3 install -r requirements.txt 12 | 13 | # Skip serverless version upgrade check https://github.com/serverless/serverless/issues/4319 14 | RUN npm install -g try-thread-sleep && \ 15 | npm install --global serverless@${SERVERLESS_VERSION} --ignore-scripts spawn-sync 16 | 17 | RUN chmod +x post_harden.sh && ./post_harden.sh 18 | 19 | ENTRYPOINT ["/bin/sh", "-c"] 20 | -------------------------------------------------------------------------------- /Backend/docker/serverless-deploy/src/requirements.txt: -------------------------------------------------------------------------------- 1 | awscli 2 | boto3 3 | pyyaml 4 | coloredlogs -------------------------------------------------------------------------------- /Backend/emr-autoscaling/README.md: -------------------------------------------------------------------------------- 1 | # emr-autoscaling 2 | 3 | | Plugin | README | 4 | | ------ | ------ | 5 | | Dropbox | [plugins/dropbox/README.md][PlDb] | 6 | | GitHub | [plugins/github/README.md][PlGh] | 7 | | Google Drive | [plugins/googledrive/README.md][PlGd] | 8 | | OneDrive | [plugins/onedrive/README.md][PlOd] | 9 | | Medium | [plugins/medium/README.md][PlMe] | 10 | | Google Analytics | [plugins/googleanalytics/README.md][PlGa] | -------------------------------------------------------------------------------- /Backend/emr-autoscaling/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Backend/emr-autoscaling/__init__.py -------------------------------------------------------------------------------- /Backend/emr-autoscaling/serverless.yml: -------------------------------------------------------------------------------- 1 | service: emr-autoscaling-ops 2 | 3 | provider: 4 | name: aws 5 | apiGateway: 6 | restApiId: 7 | 'Fn::ImportValue': RestApiId 8 | restApiRootResourceId: 9 | 'Fn::ImportValue': RootResourceId 10 | deploymentBucket: 11 | name: quickfabric-serverless-${self:custom.config.account-id}-${self:provider.region}-deploys 12 | deploymentPrefix: ${self:service.name} # The S3 prefix under which deployed artifacts should be stored. Default is serverless 13 | 14 | custom: 15 | parent: 16 | path: "../common-resource-files/serverless.yml" 17 | 18 | RequestParameterValidator: 19 | 'Fn::ImportValue': EmrRequestValidatorParams 20 | 21 | RequestBodyValidator: 22 | 'Fn::ImportValue': EmrRequestValidatorBody 23 | 24 | documentation: ${file(serverless/models/models.yml)} 25 | 26 | functions: 27 | emr-add-autoscaling: ${file(serverless/lambdas/add_autoscaling.yml)} 28 | emr-list-autoscaling: ${file(serverless/lambdas/list_autoscaling.yml)} 29 | emr-delete-autoscaling: ${file(serverless/lambdas/delete_autoscaling.yml)} 30 | 31 | plugins: 32 | - serverless-deployment-bucket 33 | - serverless-pseudo-parameters 34 | - serverless-add-api-key 35 | - serverless-aws-documentation 36 | - serverless-reqvalidator-plugin 37 | - serverless-jetpack 38 | - serverless-plugin-parent -------------------------------------------------------------------------------- /Backend/emr-autoscaling/serverless/cfn/resource.yml: -------------------------------------------------------------------------------- 1 | Resources: 2 | ApiGatewayStage: 3 | Type: AWS::ApiGateway::Stage 4 | Properties: 5 | MethodSettings: 6 | - DataTraceEnabled: true 7 | HttpMethod: "*" 8 | LoggingLevel: INFO 9 | ResourcePath: "/*" 10 | MetricsEnabled: true 11 | RestApiId: ${self:provider.apiGateway.restApiId} 12 | EmrRequestValidatorParams: 13 | Type: "AWS::ApiGateway::RequestValidator" 14 | Properties: 15 | Name: 'emr-req-validator-params' 16 | RestApiId: ${self:provider.apiGateway.restApiId} 17 | ValidateRequestBody: false 18 | ValidateRequestParameters: true 19 | EmrRequestValidatorBody: 20 | Type: "AWS::ApiGateway::RequestValidator" 21 | Properties: 22 | Name: 'emr-req-validator-body' 23 | RestApiId: ${self:provider.apiGateway.restApiId} 24 | ValidateRequestBody: true 25 | ValidateRequestParameters: true 26 | -------------------------------------------------------------------------------- /Backend/emr-autoscaling/serverless/models/EMRAutoScalingRequest.json: -------------------------------------------------------------------------------- 1 | { 2 | "required": [ 3 | "clusterId" 4 | ], 5 | "properties": { 6 | "clusterId": { 7 | "type": "string" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Backend/emr-autoscaling/serverless/models/EMRAutoScalingResponse.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "status": { 5 | "type": "string" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /Backend/emr-autoscaling/serverless/models/models.yml: -------------------------------------------------------------------------------- 1 | api: 2 | info: 3 | version: '1.0.0' 4 | title: EMR cluster Ops APIs 5 | description: APIs to perform EMR autoscaling operations 6 | models: 7 | - name: EMRAutoScalingRequest 8 | contentType: "application/json" 9 | schema: ${file(serverless/models/EMRAutoScalingRequest.json)} 10 | - name: EMRAutoScalingResponse 11 | contentType: "application/json" 12 | schema: ${file(serverless/models/EMRAutoScalingResponse.json)} -------------------------------------------------------------------------------- /Backend/emr-autoscaling/src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Backend/emr-autoscaling/src/__init__.py -------------------------------------------------------------------------------- /Backend/emr-autoscaling/src/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Backend/emr-autoscaling/src/scripts/__init__.py -------------------------------------------------------------------------------- /Backend/emr-autoscaling/src/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Backend/emr-autoscaling/src/util/__init__.py -------------------------------------------------------------------------------- /Backend/emr-autoscaling/src/util/commlib.py: -------------------------------------------------------------------------------- 1 | 2 | def construct_error_response(context, api_request_id): 3 | """ 4 | Construct error response dict 5 | :param context: AWS Context object, containing properties about the invocation, function, and execution environment. 6 | :param api_request_id: 7 | :return: dict, a dict object containing information about the aws loggroup name, stream name and lambda request id 8 | :rtype: 9 | """ 10 | 11 | error_response = { 12 | "statusCode": 500, 13 | "lambda_function_name": context.function_name, 14 | "log_group_name": context.log_group_name, 15 | "log_stream_name": context.log_stream_name, 16 | "api_request_id": api_request_id, 17 | "lambda_request_id": context.aws_request_id 18 | } 19 | 20 | return error_response 21 | -------------------------------------------------------------------------------- /Backend/emr-autoscaling/src/util/log.py: -------------------------------------------------------------------------------- 1 | """ 2 | Base script for logging Lambda functions 3 | """ 4 | 5 | import logging 6 | import sys 7 | 8 | 9 | def setup_logging(api_request_id, lambda_request_id): 10 | logger = logging.getLogger() 11 | for handler in logger.handlers: 12 | logger.removeHandler(handler) 13 | 14 | handler = logging.StreamHandler(sys.stdout) 15 | 16 | log_format = f"[%(asctime)s] [api_request_id={api_request_id} lambda_request_id={lambda_request_id}] %(levelname)s: %(message)s" 17 | handler.setFormatter(logging.Formatter(log_format, "%Y-%m-%d %H:%M:%S")) 18 | logger.addHandler(handler) 19 | logger.setLevel(logging.INFO) 20 | 21 | return logger 22 | -------------------------------------------------------------------------------- /Backend/emr-autoscaling/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Backend/emr-autoscaling/tests/__init__.py -------------------------------------------------------------------------------- /Backend/emr-autoscaling/tests/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = -p no:warnings 3 | norecursedirs = build docs/_build *.egg .tox *.venv *__init__* *site-packages* 4 | -------------------------------------------------------------------------------- /Backend/emr-autoscaling/tests/test_emr_asg_add.py: -------------------------------------------------------------------------------- 1 | from src.scripts.emr_add_autoscaling import lambda_handler as asg_add 2 | from pprint import pprint 3 | 4 | 5 | def test_emr_asg_add(create_cluster, lambda_context): 6 | cluster_id = create_cluster.get('JobFlowId') 7 | assert 'j-' in cluster_id 8 | 9 | asg_add_request = { 10 | "api_request_id": "test_asg_add", 11 | "cluster_id": cluster_id, 12 | "autoscaling_profile": "Default", 13 | "instance_group": "CORE", 14 | "min": "1", 15 | "max": "3" 16 | } 17 | try: 18 | response = asg_add(asg_add_request, lambda_context) 19 | except Exception as error: 20 | assert 'NotImplementedError' not in error.args 21 | pass 22 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Backend/emr-cluster-ops/__init__.py -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/ba_step_scripts/install-nodejs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | is_aml=`uname -r | grep amzn1.x86_64 | wc -l` 4 | 5 | if [ ${is_aml} == 1 ]; then 6 | # install node through yum 7 | sudo yum -y install nodejs npm --enablerepo=epel 8 | 9 | # install nvm and update to Node v 0.12 10 | curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.29.0/install.sh | bash 11 | . ~/.nvm/nvm.sh 12 | nvm install 0.12 13 | nvm alias default 0.12 14 | else 15 | echo "Unsupported OS" 16 | fi -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/mapping_templates/EMRClusterAddCustomStepRequest: -------------------------------------------------------------------------------- 1 | #set($inputRoot = $input.path('$')) 2 | { 3 | "api_request_id": "$context.requestId", 4 | "cluster_name": "$inputRoot.clusterName", 5 | "cluster_id": "$inputRoot.clusterId", 6 | "steps": [ 7 | #foreach($elem in $inputRoot.steps) 8 | { 9 | "Name": "$elem.name", 10 | "ActionOnFailure": "$elem.actionOnFailure", 11 | "HadoopJarStep": { 12 | "MainClass": "$elem.hadoopJarStep.mainClass", 13 | "Jar": "$elem.hadoopJarStep.jar", 14 | "Args": $elem.hadoopJarStep.stepArgs 15 | } 16 | }#if($foreach.hasNext), #end 17 | #end 18 | ] 19 | } -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/mapping_templates/EMRClusterAddCustomStepResponse: -------------------------------------------------------------------------------- 1 | #set($inputRoot = $input.path('$')) 2 | { 3 | "statusCode": 201, 4 | "clusterName": "$input.path('$.cluster_name')", 5 | "clusterId": "$input.path('$.cluster_id')", 6 | "steps": [ 7 | #foreach($elem in $inputRoot.steps) 8 | { 9 | "name": "$elem.step_name", 10 | "stepId": "$elem.step_id" 11 | }#if($foreach.hasNext), #end 12 | #end 13 | ], 14 | "apiRequestId": "$input.path('$.api_request_id')", 15 | "lambdaRequestId": "$input.path('$.lambda_request_id')" 16 | } 17 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/mapping_templates/EMRStepValidateRequest: -------------------------------------------------------------------------------- 1 | #set($inputRoot = $input.path('$')) 2 | { 3 | "api_request_id": "$context.requestId", 4 | "cluster_name": "$input.path('clusterName')", 5 | "cluster_id": "$input.path('$.clusterId')", 6 | "step_ids": [ 7 | #foreach($elem in $inputRoot.stepIds) 8 | "$elem"#if($foreach.hasNext), #end 9 | #end 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/mapping_templates/EMRStepValidateResponse: -------------------------------------------------------------------------------- 1 | #set($inputRoot = $input.path('$')) 2 | { 3 | "statusCode": 201, 4 | "clusterName": "$input.path('$.cluster_name')", 5 | "clusterId": "$input.path('$.cluster_id')", 6 | "steps": [ 7 | #foreach($elem in $inputRoot.steps) 8 | { 9 | "name": "$elem.step_name", 10 | "stepId": "$elem.step_id", 11 | "status": "$elem.status" 12 | }#if($foreach.hasNext), #end 13 | #end 14 | ], 15 | "apiRequestId": "$input.path('$.api_request_id')", 16 | "lambdaRequestId": "$input.path('$.lambda_request_id')" 17 | } 18 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/schemas/500ErrorResponse.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "errorType": { 5 | "type": "string" 6 | }, 7 | "statusCode": { 8 | "type": "number" 9 | }, 10 | "errorMessage": { 11 | "type": "string" 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/schemas/CreateEMRClusterRequest.json: -------------------------------------------------------------------------------- 1 | { 2 | "required": [ 3 | "subType", 4 | "segment", 5 | "account" 6 | ], 7 | "properties": { 8 | "segment": { 9 | "type": "string" 10 | }, 11 | "subType": { 12 | "type": "string" 13 | }, 14 | "account": { 15 | "type": "string" 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/schemas/CreateEMRClusterResponse.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "clusterName": { 5 | "type": "string" 6 | }, 7 | "subType": { 8 | "type": "string" 9 | }, 10 | "role": { 11 | "type": "string" 12 | }, 13 | "account": { 14 | "type": "string" 15 | }, 16 | "status": { 17 | "type": "string" 18 | }, 19 | "message": { 20 | "type": "string" 21 | }, 22 | "statusCode": { 23 | "type": "number" 24 | }, 25 | "dnsName": { 26 | "type": "string" 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/schemas/EMRClusterAddCustomStepRequest.json: -------------------------------------------------------------------------------- 1 | { 2 | "required": [ 3 | "clusterName", 4 | "clusterId", 5 | "steps" 6 | ], 7 | "properties": { 8 | "clusterName": { 9 | "type": "string" 10 | }, 11 | "clusterId": { 12 | "type": "string" 13 | }, 14 | "steps": { 15 | "type": "array", 16 | "items": { 17 | "type": "object", 18 | "properties": { 19 | "name": { 20 | "type": "string" 21 | }, 22 | "actionOnFailure": { 23 | "type": "string" 24 | }, 25 | "hadoopJarStep": { 26 | "type": "object", 27 | "properties": { 28 | "mainClass": { 29 | "type": "string" 30 | }, 31 | "jar": { 32 | "type": "string" 33 | }, 34 | "stepArgs": { 35 | "type": "array", 36 | "items": { 37 | "type": "string" 38 | } 39 | } 40 | } 41 | } 42 | } 43 | } 44 | } 45 | } 46 | } -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/schemas/EMRClusterAddCustomStepResponse.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "clusterName": { 5 | "type": "string" 6 | }, 7 | "steps": { 8 | "type": "array", 9 | "items": { 10 | "type": "object", 11 | "properties": { 12 | "name": { 13 | "type": "string" 14 | }, 15 | "stepId": { 16 | "type": "string" 17 | } 18 | } 19 | } 20 | }, 21 | "clusterId": { 22 | "type": "string" 23 | }, 24 | "statusCode": { 25 | "type": "number" 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/schemas/EMRClusterAddStepRequest.json: -------------------------------------------------------------------------------- 1 | { 2 | "required": [ 3 | "clusterName", 4 | "clusterId", 5 | "account", 6 | "step", 7 | "subType" 8 | ], 9 | "properties": { 10 | "clusterName": { 11 | "type": "string" 12 | }, 13 | "clusterId": { 14 | "type": "string" 15 | }, 16 | "account": { 17 | "type": "string" 18 | }, 19 | "step": { 20 | "type": "string" 21 | }, 22 | "subType": { 23 | "type": "string" 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/schemas/EMRClusterAddStepResponse.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "clusterName": { 5 | "type": "string" 6 | }, 7 | "subType": { 8 | "type": "string" 9 | }, 10 | "status": { 11 | "type": "string" 12 | }, 13 | "account": { 14 | "type": "string" 15 | }, 16 | "clusterId": { 17 | "type": "string" 18 | }, 19 | "statusCode": { 20 | "type": "number" 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/schemas/EMRDNSOperation.json: -------------------------------------------------------------------------------- 1 | { 2 | "required": [ 3 | "account", 4 | "dnsName", 5 | "clusterName", 6 | "action", 7 | "masterIp" 8 | ], 9 | "properties": { 10 | "account": { 11 | "type": "string" 12 | }, 13 | "clusterName": { 14 | "type": "string" 15 | }, 16 | "action": { 17 | "type": "string" 18 | }, 19 | "dnsName": { 20 | "type": "string" 21 | }, 22 | "masterIp": { 23 | "type": "string" 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/schemas/EMRRMProxyRequest.json: -------------------------------------------------------------------------------- 1 | { 2 | "required": [ 3 | "rmUrl", 4 | "metricType" 5 | ], 6 | "properties": { 7 | "rmUrl": { 8 | "type": "string" 9 | }, 10 | "metricType": { 11 | "type": "string" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/schemas/EMRRMProxyResponse.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "metricType": { 5 | "type": "string" 6 | }, 7 | "metricStats": { 8 | "type": "string" 9 | }, 10 | "statusCode": { 11 | "type": "number" 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/schemas/EMRStepValidateRequest.json: -------------------------------------------------------------------------------- 1 | { 2 | "required": [ 3 | "clusterName", 4 | "clusterId", 5 | "stepIds" 6 | ], 7 | "properties": { 8 | "clusterName": { 9 | "type": "string" 10 | }, 11 | "clusterId": { 12 | "type": "string" 13 | }, 14 | "stepIds": { 15 | "type": "array", 16 | "items": { 17 | "type": "string" 18 | } 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/schemas/EMRStepValidateResponse.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "clusterName": { 5 | "type": "string" 6 | }, 7 | "steps": { 8 | "type": "array", 9 | "items": { 10 | "type": "object", 11 | "properties": { 12 | "name": { 13 | "type": "string" 14 | }, 15 | "stepId": { 16 | "type": "string" 17 | }, 18 | "status": { 19 | "type": "string" 20 | } 21 | } 22 | } 23 | }, 24 | "clusterId": { 25 | "type": "string" 26 | }, 27 | "statusCode": { 28 | "type": "number" 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/schemas/ProtectEMRClusterRequest.json: -------------------------------------------------------------------------------- 1 | { 2 | "required": [ 3 | "clusterName", 4 | "clusterId", 5 | "terminationProtected" 6 | ], 7 | "properties": { 8 | "clusterName": { 9 | "type": "string" 10 | }, 11 | "clusterId": { 12 | "type": "string" 13 | }, 14 | "terminationProtected": { 15 | "type": "string" 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/schemas/ProtectEMRClusterResponse.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "clusterName": { 5 | "type": "string" 6 | }, 7 | "clusterId": { 8 | "type": "string" 9 | }, 10 | "status": { 11 | "type": "string" 12 | }, 13 | "terminationProtected": { 14 | "type": "string" 15 | }, 16 | "message": { 17 | "type": "string" 18 | }, 19 | "statusCode": { 20 | "type": "number" 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/schemas/TerminateEMRClusterRequest.json: -------------------------------------------------------------------------------- 1 | { 2 | "required": [ 3 | "clusterName", 4 | "clusterId" 5 | ], 6 | "properties": { 7 | "clusterName": { 8 | "type": "string" 9 | }, 10 | "clusterId": { 11 | "type": "string" 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/schemas/TerminateEMRClusterResponse.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "clusterName": { 5 | "type": "string" 6 | }, 7 | "clusterId": { 8 | "type": "string" 9 | }, 10 | "status": { 11 | "type": "string" 12 | }, 13 | "message": { 14 | "type": "string" 15 | }, 16 | "statusCode": { 17 | "type": "number" 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/schemas/ValidateEMRClusterRequest.json: -------------------------------------------------------------------------------- 1 | { 2 | "required": [ 3 | "clusterId" 4 | ], 5 | "properties": { 6 | "clusterId": { 7 | "type": "string" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/serverless/models/schemas/ValidateEMRClusterResponse.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "clusterName": { 5 | "type": "string" 6 | }, 7 | "status": { 8 | "type": "string" 9 | }, 10 | "clusterId": { 11 | "type": "string" 12 | }, 13 | "rmUrl": { 14 | "type": "string" 15 | }, 16 | "masterIp": { 17 | "type": "string" 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Backend/emr-cluster-ops/src/__init__.py -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/src/conf/common/common-tags.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Key": "intuit:billing:component", 4 | "Value": "__role__" 5 | }, 6 | { 7 | "Key": "intuit:billing:user-app", 8 | "Value": "__name__" 9 | }, 10 | { 11 | "Key": "intuit:billing:appenv", 12 | "Value": "prd" 13 | }, 14 | { 15 | "Key": "bu", 16 | "Value": "sbseg" 17 | }, 18 | { 19 | "Key": "env", 20 | "Value": "prod" 21 | }, 22 | { 23 | "Key": "app", 24 | "Value": "__name__" 25 | }, 26 | { 27 | "Key": "Name", 28 | "Value": "__name__-EMR" 29 | }, 30 | { 31 | "Key": "type", 32 | "Value": "__type__" 33 | }, 34 | { 35 | "Key": "ami:restack:date", 36 | "Value": "__restack__" 37 | } 38 | ] -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/src/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Backend/emr-cluster-ops/src/scripts/__init__.py -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/src/scripts/emr_create.py: -------------------------------------------------------------------------------- 1 | """ 2 | Lambda function to launch EMR cluster using Boto3 SDK 3 | """ 4 | 5 | from src.scripts.emr_create_nonkerb import lambda_handler as emr_nonkerb 6 | from src.util.commlib import construct_error_response 7 | from src.util import exceptions 8 | 9 | 10 | def lambda_handler(event, context): 11 | cluster_type = event.get('sub_type', 'unknown').lower() 12 | if cluster_type in {"nonkerb"}: 13 | return emr_nonkerb(event, context) 14 | else: 15 | # Throw an exception for unsupported cluster type passed 16 | api_request_id = event.get('api_request_id') 17 | error_response = construct_error_response(context, api_request_id) 18 | raise exceptions.EMRClusterCreationException(error_response) -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/src/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Backend/emr-cluster-ops/src/util/__init__.py -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Backend/emr-cluster-ops/tests/__init__.py -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/tests/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = -p no:warnings 3 | norecursedirs = build docs/_build *.eggs .tox *.venv *__init__* 4 | testpaths = ./tests 5 | 6 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/tests/test_add_step.py: -------------------------------------------------------------------------------- 1 | from pprint import pprint 2 | from src.scripts.emr_add_steps import lambda_handler as emr_add_step 3 | 4 | 5 | def test_emr_cluster_add_step(create_cluster, lambda_context): 6 | cluster_id = create_cluster.get('cluster_id') 7 | cluster_name = create_cluster.get('cluster_name') 8 | cluster_type = create_cluster.get('cluster_type') 9 | 10 | assert 'j-' in cluster_id 11 | 12 | print(f"Cluster id {create_cluster.get('cluster_id')}") 13 | 14 | emr_add_step_request = { 15 | "cluster_name": cluster_name, 16 | "account": create_cluster.get('account'), 17 | "step": "Setup Syslog", 18 | "cluster_id": cluster_id, 19 | "cluster_type": cluster_type} 20 | 21 | response = emr_add_step(emr_add_step_request, lambda_context) 22 | 23 | assert 'step_id' in response 24 | print(f"step id {response.get('step_id')}") 25 | 26 | 27 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/tests/test_emr_creation.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pprint import pprint 3 | 4 | 5 | def test_emr_create_cluster(create_cluster): 6 | print("\n") 7 | pprint(create_cluster, indent=4, compact=True) 8 | assert 'j-' in create_cluster.get('cluster_id') 9 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/tests/test_emr_precheck.py: -------------------------------------------------------------------------------- 1 | from src.scripts.emr_precheck import lambda_handler as emr_precheck 2 | from pprint import pprint 3 | 4 | 5 | def test_emr_prechcek_cluster_notpresent(lambda_context): 6 | cluster_precheck_request = { 7 | "api_request_id": "test_emr_precheck", 8 | "cluster_name": "nonkerb-testing-tes" 9 | } 10 | response = emr_precheck(cluster_precheck_request, lambda_context) 11 | assert 'status' in response 12 | print('Cluster status ::', response.get('status')) 13 | 14 | 15 | def test_emr_prechcek_cluster_present(create_cluster, lambda_context): 16 | assert 'j-' in create_cluster.get('cluster_id') 17 | print('Cluster id', create_cluster.get('cluster_id')) 18 | 19 | cluster_name = create_cluster.get('cluster_name') 20 | 21 | cluster_precheck_request = { 22 | "api_request_id": "test_emr_precheck", 23 | "cluster_name": cluster_name 24 | } 25 | 26 | response = emr_precheck(cluster_precheck_request,lambda_context ) 27 | assert 'status' in response 28 | print('Cluster status ::', response.get('status')) 29 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/tests/test_emr_protect.py: -------------------------------------------------------------------------------- 1 | from src.scripts.emr_protect import lambda_handler as emr_protect 2 | 3 | 4 | def test_emr_cluster_protect(create_cluster, lambda_context): 5 | cluster_id = create_cluster.get('cluster_id') 6 | cluster_name = create_cluster.get('cluster_name') 7 | cluster_type = create_cluster.get('cluster_type') 8 | 9 | assert 'j-' in cluster_id 10 | print(f"Cluster ID :: {cluster_id}") 11 | 12 | emr_protect_request = { 13 | "api_request_id": "test_terminate_cluster", 14 | "cluster_name": cluster_name, 15 | "cluster_type": cluster_type, 16 | "cluster_id": cluster_id, 17 | "termination_protected": "status" 18 | } 19 | 20 | response = emr_protect(emr_protect_request, lambda_context) 21 | assert response.get('terminationProtected') in ['enabled', 'disabled'] 22 | 23 | print(f"Cluster termination protection status :: {response.get('terminationProtected')}") 24 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/tests/test_emr_status.py: -------------------------------------------------------------------------------- 1 | from src.scripts.emr_status import lambda_handler as emr_status 2 | from pprint import pprint 3 | 4 | 5 | def test_emr_cluster_status_check(create_cluster, lambda_context): 6 | print("\n") 7 | pprint(create_cluster, indent=4, compact=True) 8 | assert 'j-' in create_cluster.get('cluster_id') 9 | 10 | cluster_id = create_cluster.get('cluster_id') 11 | 12 | cluster_status_request = { 13 | "api_request_id": "test_emr_status_check", 14 | "cluster_id": cluster_id 15 | } 16 | 17 | response = emr_status(cluster_status_request,lambda_context ) 18 | assert 'status' in response 19 | print("\n") 20 | pprint(response) -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/tests/test_emr_terminate.py: -------------------------------------------------------------------------------- 1 | from src.scripts.emr_terminate import lambda_handler as emr_terminate 2 | 3 | 4 | def test_terminate_emr_cluster(create_cluster, lambda_context): 5 | cluster_id = create_cluster.get('cluster_id') 6 | cluster_name = create_cluster.get('cluster_name') 7 | cluster_type = create_cluster.get('cluster_type') 8 | 9 | assert 'j-' in cluster_id 10 | print(f"Cluster ID :: {cluster_id}") 11 | 12 | emr_terminate_request = { 13 | "api_request_id": "test_terminate_cluster", 14 | "cluster_name": cluster_name, 15 | "cluster_type": cluster_type, 16 | "cluster_id": cluster_id, 17 | "force": True 18 | } 19 | 20 | response = emr_terminate(emr_terminate_request, lambda_context) 21 | 22 | assert 'TERMINATION' in response.get('status') 23 | 24 | print(f"Cluster termination status :: {response.get('status')}") 25 | -------------------------------------------------------------------------------- /Backend/emr-cluster-ops/tests/test_emr_validate.py: -------------------------------------------------------------------------------- 1 | from src.scripts.emr_validate import lambda_handler as emr_validate 2 | from pprint import pprint 3 | 4 | 5 | def test_emr_cluster_validate(create_cluster, lambda_context): 6 | cluster_id = create_cluster.get('cluster_id') 7 | assert 'j-' in cluster_id 8 | 9 | cluster_validate_request = { 10 | "api_request_id": "test_emr_validation", 11 | "cluster_name": create_cluster.get('cluster_name'), 12 | "cluster_type": create_cluster.get('cluster_type') 13 | } 14 | 15 | try: 16 | response = emr_validate(cluster_validate_request, lambda_context) 17 | except Exception as error: 18 | assert 'NotImplementedError' not in error.args 19 | pass 20 | -------------------------------------------------------------------------------- /Backend/emr-testsuites/README.md: -------------------------------------------------------------------------------- 1 | # emr-autoscaling 2 | -------------------------------------------------------------------------------- /Backend/emr-testsuites/serverless/cfn/resource.yml: -------------------------------------------------------------------------------- 1 | Resources: 2 | ApiGatewayStage: 3 | Type: AWS::ApiGateway::Stage 4 | Properties: 5 | MethodSettings: 6 | - DataTraceEnabled: true 7 | HttpMethod: "*" 8 | LoggingLevel: INFO 9 | ResourcePath: "/*" 10 | MetricsEnabled: true 11 | RestApiId: ${self:provider.apiGateway.restApiId} 12 | EmrRequestValidatorParams: 13 | Type: "AWS::ApiGateway::RequestValidator" 14 | Properties: 15 | Name: 'emr-req-validator-params' 16 | RestApiId: ${self:provider.apiGateway.restApiId} 17 | ValidateRequestBody: false 18 | ValidateRequestParameters: true 19 | EmrRequestValidatorBody: 20 | Type: "AWS::ApiGateway::RequestValidator" 21 | Properties: 22 | Name: 'emr-req-validator-body' 23 | RestApiId: ${self:provider.apiGateway.restApiId} 24 | ValidateRequestBody: true 25 | ValidateRequestParameters: true 26 | -------------------------------------------------------------------------------- /Backend/emr-testsuites/serverless/models/EMRClusterTestAppRequest.json: -------------------------------------------------------------------------------- 1 | { 2 | "required": [ 3 | "clusterName", 4 | "clusterId", 5 | "app" 6 | ], 7 | "properties": { 8 | "clusterName": { 9 | "type": "string" 10 | }, 11 | "clusterId": { 12 | "type": "string" 13 | }, 14 | "app": { 15 | "type": "string" 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /Backend/emr-testsuites/serverless/models/EMRClusterTestAppResponse.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "clusterName": { 5 | "type": "string" 6 | }, 7 | "app": { 8 | "type": "string" 9 | }, 10 | "status": { 11 | "type": "string" 12 | }, 13 | "stepId": { 14 | "type": "string" 15 | }, 16 | "clusterId": { 17 | "type": "string" 18 | }, 19 | "statusCode": { 20 | "type": "number" 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Backend/emr-testsuites/serverless/models/EMRTestSuiteRequest.json: -------------------------------------------------------------------------------- 1 | { 2 | "required": [ 3 | "clusterId" 4 | ], 5 | "properties": { 6 | "clusterId": { 7 | "type": "string" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Backend/emr-testsuites/serverless/models/EMRTestSuiteResponse.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "status": { 5 | "type": "string" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /Backend/emr-testsuites/serverless/models/models.yml: -------------------------------------------------------------------------------- 1 | api: 2 | info: 3 | version: '1.0.0' 4 | title: EMR cluster Ops APIs 5 | description: APIs to perform EMR Test Suite operations 6 | models: 7 | - name: EMRTestSuiteRequest 8 | contentType: "application/json" 9 | schema: ${file(serverless/models/EMRTestSuiteRequest.json)} 10 | - name: EMRTestSuiteResponse 11 | contentType: "application/json" 12 | schema: ${file(serverless/models/EMRTestSuiteResponse.json)} 13 | - name: EMRClusterTestAppRequest 14 | contentType: "application/json" 15 | schema: ${file(serverless/models/EMRClusterTestAppRequest.json)} 16 | - name: EMRClusterTestAppResponse 17 | contentType: "application/json" 18 | schema: ${file(serverless/models/EMRClusterTestAppResponse.json)} -------------------------------------------------------------------------------- /Backend/emr-testsuites/src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Backend/emr-testsuites/src/__init__.py -------------------------------------------------------------------------------- /Backend/emr-testsuites/src/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Backend/emr-testsuites/src/scripts/__init__.py -------------------------------------------------------------------------------- /Backend/emr-testsuites/src/scripts/emr_test_steps.py: -------------------------------------------------------------------------------- 1 | """ 2 | Lambda function to get list of steps executed on emr cluster. 3 | """ 4 | 5 | from src.util import exceptions 6 | from src.util.log import setup_logging 7 | from src.util.emrlib import get_emr_steps 8 | 9 | 10 | def lambda_handler(event, context): 11 | api_request_id = event.get('api_request_id') 12 | logger = setup_logging(api_request_id, context.aws_request_id) 13 | cluster_id = event.get('cluster_id') 14 | 15 | success_response = { 16 | 'statusCode': 201, 17 | "api_request_id": api_request_id, 18 | "lambda_request_id": context.aws_request_id, 19 | 'cluster_id': cluster_id, 20 | 'steps_count': 0 21 | } 22 | 23 | error_response = { 24 | "statusCode": 500, 25 | "errorType": "NoClusterFound", 26 | "errorMessage": "Unable to fetch cluster step details." 27 | } 28 | 29 | logger.info("Getting steps by cluster_id") 30 | try: 31 | response = get_emr_steps(cluster_id) 32 | except Exception as e: 33 | logger.error(e) 34 | raise exceptions.EMRTestRunException(error_response) 35 | else: 36 | success_response.update(steps_count=len(response)) 37 | success_response.update(steps=response) 38 | 39 | return success_response 40 | -------------------------------------------------------------------------------- /Backend/emr-testsuites/src/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Backend/emr-testsuites/src/util/__init__.py -------------------------------------------------------------------------------- /Backend/emr-testsuites/src/util/log.py: -------------------------------------------------------------------------------- 1 | """ 2 | Base script for logging Lambda functions 3 | """ 4 | 5 | import logging 6 | import sys 7 | 8 | 9 | def setup_logging(api_request_id, lambda_request_id): 10 | logger = logging.getLogger() 11 | for handler in logger.handlers: 12 | logger.removeHandler(handler) 13 | 14 | handler = logging.StreamHandler(sys.stdout) 15 | 16 | log_format = f"[%(asctime)s] [api_request_id={api_request_id} lambda_request_id={lambda_request_id}] %(levelname)s: %(message)s" 17 | handler.setFormatter(logging.Formatter(log_format, "%Y-%m-%d %H:%M:%S")) 18 | logger.addHandler(handler) 19 | logger.setLevel(logging.INFO) 20 | 21 | return logger 22 | -------------------------------------------------------------------------------- /Backend/emr-testsuites/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Backend/emr-testsuites/tests/__init__.py -------------------------------------------------------------------------------- /Backend/emr-testsuites/tests/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = -p no:warnings 3 | norecursedirs = build docs/_build *.egg .tox *.venv *__init__* 4 | -------------------------------------------------------------------------------- /Backend/emr-testsuites/tests/test_emr_app.py: -------------------------------------------------------------------------------- 1 | from src.scripts.emr_test_app import lambda_handler as emr_app 2 | 3 | 4 | def test_emr_test_app(create_cluster, lambda_context): 5 | cluster_id = create_cluster.get('JobFlowId') 6 | assert 'j-' in cluster_id 7 | 8 | cluster_name = create_cluster.get('Name') 9 | 10 | test_app_request = { 11 | "api_request_id": "test_app_hive", 12 | "cluster_name": cluster_name, 13 | "cluster_id": cluster_id, 14 | "app": "hive" 15 | } 16 | 17 | try: 18 | response = emr_app(test_app_request, lambda_context) 19 | except Exception as error: 20 | print(error) 21 | else: 22 | assert 'step_id' in response 23 | print(f"Testing Hive app via step id {response.get('step_id')}") -------------------------------------------------------------------------------- /Backend/emr-testsuites/tests/test_emr_asg.py: -------------------------------------------------------------------------------- 1 | from src.scripts.emr_test_autoscaling import lambda_handler as list_asg 2 | from pprint import pprint 3 | 4 | 5 | def test_emr_list_asg(create_cluster, lambda_context): 6 | cluster_id = create_cluster.get('JobFlowId') 7 | assert 'j-' in cluster_id 8 | 9 | cluster_name = create_cluster.get('Name') 10 | 11 | list_asg_request = { 12 | "api_request_id": "test_list_asg", 13 | "cluster_name": cluster_name, 14 | "cluster_id": cluster_id, 15 | "instance_group": "CORE" 16 | } 17 | 18 | try: 19 | response = list_asg(list_asg_request, lambda_context) 20 | except Exception as error: 21 | print(error) 22 | else: 23 | assert 'state' in response 24 | print("Autoscaling details", response) -------------------------------------------------------------------------------- /Backend/emr-testsuites/tests/test_emr_ba.py: -------------------------------------------------------------------------------- 1 | from src.scripts.emr_test_bootstrap import lambda_handler as emr_bas 2 | 3 | 4 | def test_emr_bas(create_cluster, lambda_context): 5 | cluster_id = create_cluster.get('JobFlowId') 6 | assert 'j-' in cluster_id 7 | 8 | cluster_name = create_cluster.get('Name') 9 | 10 | bas_request = { 11 | "api_request_id": "test_bootstrap_actions", 12 | "cluster_id": cluster_id, 13 | "cluster_name": cluster_name 14 | } 15 | try: 16 | response = emr_bas(bas_request, lambda_context) 17 | except Exception as error: 18 | print(error) 19 | else: 20 | assert 2 == response.get('bootstrap_count') 21 | print("bootstrap count", response.get('bootstrap_count') ) 22 | print("bootstraps", response.get('bootstrap_names') ) -------------------------------------------------------------------------------- /Backend/emr-testsuites/tests/test_emr_steps.py: -------------------------------------------------------------------------------- 1 | from src.scripts.emr_test_steps import lambda_handler as emr_steps 2 | 3 | 4 | def test_emr_steps(create_cluster, lambda_context): 5 | cluster_id = create_cluster.get('JobFlowId') 6 | assert 'j-' in cluster_id 7 | 8 | cluster_name = create_cluster.get('Name') 9 | 10 | steps_request = { 11 | "api_request_id": "test_bootstrap_actions", 12 | "cluster_id": cluster_id, 13 | "cluster_name": cluster_name 14 | } 15 | 16 | try: 17 | response = emr_steps(steps_request, lambda_context) 18 | except Exception as error: 19 | print(error) 20 | else: 21 | assert 2 == response.get('steps_count') and 'steps' in response 22 | print("Step count", response.get('steps_count') ) 23 | print("Steps", response.get('steps')) -------------------------------------------------------------------------------- /Backend/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = -p no:warnings 3 | norecursedirs = build docs/_build *.eggs* *.eggs *.egg .tox *.venv *__init__* *site-packages* 4 | 5 | -------------------------------------------------------------------------------- /Backend/serverless-deploy/README.md: -------------------------------------------------------------------------------- 1 | # emr-ci-deploy -------------------------------------------------------------------------------- /Backend/serverless-deploy/logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | 4 | 5 | def setup_logger(debug=False): 6 | """ 7 | Configure logger 8 | :return: logger handler 9 | """ 10 | 11 | level = logging.DEBUG if debug else logging.INFO 12 | logger = logging.getLogger("Quickfabric Deploy") 13 | logger.handlers = [] 14 | logger.propagate = False 15 | handler = logging.StreamHandler(sys.stdout) 16 | formatter = logging.Formatter("%(asctime)s - %(name)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s") 17 | handler.setFormatter(formatter) 18 | logger.addHandler(handler) 19 | 20 | logger.setLevel(level) 21 | 22 | return logger 23 | -------------------------------------------------------------------------------- /Backend/serverless-deploy/plugin_metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "serverless-pseudo-parameters": "2.5.0", 3 | "serverless-add-api-key": "4.0.2", 4 | "serverless-aws-documentation": "1.1.0", 5 | "serverless-deployment-bucket": "1.1.0", 6 | "serverless-jetpack": "0.7.0", 7 | "serverless-plugin-stage-variables": "1.9.5", 8 | "serverless-reqvalidator-plugin": "1.0.3", 9 | "serverless-plugin-parent": "latest", 10 | "serverless-s3-sync": "latest", 11 | "serverless-plugin-existing-s3": "latest", 12 | "serverless-api-stage": "latest" 13 | } -------------------------------------------------------------------------------- /Backend/setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test=pytest -------------------------------------------------------------------------------- /Backend/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | test_requirements = ['pytest','pytest-sugar', 4 | 'boto3', 'aws_lambda_context', 'coverage', 5 | 'pytest-cov', 'moto'] 6 | 7 | 8 | setup( 9 | setup_requires=['pyyaml', 'pytest-runner', 'pytest'], 10 | dependency_links=[ 11 | 'git+https://github.com/sandipnahak/moto.git@master#egg=moto' 12 | ], 13 | install_requires=['pytest'] + test_requirements, 14 | tests_require=['pytest'] + test_requirements 15 | ) -------------------------------------------------------------------------------- /DB/.gitignore: -------------------------------------------------------------------------------- 1 | Dump20200107.sql 2 | -------------------------------------------------------------------------------- /DB/Dockerfile: -------------------------------------------------------------------------------- 1 | # Derived from official mysql image (our base image) 2 | FROM mysql 3 | # Add a database 4 | #ENV MYSQL_DATABASE company 5 | # Add the content of the sql-scripts/ directory to your image 6 | # All scripts in docker-entrypoint-initdb.d/ are automatically 7 | # executed during container startup 8 | COPY quickfabric_bootstrap.sql /docker-entrypoint-initdb.d/ 9 | COPY patch /patch 10 | RUN chmod 755 /patch/patch.sh 11 | -------------------------------------------------------------------------------- /DB/QuickFabric_ER_Diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/DB/QuickFabric_ER_Diagram.png -------------------------------------------------------------------------------- /DB/QuickFabric_ER_model.mwb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/DB/QuickFabric_ER_model.mwb -------------------------------------------------------------------------------- /DB/dcl.sql: -------------------------------------------------------------------------------- 1 | CREATE USER 'qf_admin'@'%' IDENTIFIED BY 'supersecret'; 2 | GRANT ALL PRIVILEGES ON * . * TO 'qf_admin'@'%'; 3 | FLUSH PRIVILEGES; -------------------------------------------------------------------------------- /DB/ddl/account_configurations.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Table structure for table `account_configurations` 3 | -- 4 | CREATE TABLE IF NOT EXISTS `account_configurations` ( 5 | `int` int(11) NOT NULL AUTO_INCREMENT, 6 | `config_id` int(11) NOT NULL, 7 | `account_id` varchar(128) NOT NULL, 8 | `config_value` varchar(128) NOT NULL, 9 | `is_encrypted` varchar(45) NOT NULL DEFAULT '0', 10 | PRIMARY KEY (`int`), 11 | UNIQUE KEY `composite_unique_config_id_and_account_id` (`config_id`,`account_id`), 12 | KEY `fk_configuration_id_idx` (`config_id`), 13 | KEY `fk_account_id_idx` (`account_id`), 14 | CONSTRAINT `fk_account_id` FOREIGN KEY (`account_id`) REFERENCES `aws_account_profile` (`account_id`) ON DELETE NO ACTION ON UPDATE NO ACTION, 15 | CONSTRAINT `fk_configuration_id` FOREIGN KEY (`config_id`) REFERENCES `configuration_definitions` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION 16 | ); 17 | -------------------------------------------------------------------------------- /DB/ddl/application_configurations.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE `application_configurations` ( 2 | `id` int(11) NOT NULL AUTO_INCREMENT, 3 | `config_id` int(11) NOT NULL, 4 | `config_value` varchar(128) NOT NULL, 5 | `is_encrypted` tinyint(4) NOT NULL DEFAULT '0', 6 | PRIMARY KEY (`id`), 7 | UNIQUE KEY `config_id_UNIQUE` (`config_id`), 8 | KEY `fk_configuration_id_idx` (`config_id`), 9 | CONSTRAINT `fk_global_configuration_id` FOREIGN KEY (`config_id`) REFERENCES `configuration_definitions` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION 10 | ); -------------------------------------------------------------------------------- /DB/ddl/aws_account_profile.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Table structure for table `aws_account_profile` 3 | -- 4 | 5 | CREATE TABLE IF NOT EXISTS `aws_account_profile` ( 6 | `id` int(11) NOT NULL AUTO_INCREMENT, 7 | `account_id` varchar(256) NOT NULL, 8 | `account_env` varchar(256) NOT NULL, 9 | `account_owner` varchar(256) NOT NULL, 10 | PRIMARY KEY (`id`), 11 | UNIQUE KEY `account_id_UNIQUE` (`account_id`) 12 | ) AUTO_INCREMENT=200000; 13 | -------------------------------------------------------------------------------- /DB/ddl/cluster_metrics.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Table structure for table `cluster_metrics` 3 | -- 4 | 5 | CREATE TABLE IF NOT EXISTS `cluster_metrics` ( 6 | `emr_id` varchar(512) NOT NULL DEFAULT '', 7 | `rm_url` varchar(1024) DEFAULT NULL, 8 | `refresh_timestamp` datetime DEFAULT NULL, 9 | `metrics_json` text, 10 | `emr_status` varchar(100) DEFAULT NULL, 11 | `cost` float DEFAULT '100', 12 | `available_memory_perc` float DEFAULT NULL, 13 | `available_cores_perc` float DEFAULT NULL, 14 | `emr_name` varchar(255) DEFAULT NULL, 15 | `total_nodes` int(11) DEFAULT NULL, 16 | `containers_pending` int(11) DEFAULT NULL, 17 | `apps_pending` int(11) DEFAULT NULL, 18 | `apps_running` int(11) DEFAULT NULL, 19 | `apps_succeeded` int(11) DEFAULT NULL, 20 | `apps_failed` int(11) DEFAULT NULL, 21 | `created_by` varchar(45) DEFAULT NULL, 22 | `account` varchar(45) DEFAULT NULL, 23 | `type` varchar(100) DEFAULT NULL, 24 | `cluster_create_timestamp` datetime DEFAULT NULL, 25 | `segment` varchar(100) DEFAULT NULL 26 | ); 27 | -------------------------------------------------------------------------------- /DB/ddl/cluster_metrics_history.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Table structure for table `cluster_metrics_history` 3 | -- 4 | 5 | CREATE TABLE IF NOT EXISTS `cluster_metrics_history` ( 6 | `emr_id` varchar(512) NOT NULL DEFAULT '', 7 | `rm_url` varchar(1024) DEFAULT NULL, 8 | `refresh_timestamp` datetime DEFAULT NULL, 9 | `metrics_json` text, 10 | `emr_status` varchar(100) DEFAULT NULL, 11 | `cost` float DEFAULT '100', 12 | `available_memory_perc` float DEFAULT NULL, 13 | `available_cores_perc` float DEFAULT NULL, 14 | `emr_name` varchar(255) DEFAULT NULL, 15 | `total_nodes` int(11) DEFAULT NULL, 16 | `containers_pending` int(11) DEFAULT NULL, 17 | `apps_pending` int(11) DEFAULT NULL, 18 | `apps_running` int(11) DEFAULT NULL, 19 | `apps_succeeded` int(11) DEFAULT NULL, 20 | `apps_failed` int(11) DEFAULT NULL, 21 | `created_by` varchar(45) DEFAULT NULL, 22 | `account` varchar(45) DEFAULT NULL, 23 | `cluster_create_timestamp` datetime DEFAULT NULL, 24 | `type` varchar(100) DEFAULT NULL, 25 | `segment` varchar(50) DEFAULT NULL 26 | ); 27 | -------------------------------------------------------------------------------- /DB/ddl/cluster_step_request.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS `cluster_step_request` ( 2 | `cluster_name` varchar(100) DEFAULT NULL, 3 | `cluster_id` varchar(45) DEFAULT NULL, 4 | `step_id` varchar(45) DEFAULT NULL, 5 | `api_request_id` varchar(45) DEFAULT NULL, 6 | `lambda_request_id` varchar(45) DEFAULT NULL, 7 | `updated_ts` datetime DEFAULT NULL, 8 | `step_arg` varchar(3000) DEFAULT NULL, 9 | `created_ts` datetime DEFAULT NULL, 10 | `name` varchar(45) DEFAULT NULL, 11 | `action_on_failure` varchar(200) DEFAULT NULL, 12 | `main_class` varchar(45) DEFAULT NULL, 13 | `jar` varchar(100) DEFAULT NULL, 14 | `status` varchar(45) DEFAULT NULL, 15 | `created_by` varchar(55) DEFAULT NULL, 16 | `step_type` varchar(50) NOT NULL DEFAULT 'Custom' 17 | ); 18 | -------------------------------------------------------------------------------- /DB/ddl/configuration_data_types.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS `configuration_data_types` ( 2 | `id` int(11) NOT NULL AUTO_INCREMENT, 3 | `data_type_name` varchar(45) NOT NULL, 4 | PRIMARY KEY (`id`), 5 | UNIQUE KEY `data_type_name_UNIQUE` (`data_type_name`) 6 | ); -------------------------------------------------------------------------------- /DB/ddl/configuration_definitions.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS `configuration_definitions` ( 2 | `id` int(11) NOT NULL AUTO_INCREMENT, 3 | `config_name` varchar(128) NOT NULL, 4 | `config_description` varchar(256) NOT NULL, 5 | `config_type` int(11) NOT NULL, 6 | `data_type` int(11) NOT NULL DEFAULT '1', 7 | `encryption_required` tinyint(4) NOT NULL DEFAULT '0', 8 | `is_mandatory` tinyint(4) NOT NULL DEFAULT '0', 9 | `is_user_accessible` tinyint(4) NOT NULL DEFAULT '0', 10 | PRIMARY KEY (`id`), 11 | UNIQUE KEY `config_name_UNIQUE` (`config_name`), 12 | KEY `configuration_type_idx` (`config_type`), 13 | KEY `fk_data_type_idx` (`data_type`), 14 | CONSTRAINT `fk_configuration_type` FOREIGN KEY (`config_type`) REFERENCES `configuration_types` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION, 15 | CONSTRAINT `fk_configurations_definition_configuration_data_type` FOREIGN KEY (`data_type`) REFERENCES `configuration_data_types` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION 16 | ); 17 | -------------------------------------------------------------------------------- /DB/ddl/configuration_types.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS `configuration_types` ( 2 | `id` int(11) NOT NULL AUTO_INCREMENT, 3 | `config_type` varchar(45) NOT NULL, 4 | PRIMARY KEY (`id`), 5 | UNIQUE KEY `config_type_UNIQUE` (`config_type`) 6 | ); 7 | -------------------------------------------------------------------------------- /DB/ddl/db_patch.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Table structure for table `db_patch` 3 | -- 4 | CREATE TABLE `db_patch` ( 5 | `id` int(11) NOT NULL AUTO_INCREMENT, 6 | `patch_name` varchar(128) NOT NULL, 7 | `patch_status` varchar(45) NOT NULL, 8 | PRIMARY KEY (`id`), 9 | UNIQUE KEY `patch_name_UNIQUE` (`patch_name`) 10 | ); 11 | -------------------------------------------------------------------------------- /DB/ddl/emr_billing_component_cost.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Table structure for table `emr_billing_component_cost` 3 | -- 4 | CREATE TABLE IF NOT EXISTS `emr_billing_component_cost` ( 5 | `emr_name` varchar(45) NOT NULL, 6 | `emr_cost` int(11) DEFAULT NULL, 7 | PRIMARY KEY (`emr_name`) 8 | ); 9 | -------------------------------------------------------------------------------- /DB/ddl/emr_billing_component_cost_hist.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Table structure for table `emr_billing_component_cost_hist` 3 | -- 4 | CREATE TABLE IF NOT EXISTS `emr_billing_component_cost_hist` ( 5 | `emr_name` varchar(45) NOT NULL, 6 | `emr_cost` int(11) DEFAULT NULL, 7 | `created_date` datetime DEFAULT CURRENT_TIMESTAMP 8 | ); 9 | -------------------------------------------------------------------------------- /DB/ddl/emr_functional_testsuites.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Table structure for table `emr_functional_testsuites` 3 | -- 4 | CREATE TABLE IF NOT EXISTS `emr_functional_testsuites` ( 5 | `id` int(11) NOT NULL AUTO_INCREMENT, 6 | `name` varchar(767) NOT NULL, 7 | `description` varchar(256) DEFAULT NULL, 8 | `criteria` varchar(256) DEFAULT NULL, 9 | `cluster_type` varchar(256) NOT NULL, 10 | `cluster_segment` varchar(256) NOT NULL, 11 | `timeout` int(11) NOT NULL DEFAULT '60', 12 | `expires_minutes` int(11) NOT NULL DEFAULT '60', 13 | `mandatory` tinyint(1) NOT NULL DEFAULT '0', 14 | `disabled` tinyint(1) NOT NULL DEFAULT '0', 15 | PRIMARY KEY (`id`), 16 | UNIQUE KEY `TestName_ClusterSegment_ClusterType_Unique_Constraint` (`name`,`cluster_type`,`cluster_segment`) 17 | ); 18 | -------------------------------------------------------------------------------- /DB/ddl/emr_functional_testsuites_status.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Table structure for table `emr_functional_testsuites_status` 3 | -- 4 | CREATE TABLE IF NOT EXISTS `emr_functional_testsuites_status` ( 5 | `execution_id` int(11) NOT NULL AUTO_INCREMENT, 6 | `name` varchar(1024) NOT NULL, 7 | `status` varchar(256) NOT NULL, 8 | `cluster_id` varchar(256) NOT NULL, 9 | `cluster_name` varchar(256) NOT NULL, 10 | `cluster_type` varchar(256) NOT NULL, 11 | `cluster_segment` varchar(256) NOT NULL, 12 | `execution_start_time` datetime NOT NULL, 13 | `execution_end_time` datetime DEFAULT NULL, 14 | `executed_by` varchar(256) NOT NULL, 15 | `remark` varchar(1024) DEFAULT NULL, 16 | `expires_minutes` int(11) NOT NULL DEFAULT '60', 17 | `mandatory` tinyint(1) NOT NULL DEFAULT '0', 18 | `disabled` tinyint(1) NOT NULL DEFAULT '0', 19 | PRIMARY KEY (`execution_id`) 20 | ); 21 | -------------------------------------------------------------------------------- /DB/ddl/emr_functional_testsuites_status_history.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Table structure for table `emr_functional_testsuites_status_history` 3 | -- 4 | CREATE TABLE IF NOT EXISTS `emr_functional_testsuites_status_history` ( 5 | `execution_id` int(11) NOT NULL AUTO_INCREMENT, 6 | `name` varchar(1024) NOT NULL, 7 | `status` varchar(256) NOT NULL, 8 | `cluster_id` varchar(256) NOT NULL, 9 | `cluster_name` varchar(256) NOT NULL, 10 | `cluster_type` varchar(256) NOT NULL, 11 | `cluster_segment` varchar(256) NOT NULL, 12 | `execution_start_time` datetime NOT NULL, 13 | `execution_end_time` datetime DEFAULT NULL, 14 | `executed_by` varchar(256) NOT NULL, 15 | `remark` varchar(1024) DEFAULT NULL, 16 | `expires_minutes` int(11) NOT NULL DEFAULT '60', 17 | `mandatory` tinyint(1) NOT NULL DEFAULT '0', 18 | `disabled` tinyint(1) NOT NULL DEFAULT '0', 19 | PRIMARY KEY (`execution_id`) 20 | ); 21 | -------------------------------------------------------------------------------- /DB/ddl/report_subscriptions.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Table structure for table `report_subscriptions` 3 | -- 4 | CREATE TABLE IF NOT EXISTS `report_subscriptions` ( 5 | `report_subscription_id` int(11) NOT NULL AUTO_INCREMENT, 6 | `report_id` int(11) NOT NULL, 7 | `user_id` int(11) NOT NULL, 8 | PRIMARY KEY (`report_subscription_id`) 9 | ); 10 | -------------------------------------------------------------------------------- /DB/ddl/reports.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Table structure for table `reports` 3 | -- 4 | 5 | CREATE TABLE IF NOT EXISTS `reports` ( 6 | `report_id` int(11) NOT NULL AUTO_INCREMENT, 7 | `report_name` varchar(256) NOT NULL, 8 | PRIMARY KEY (`report_id`) 9 | ); 10 | -------------------------------------------------------------------------------- /DB/ddl/roles.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Table structure for table `roles` 3 | -- 4 | CREATE TABLE IF NOT EXISTS `roles` ( 5 | `role_id` int(11) NOT NULL AUTO_INCREMENT, 6 | `role_name` varchar(128) NOT NULL, 7 | `service_id` int(11) NOT NULL, 8 | PRIMARY KEY (`role_id`), 9 | KEY `role_service_foreign_key_idx` (`service_id`), 10 | CONSTRAINT `role_to_services_foreign_key` FOREIGN KEY (`service_id`) REFERENCES `services` (`service_id`) ON DELETE NO ACTION ON UPDATE NO ACTION 11 | ) AUTO_INCREMENT=400000; 12 | -------------------------------------------------------------------------------- /DB/ddl/segments.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Table structure for table `segments` 3 | -- 4 | CREATE TABLE IF NOT EXISTS `segments` ( 5 | `segment_id` int(11) NOT NULL AUTO_INCREMENT, 6 | `segment_name` varchar(128) NOT NULL, 7 | `business_owner` varchar(128) DEFAULT NULL, 8 | `business_owner_email` varchar(128) DEFAULT NULL, 9 | PRIMARY KEY (`segment_id`) 10 | ) AUTO_INCREMENT=300000; 11 | -------------------------------------------------------------------------------- /DB/ddl/services.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Table structure for table `services` 3 | -- 4 | CREATE TABLE IF NOT EXISTS `services` ( 5 | `service_id` int(11) NOT NULL AUTO_INCREMENT, 6 | `service_type` varchar(45) NOT NULL, 7 | PRIMARY KEY (`service_id`) 8 | ) AUTO_INCREMENT 500000; 9 | -------------------------------------------------------------------------------- /DB/ddl/user.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Table structure for table `user` 3 | -- 4 | CREATE TABLE IF NOT EXISTS `user` ( 5 | `user_id` int(11) NOT NULL AUTO_INCREMENT, 6 | `email_id` varchar(50) NOT NULL, 7 | `first_name` varchar(20) NOT NULL, 8 | `last_name` varchar(20) NOT NULL, 9 | `creation_date` varchar(50) NOT NULL, 10 | `passcode` varchar(60) NOT NULL, 11 | `super_admin` tinyint(1) NOT NULL DEFAULT '0', 12 | PRIMARY KEY (`user_id`) 13 | ); 14 | -------------------------------------------------------------------------------- /DB/ddl/user_account_segment_mapping.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Table structure for table `user_account_segment_mapping` 3 | -- 4 | CREATE TABLE IF NOT EXISTS `user_account_segment_mapping` ( 5 | `user_account_segment_mapping_id` int(11) NOT NULL AUTO_INCREMENT, 6 | `user_id` int(11) NOT NULL, 7 | `aws_account_id` int(11) NOT NULL, 8 | `segment_id` int(11) NOT NULL, 9 | PRIMARY KEY (`user_account_segment_mapping_id`), 10 | UNIQUE KEY `unique_user_account_segment` (`user_id`,`aws_account_id`,`segment_id`), 11 | KEY `user_foreign_key_idx` (`user_id`), 12 | KEY `account_foreign_key_idx` (`aws_account_id`), 13 | KEY `segment_foregin_key_idx` (`segment_id`), 14 | CONSTRAINT `aws_account_foreign_key` FOREIGN KEY (`aws_account_id`) REFERENCES `aws_account_profile` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION, 15 | CONSTRAINT `segment_foregin_key` FOREIGN KEY (`segment_id`) REFERENCES `segments` (`segment_id`) ON DELETE NO ACTION ON UPDATE NO ACTION, 16 | CONSTRAINT `user_foreign_key` FOREIGN KEY (`user_id`) REFERENCES `user` (`user_id`) ON DELETE NO ACTION ON UPDATE NO ACTION 17 | ) AUTO_INCREMENT=600000; 18 | -------------------------------------------------------------------------------- /DB/ddl/user_account_segment_role_mapping.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Table structure for table `user_account_segment_role_mapping` 3 | -- 4 | CREATE TABLE IF NOT EXISTS `user_account_segment_role_mapping` ( 5 | `user_account_segment_role_mapping_id` int(11) NOT NULL AUTO_INCREMENT, 6 | `user_account_segment_mapping_id` int(11) NOT NULL, 7 | `role_id` int(11) NOT NULL, 8 | PRIMARY KEY (`user_account_segment_role_mapping_id`), 9 | KEY `action_mapping_key_idx` (`role_id`), 10 | KEY `account_segment_mapping_reference_key_idx` (`user_account_segment_mapping_id`), 11 | CONSTRAINT `roles_reference_key` FOREIGN KEY (`role_id`) REFERENCES `roles` (`role_id`) ON DELETE NO ACTION ON UPDATE NO ACTION, 12 | CONSTRAINT `user_account_segment_mapping_reference_key` FOREIGN KEY (`user_account_segment_mapping_id`) REFERENCES `user_account_segment_mapping` (`user_account_segment_mapping_id`) ON DELETE NO ACTION ON UPDATE NO ACTION 13 | ) AUTO_INCREMENT 1000000; 14 | -------------------------------------------------------------------------------- /DB/ddl/workflow.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Table structure for table `workflow` 3 | -- 4 | CREATE TABLE IF NOT EXISTS `workflow` ( 5 | `id` int(11) NOT NULL AUTO_INCREMENT, 6 | `workflow_name` varchar(45) NOT NULL, 7 | `workflow_step` varchar(45) NOT NULL, 8 | `lookup_table` varchar(45) NOT NULL, 9 | PRIMARY KEY (`id`) 10 | ); 11 | -------------------------------------------------------------------------------- /DB/dml/application_configurations.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO `application_configurations` (`id`, `config_id`, `config_value`, `is_encrypted`) VALUES 2 | (1,39,'noreply@company.com',0), 3 | (2,34,'true',0), 4 | (3,1,'true',0), 5 | (4,4,'true',0), 6 | (5,5,'true',0), 7 | (6,6,'true',0), 8 | (7,24,'true',0), 9 | (8,27,'true',0), 10 | (9,28,'true',0), 11 | (10,29,'true',0), 12 | (11,37,'true',0), 13 | (12,26,'2',0), 14 | (14,3,'true',0), 15 | (15,13,'true',0), 16 | (16,2,'true',0), 17 | (17,7,'false',0), 18 | (18,8,'true',0), 19 | (19,9,'true',0), 20 | (20,10,'true',0), 21 | (21,12,'true',0), 22 | (22,14,'true',0), 23 | (23,15,'true',0), 24 | (24,22,'true',0), 25 | (25,30,'true',0), 26 | (26,31,'false',0), 27 | (27,32,'true',0), 28 | (28,40,'true',0), 29 | (29,52,'false',0), 30 | (30,23,'true',0), 31 | (31,33,'noreply@company.com',0), 32 | (32,51,'noreply@company.com',0); 33 | -------------------------------------------------------------------------------- /DB/dml/aws_account_profile.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO `aws_account_profile` (`id`, `account_id`, `account_env`, `account_owner`) VALUES 2 | (200000, '100000000000','Prod','Socrates'), 3 | (200001, '200000000000','Prod','Plato'), 4 | (200002, '300000000000','Dev','Aristotle'); 5 | -------------------------------------------------------------------------------- /DB/dml/cluster_metrics.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO `cluster_metrics` ( 2 | `emr_id`, 3 | `rm_url`, 4 | `refresh_timestamp`, 5 | `emr_status`, 6 | `available_memory_perc`, 7 | `available_cores_perc`, 8 | `emr_name`, 9 | `total_nodes`, 10 | `containers_pending`, 11 | `apps_pending`, 12 | `apps_running`, 13 | `apps_succeeded`, 14 | `apps_failed`, 15 | `created_by`, 16 | `account`, 17 | `type`, 18 | `cluster_create_timestamp`, 19 | `segment`) VALUES 20 | ('j-123ABC1IABCX1','http://ip-00-00-000-00.us-west-2.compute.internal:8088',CURRENT_TIMESTAMP(),'WAITING',1,1,'exploratory-sales',20,0,0,0,0,0,'QuickFabric User','100000000000','exploratory','2020-01-01 15:13:32','sales'), 21 | ('j-234XYZ2HXYZY2','http://ip-11-11-111-11.us-west-2.compute.internal:8088',CURRENT_TIMESTAMP(),'WAITING',0.17,0.31,'scheduled-sales',37,1000,1,5,2,1,'QuickFabric User','100000000000','scheduled','2020-01-02 20:14:20','sales'), 22 | ('j-456QWE3JQWEZ3','http://ip-22-22-222-22.us-west-2.compute.internal:8088',CURRENT_TIMESTAMP(),'WAITING',0.23,0.40,'exploratory-sales-test1',17,0,0,0,0,0,'QuickFabric User','200000000000','exploratory','2020-01-03 08:00:17','sales'); 23 | -------------------------------------------------------------------------------- /DB/dml/configuration_data_types.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Dumping data for table `configuration_data_types` 3 | -- 4 | INSERT INTO `configuration_data_types` (`id`, `data_type_name`) VALUES 5 | (3,'boolean'), 6 | (4,'date'), 7 | (5,'datetime'), 8 | (6,'decimal'), 9 | (2,'int'), 10 | (7, 'long'), 11 | (1,'string'); 12 | -------------------------------------------------------------------------------- /DB/dml/configuration_types.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO `configuration_types` (`id`, `config_type`) VALUES 2 | (2,'Account'), 3 | (1,'Application'); -------------------------------------------------------------------------------- /DB/dml/emr_billing_component_cost.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO `emr_billing_component_cost` (`emr_name`, `emr_cost`) VALUES 2 | ('exploratory-sales',700), 3 | ('scheduled-sales',1000), 4 | ('exploratory-sales-test1',500); 5 | -------------------------------------------------------------------------------- /DB/dml/reports.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO `reports` (`report_name`) VALUES 2 | ('AMI Rotation'), 3 | ('Cluster Metrics'); 4 | -------------------------------------------------------------------------------- /DB/dml/roles.sql: -------------------------------------------------------------------------------- 1 | 2 | INSERT INTO `roles` (`role_id`, `role_name`, `service_id`) VALUES 3 | (400000,'CreateCluster',500000), 4 | (400001,'TerminateCluster',500000), 5 | (400002,'AddStep',500000), 6 | (400003,'RotateAMI',500000), 7 | (400004,'FlipDNS',500000), 8 | (400005,'CloneCluster',500000), 9 | (400006,'Admin',500000), 10 | (400007,'Read',500000), 11 | (400008,'runclusterhealthchecks',500000); 12 | -------------------------------------------------------------------------------- /DB/dml/segments.sql: -------------------------------------------------------------------------------- 1 | 2 | INSERT INTO `segments` (`segment_id`, `segment_name`, `business_owner`, `business_owner_email`) VALUES 3 | (300000, 'sales','The Boss','theboss@company.com') 4 | -------------------------------------------------------------------------------- /DB/dml/services.sql: -------------------------------------------------------------------------------- 1 | 2 | INSERT INTO `services` (`service_id`,`service_type`) VALUES 3 | (500000,'EMR'); 4 | -------------------------------------------------------------------------------- /DB/dml/user.sql: -------------------------------------------------------------------------------- 1 | 2 | INSERT INTO `user` (`user_id`, `email_id`, `first_name`, `last_name`, `creation_date`,`passcode`, `super_admin`) VALUES 3 | (1,'user@company.com','QuickFabric','User','2019-06-11 17:41:06','$2a$10$nGJMv4QsvfqBxW269wvgSepuzpJNX0IbZatlGjvEVXfsjkdDCCRsK',1); 4 | -------------------------------------------------------------------------------- /DB/dml/user_account_segment_mapping.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO user_account_segment_mapping(`user_account_segment_mapping_id`, `user_id`, `aws_account_id`, `segment_id`) VALUES 2 | (600000, 1, 200000, 300000), -- QuickFabric User to all acounts for sales segment 3 | (600001, 1, 200001, 300000), 4 | (600002, 1, 200002, 300000); 5 | -------------------------------------------------------------------------------- /DB/dml/user_account_segment_role_mapping.sql: -------------------------------------------------------------------------------- 1 | 2 | INSERT INTO user_account_segment_role_mapping(`user_account_segment_role_mapping_id`, `role_id`, `user_account_segment_mapping_id`) VALUES 3 | (1000000, 9, 1000000), -- READ access for QuickFabric User for sales segment under all accounts 4 | (1000001, 9, 1000001), 5 | (1000002, 9, 1000002); 6 | -------------------------------------------------------------------------------- /DB/dml/workflow.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO `workflow` (`id`, `workflow_name`, `workflow_step`, `lookup_table`)VALUES 2 | (1,'CreateCluster','create_new_cluster','emr_cluster_metadata'), 3 | (2,'CreateCluster','cluster_bootstraps','cluster_step_request'), 4 | (3,'CreateCluster','cluster_custom_steps','cluster_step_request'), 5 | (4,'CreateCluster','health_check','emr_functional_testsuites_status'), 6 | (5,'RotateAMI-NonHA','terminate_current_cluster','emr_cluster_metadata'), 7 | (6,'RotateAMI-NonHA','create_new_cluster','emr_cluster_metadata'), 8 | (7,'RotateAMI-NonHA','cluster_bootstraps','cluster_step_request'), 9 | (8,'RotateAMI-NonHA','cluster_custom_steps','cluster_step_request'), 10 | (9,'RotateAMI-NonHA','health_check','emr_functional_testsuites_status'), 11 | (10,'RotateAMI-HA','create_new_cluster','emr_cluster_metadata'), 12 | (11,'RotateAMI-HA','cluster_bootstraps','cluster_step_request'), 13 | (12,'RotateAMI-HA','cluster_custom_steps','cluster_step_request'), 14 | (13,'RotateAMI-HA','health_check','emr_functional_testsuites_status'), 15 | (14,'RotateAMi-HA','mark_current_cluster_for_termination','emr_cluster_metadata'); -------------------------------------------------------------------------------- /DB/patch/patch.sh: -------------------------------------------------------------------------------- 1 | # connection="mysql -w -u root -p$MYSQL_ROOT_PASSWORD -D quickfabric" 2 | # $connection 15 | `; 16 | -------------------------------------------------------------------------------- /Frontend/src/__tests__/components/admin/admin.test.jsx: -------------------------------------------------------------------------------- 1 | import { shallow } from 'enzyme'; 2 | import toJson from 'enzyme-to-json'; 3 | import React from 'react'; 4 | import { Provider } from 'react-redux'; 5 | import configureMockStore from "redux-mock-store"; 6 | 7 | import { Admin } from '../../../components'; 8 | 9 | const initialState = { 10 | uiListArray: { 11 | roles: [], 12 | accounts: [], 13 | response: { 14 | segments: [] 15 | } 16 | } 17 | } 18 | 19 | const mockStore = configureMockStore(); 20 | const store = mockStore(initialState); 21 | const component = shallow( 22 | 23 | 24 | ) 25 | 26 | describe('Testing for Admin', () => { 27 | it('Should render the component', () => { 28 | expect(toJson(component)).toMatchSnapshot(); 29 | }) 30 | }) -------------------------------------------------------------------------------- /Frontend/src/__tests__/components/emrmanagement/__snapshots__/CreateCluster.test.jsx.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`Testing Create cluster form Should render the component 1`] = ` 4 | 12 | `; 13 | -------------------------------------------------------------------------------- /Frontend/src/__tests__/components/utils/components/ToggleSlider.test.jsx: -------------------------------------------------------------------------------- 1 | import { shallow } from 'enzyme'; 2 | import toJson from 'enzyme-to-json'; 3 | import React from 'react'; 4 | import { ToggleSlider } from '../../../../utils/components/ToggleSlider'; 5 | 6 | const handleToggle = () => {console.log('test toggle')} 7 | const initialValue = { 8 | id: 0, 9 | handleToggleChange: handleToggle, 10 | toggleOn: false, 11 | isDisabled: false, 12 | toggleType: 'Encrypt' 13 | }; 14 | 15 | const component = shallow( ) 22 | 23 | describe('Testing Toggle Slider', () => { 24 | it('Should render the component', () => { 25 | expect(toJson(component)).toMatchSnapshot(); 26 | }) 27 | }) -------------------------------------------------------------------------------- /Frontend/src/__tests__/components/utils/components/__snapshots__/ToggleSlider.test.jsx.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`Testing Toggle Slider Should render the component 1`] = ` 4 |
5 |
6 | 13 | 31 |
32 |
33 | `; 34 | -------------------------------------------------------------------------------- /Frontend/src/__tests__/containers/App.test.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { shallow } from 'enzyme'; 3 | import AppContainer from 'Containers/AppContainer'; 4 | 5 | describe('App', () => { 6 | describe('component', () => { 7 | let element; 8 | beforeEach(() => { 9 | element = ; 10 | }); 11 | 12 | it('renders as expected', () => { 13 | const component = shallow(element); 14 | expect(component).toMatchSnapshot(); 15 | }); 16 | }); 17 | }); -------------------------------------------------------------------------------- /Frontend/src/__tests__/setup/setupEnzyme.js: -------------------------------------------------------------------------------- 1 | import Enzyme from 'enzyme'; 2 | import Adapter from 'enzyme-adapter-react-16'; 3 | 4 | Enzyme.configure({ adapter: new Adapter() }); -------------------------------------------------------------------------------- /Frontend/src/actions/actionTypes/EMRCostActionTypes.js: -------------------------------------------------------------------------------- 1 | const EMRCostActionTypes = { 2 | EMRCOST_STATUS_FETCHING: 'EMRCOST_STATUS_FETCHING', 3 | EMRCOST_STATUS_SUCCESS: 'EMRCOST_STATUS_SUCCESS', 4 | EMRCOST_STATUS_ERROR: 'EMRCOST_STATUS_ERROR', 5 | GET_CLUSTER_COST_MONTHLY_FETCHING:'GET_CLUSTER_COST_MONTHLY_FETCHING', 6 | GET_CLUSTER_COST_MONTHLY_SUCCESS:'GET_CLUSTER_COST_MONTHLY_SUCCESS', 7 | GET_CLUSTER_COST_MONTHLY_ERROR:'GET_CLUSTER_COST_MONTHLY_ERROR', 8 | GET_CLUSTER_COST_WEEKLY_FETCHING:'GET_CLUSTER_COST_WEEKLY_FETCHING', 9 | GET_CLUSTER_COST_WEEKLY_SUCCESS:'GET_CLUSTER_COST_WEEKLY_SUCCESS', 10 | GET_CLUSTER_COST_WEEKLY_ERROR:'GET_CLUSTER_COST_WEEKLY_ERROR', 11 | GET_CLUSTER_COST_CUSTOM_FETCHING:'GET_CLUSTER_COST_CUSTOM_FETCHING', 12 | GET_CLUSTER_COST_CUSTOM_SUCCESS:'GET_CLUSTER_COST_CUSTOM_SUCCESS', 13 | GET_CLUSTER_COST_CUSTOM_ERROR:'GET_CLUSTER_COST_CUSTOM_ERROR' 14 | }; 15 | 16 | export default EMRCostActionTypes; -------------------------------------------------------------------------------- /Frontend/src/actions/actionTypes/ProfileActionTypes.js: -------------------------------------------------------------------------------- 1 | const ProfileActionTypes = { 2 | GET_SUBSCRIPTIONS_FETCHING: 'GET_SUBSCRIPTIONS_FETCHING', 3 | GET_SUBSCRIPTIONS_ERROR: 'GET_SUBSCRIPTIONS_ERROR', 4 | GET_SUBSCRIPTIONS_SUCCESS: 'GET_SUBSCRIPTIONS_SUCCESS', 5 | UPDATE_SUBSCRIPTIONS_ERROR: 'UPDATE_SUBSCRIPTIONS_ERROR', 6 | UPDATE_SUBSCRIPTIONS_SUCCESS: 'UPDATE_SUBSCRIPTIONS_SUCCESS', 7 | CLEAR_UPDATES: 'CLEAR_UPDATES' 8 | } 9 | 10 | export default ProfileActionTypes; -------------------------------------------------------------------------------- /Frontend/src/actions/actionTypes/UserActionTypes.js: -------------------------------------------------------------------------------- 1 | const UserActionTypes = { 2 | SIGN_IN_FETCHING: 'SIGN_IN_FETCHING', 3 | SIGN_IN_SUCCESS: 'SIGN_IN_SUCCESS', 4 | SIGN_IN_ERROR: 'SIGN_IN_ERROR', 5 | SIGN_OUT: 'SIGN_OUT', 6 | CLEAR_SIGN_IN_ERROR: 'CLEAR_SIGN_IN_ERROR', 7 | AUTHENTICATE_USER_FETCHING: 'AUTHENTICATE_USER_FETCHING', 8 | AUTHENTICATE_USER_SUCCESS: 'AUTHENTICATE_USER_SUCCESS', 9 | AUTHENTICATE_USER_ERROR: 'AUTHENTICATE_USER_ERROR' 10 | }; 11 | 12 | export default UserActionTypes; -------------------------------------------------------------------------------- /Frontend/src/api-config.js: -------------------------------------------------------------------------------- 1 | let baseURL 2 | 3 | const hostname = window && window.location && window.location.hostname 4 | 5 | baseURL = 'http://'+hostname+'/quickfabric/services' 6 | 7 | export default baseURL 8 | -------------------------------------------------------------------------------- /Frontend/src/assets/help/AccountConfig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/AccountConfig.png -------------------------------------------------------------------------------- /Frontend/src/assets/help/AccountSetup.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/AccountSetup.gif -------------------------------------------------------------------------------- /Frontend/src/assets/help/Admin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/Admin.png -------------------------------------------------------------------------------- /Frontend/src/assets/help/AppConfig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/AppConfig.png -------------------------------------------------------------------------------- /Frontend/src/assets/help/ClusterCost.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/ClusterCost.png -------------------------------------------------------------------------------- /Frontend/src/assets/help/Cluster_Details_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/Cluster_Details_1.png -------------------------------------------------------------------------------- /Frontend/src/assets/help/Cluster_Details_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/Cluster_Details_2.png -------------------------------------------------------------------------------- /Frontend/src/assets/help/Cluster_Operations.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/Cluster_Operations.gif -------------------------------------------------------------------------------- /Frontend/src/assets/help/CreateClusterWorkflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/CreateClusterWorkflow.png -------------------------------------------------------------------------------- /Frontend/src/assets/help/Create_Cluster.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/Create_Cluster.gif -------------------------------------------------------------------------------- /Frontend/src/assets/help/EMRCost.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/EMRCost.png -------------------------------------------------------------------------------- /Frontend/src/assets/help/EMR_Graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/EMR_Graph.png -------------------------------------------------------------------------------- /Frontend/src/assets/help/Expert_Advice.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/Expert_Advice.png -------------------------------------------------------------------------------- /Frontend/src/assets/help/QuickFabric_ER_Diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/QuickFabric_ER_Diagram.png -------------------------------------------------------------------------------- /Frontend/src/assets/help/UserProfile.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/UserProfile.png -------------------------------------------------------------------------------- /Frontend/src/assets/help/ViewWorkflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/ViewWorkflow.png -------------------------------------------------------------------------------- /Frontend/src/assets/help/addSteps.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/addSteps.gif -------------------------------------------------------------------------------- /Frontend/src/assets/help/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/architecture.png -------------------------------------------------------------------------------- /Frontend/src/assets/help/autoPilot.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/autoPilot.gif -------------------------------------------------------------------------------- /Frontend/src/assets/help/cloneCluster.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/cloneCluster.gif -------------------------------------------------------------------------------- /Frontend/src/assets/help/cluster_details.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/cluster_details.gif -------------------------------------------------------------------------------- /Frontend/src/assets/help/dnsFlip.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/dnsFlip.gif -------------------------------------------------------------------------------- /Frontend/src/assets/help/drussotto.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/drussotto.jpg -------------------------------------------------------------------------------- /Frontend/src/assets/help/gbagdi.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/gbagdi.jpg -------------------------------------------------------------------------------- /Frontend/src/assets/help/gdoon.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/gdoon.jpg -------------------------------------------------------------------------------- /Frontend/src/assets/help/ktran7.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/ktran7.jpg -------------------------------------------------------------------------------- /Frontend/src/assets/help/kverma.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/kverma.jpg -------------------------------------------------------------------------------- /Frontend/src/assets/help/nkk.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/nkk.jpg -------------------------------------------------------------------------------- /Frontend/src/assets/help/rotateAMI.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/rotateAMI.gif -------------------------------------------------------------------------------- /Frontend/src/assets/help/suttamchandani.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/suttamchandani.jpg -------------------------------------------------------------------------------- /Frontend/src/assets/help/terminateCluster.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/terminateCluster.gif -------------------------------------------------------------------------------- /Frontend/src/assets/help/vsood.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/help/vsood.jpg -------------------------------------------------------------------------------- /Frontend/src/assets/login/data.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/login/data.jpg -------------------------------------------------------------------------------- /Frontend/src/assets/login/gitLogo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/login/gitLogo.png -------------------------------------------------------------------------------- /Frontend/src/assets/navbar/qdlogo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/assets/navbar/qdlogo.png -------------------------------------------------------------------------------- /Frontend/src/components/EMRCost/emrCost.scss: -------------------------------------------------------------------------------- 1 | .emr-cost-component { 2 | padding-top: 20px; 3 | margin-left: 90px; 4 | 5 | .button-list { 6 | padding: 10px 0px; 7 | } 8 | 9 | .pt-tab-list { 10 | margin-top: 20px; 11 | } 12 | 13 | .line { 14 | border-bottom: solid 1px #CED9E0; 15 | } 16 | 17 | .value { 18 | margin-top: -4px; 19 | margin-left: 20px; 20 | } 21 | 22 | .cell { 23 | width: 10px; 24 | height: 10px; 25 | border-radius: 50%; 26 | border: 1px solid; 27 | float: left; 28 | } 29 | 30 | .positive { 31 | background-color: #53B700; 32 | border-color: #53B700; 33 | } 34 | 35 | .negative { 36 | background-color: #DB3737; 37 | border-color: #DB3737; 38 | } 39 | 40 | } 41 | 42 | .pt-tab:focus { 43 | outline: 0; 44 | } 45 | 46 | .tab-button { 47 | background-color: none; 48 | } 49 | 50 | .exportBtn { 51 | width: 160px; 52 | height: 40px; 53 | font-size: 14px; 54 | background-color: white; 55 | border: solid #0097E6; 56 | border-width: 2px; 57 | border-radius: 20px; 58 | font-weight: 500; 59 | color: #0097E6; 60 | 61 | span { 62 | position: absolute; 63 | margin-top: 7px; 64 | margin-left: -47px; 65 | } 66 | } 67 | 68 | .material-icons { 69 | float: left 70 | } 71 | -------------------------------------------------------------------------------- /Frontend/src/components/EMRHealth/ToggleText.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | const ToggleText = props => ( 4 | {props.label} 5 | ); 6 | 7 | export default ToggleText; -------------------------------------------------------------------------------- /Frontend/src/components/EMRHealth/emrHealthCell.js: -------------------------------------------------------------------------------- 1 | import React, { Component } from 'react' 2 | 3 | export default class JobStatusCell extends Component { 4 | render() { 5 | const emr_status = this.props.dataItem.emrStatus 6 | 7 | switch (emr_status) { 8 | case 'healthy': 9 | return
{emr_status}
10 | case 'unhealthy': 11 | return
{emr_status}
12 | case 'RUNNING': 13 | return
{emr_status}
14 | case 'WAITING': 15 | return
HEALTHY
16 | default: 17 | return {emr_status} 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /Frontend/src/components/EMRManagement/AMIRotationDays.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | /** 4 | * Show status colored according to the value. 5 | */ 6 | export default class AMIRotationStatus extends React.Component { 7 | render() { 8 | const ami_rotation_days = this.props.dataItem.rotationDaysToGo; 9 | switch(true) { 10 | case (ami_rotation_days.includes('Left')): 11 | return
{ami_rotation_days}
12 | 13 | case ami_rotation_days.includes('Overdue'): 14 | return
{ami_rotation_days}
15 | 16 | default: 17 | return
{ami_rotation_days}
18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /Frontend/src/components/Help/Help.scss: -------------------------------------------------------------------------------- 1 | .help-list { 2 | background-color: #f7f7f7; 3 | padding: 45px; 4 | width: 95%; 5 | /* padding-bottom: 33%; */ 6 | border-radius: 12px; 7 | margin-bottom: 10px; 8 | } 9 | .help-list ol { 10 | color: #106ba3; 11 | font-size: 17px; 12 | } 13 | .help-list p { 14 | color: black; 15 | font-size: 17px; 16 | } 17 | .help-list-content h2 { 18 | margin-top: 50px; 19 | } 20 | .help-list-content p { 21 | font-size: 15px; 22 | } 23 | .list-1 { 24 | font-weight: 550; 25 | } 26 | .list-2 { 27 | font-weight: 500; 28 | } 29 | .help-images { 30 | margin: 20px; 31 | } 32 | .team-content { 33 | text-align: center; 34 | margin: 20px; 35 | } 36 | .help-container { 37 | padding-top: 63px; 38 | margin-left: 90px; 39 | } 40 | .help-container a { 41 | margin: 0 auto; 42 | } 43 | .subsection, .subsubsection { 44 | margin-left: 2%; 45 | } -------------------------------------------------------------------------------- /Frontend/src/components/index.js: -------------------------------------------------------------------------------- 1 | export {default as SignIn} from './SignIn' 2 | export {default as EMRCost} from './EMRCost' 3 | export {default as EMRHealth} from './EMRHealth' 4 | export {default as EMRManagement} from './EMRManagement' 5 | export {default as NavBar} from './NavBar' 6 | export {default as Profile} from './Profile' 7 | export {default as Admin} from './Admin' 8 | export {default as Help} from './Help' -------------------------------------------------------------------------------- /Frontend/src/containers/AdminContainer.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Admin } from '../components'; 3 | import { connect } from 'react-redux'; 4 | 5 | import { 6 | fetchUIDropdownList 7 | } from '../actions/emrManagement'; 8 | 9 | /** 10 | * Container for Admin Component. 11 | */ 12 | class AdminContainer extends React.Component { 13 | render() { 14 | if(!this.props.uiListFetching && this.props.uiListSuccess) { 15 | return ( 16 | 17 | ) 18 | } else { 19 | this.props.fetchUIDropdownList(this.props.token) 20 | return ( 21 |
22 | ) 23 | } 24 | 25 | } 26 | } 27 | 28 | const mapStateToProps = state => { 29 | return { 30 | uiListData: state.emrMetadataData.uiListData, 31 | uiListFetching: state.emrMetadataData.uiListFetching, 32 | uiListSuccess: state.emrMetadataData.uiListSuccess 33 | } 34 | } 35 | 36 | const mapDispatchToProps = dispatch => { 37 | return { 38 | fetchUIDropdownList: (token) => {return dispatch(fetchUIDropdownList(token))} 39 | } 40 | } 41 | 42 | export default connect(mapStateToProps, mapDispatchToProps)(AdminContainer); -------------------------------------------------------------------------------- /Frontend/src/containers/HelpContainer.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Help } from '../components'; 3 | 4 | /** 5 | * Container for Help page component. 6 | */ 7 | export default class HelpContainer extends React.Component { 8 | render() { 9 | return ( 10 |
11 | ) 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /Frontend/src/containers/ProfileContainer.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Profile } from '../components'; 3 | import { connect } from 'react-redux'; 4 | 5 | /** 6 | * Container for user profile component. 7 | */ 8 | class ProfileContainer extends React.Component { 9 | constructor(props) { 10 | super(props); 11 | } 12 | 13 | render() { 14 | return ( 15 | 16 | ) 17 | } 18 | } 19 | 20 | export default connect()(ProfileContainer); -------------------------------------------------------------------------------- /Frontend/src/containers/SignInContainer.jsx: -------------------------------------------------------------------------------- 1 | import React, { Component } from 'react' 2 | import { connect } from 'react-redux' 3 | import { withRouter } from 'react-router-dom' 4 | import { SignIn } from '../components' 5 | import { signInUser } from '../actions/user' 6 | 7 | /** 8 | * Container to sign in a user. 9 | */ 10 | class SignInContainer extends Component { 11 | constructor(props) { 12 | super(props) 13 | this.handleSignIn = this.handleSignIn.bind(this) 14 | } 15 | 16 | render() { 17 | return ( 18 |
19 | 20 |
21 | ) 22 | } 23 | 24 | handleSignIn(username, passcode, jwt) { 25 | this.props.signIn(username, passcode, jwt) 26 | } 27 | } 28 | 29 | 30 | const mapDispatchToProps = dispatch => { 31 | return { 32 | signIn: (username, passcode) => { 33 | return dispatch(signInUser(username, passcode)) 34 | } 35 | } 36 | } 37 | 38 | export default connect(null, mapDispatchToProps)(withRouter(SignInContainer)) -------------------------------------------------------------------------------- /Frontend/src/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import ReactDOM from 'react-dom' 3 | import { Provider } from 'react-redux' 4 | import { BrowserRouter } from 'react-router-dom' 5 | import store from './store/index' 6 | import AppContainer from './containers/AppContainer' 7 | import './main.css'; 8 | 9 | ReactDOM.render( 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | , 18 | document.getElementById('root') 19 | ) -------------------------------------------------------------------------------- /Frontend/src/main.css: -------------------------------------------------------------------------------- 1 | @import "~@blueprintjs/core/lib/css/blueprint.css"; 2 | @import "~@blueprintjs/icons/lib/css/blueprint-icons.css"; 3 | 4 | /* Global Styles */ 5 | html, body { 6 | margin: 0; 7 | height: 0; 8 | } 9 | 10 | body { 11 | min-width: 1280px; 12 | font-family: "Avenir", serif; 13 | font-size: 62.5%; 14 | } 15 | 16 | *:focus { 17 | outline: none !important; 18 | } 19 | 20 | /* App Container View */ 21 | .app-container-view-dash { 22 | margin-left: 160px; /* Same as the width of the sidebar */ 23 | margin-top: 50px; /* Same as the height of the navbar */ 24 | } 25 | -------------------------------------------------------------------------------- /Frontend/src/reducers/index.js: -------------------------------------------------------------------------------- 1 | import { combineReducers } from 'redux' 2 | import { user } from './user' 3 | import { allEmrHealthData } from './emrhealth' 4 | import { emrCostData } from './emrcost' 5 | import { emrMetadataData } from './emrmanagement' 6 | import { adminMetadata } from './admin' 7 | import { profileMetadata } from './profile' 8 | 9 | const rootReducer = combineReducers({ 10 | user, 11 | allEmrHealthData, 12 | emrCostData, 13 | emrMetadataData, 14 | adminMetadata, 15 | profileMetadata 16 | }) 17 | 18 | export default rootReducer -------------------------------------------------------------------------------- /Frontend/src/utils/components/CheckJiraEnabled.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Check whether Jira is enabled globally and for the account. 3 | */ 4 | export const checkJiraEnabled = (globalJira, accountJira) => { 5 | return (globalJira !== undefined && 6 | globalJira.configValue === "true") && 7 | (accountJira !== undefined && 8 | accountJira.configValue === "true") 9 | } -------------------------------------------------------------------------------- /Frontend/src/utils/components/with-state.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { process } from '@progress/kendo-data-query'; 3 | import './../../components/EMRHealth/emrHealth1.scss' 4 | 5 | export function withState(WrappedGrid) { 6 | return class StatefulGrid extends React.Component { 7 | constructor(props) { 8 | super(props); 9 | if (props.pageable === false) { 10 | this.state = {}; 11 | } else { 12 | this.state = { skip: 0, take: 10 }; 13 | } 14 | } 15 | 16 | render() { 17 | return ( 18 | { this.setState(e.data); }} 27 | /> 28 | ); 29 | } 30 | }; 31 | } -------------------------------------------------------------------------------- /Frontend/src/utils/styles/shared.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/QuickFabric/a7200cfdde42a033fbed04384028833b8b100a6f/Frontend/src/utils/styles/shared.css -------------------------------------------------------------------------------- /Frontend/src/utils/utils/convertStrToBoolean.js: -------------------------------------------------------------------------------- 1 | export const converStrToBoolean = (val) => { 2 | if(typeof val === 'string') { 3 | if(val === 'False' || val === 'false') { 4 | return false 5 | } else if(val === 'True' || val === 'true') { 6 | return true 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /Middleware/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store/ 2 | .classpath/ 3 | .project/ 4 | 5 | # See https://help.github.com/ignore-files/ for more about ignoring files. 6 | 7 | .vscode 8 | 9 | # Mac-specific 10 | .DS_Store 11 | emr/target 12 | schedulers/target 13 | /commons/commons.iml 14 | /dcc-application.iml 15 | /dcc-application.ipr 16 | /dcc-application.iws 17 | /emr/emr.iml 18 | /.ideaDataSources/ 19 | /dataSources/ 20 | /logs/ 21 | /emr/logs/ 22 | -------------------------------------------------------------------------------- /Middleware/README.md: -------------------------------------------------------------------------------- 1 | # Quickfabric Middleware Local Build 2 | --- 3 | 4 | # To build Quickfabric Middleware locally 5 | 6 | Requirements 7 | ------------ 8 | 9 | - [Java](https://www.oracle.com/technetwork/java/javase/overview/java8-2100321.html) 1.8+ 10 | - [mvn](https://maven.apache.org/download.cgi) 3.6+ 11 | 12 | ## Build 13 | 14 | ### Without Unit Tests 15 | 16 | ```bash 17 | mvn clean install -DskipTests 18 | ``` 19 | 20 | ### With Unit Tests 21 | 22 | ```bash 23 | mvn clean install 24 | ``` 25 | 26 | Corresponding Jar files for commons, emr and schedulers can be found under respective target folders. 27 | 28 | You can use a local tomcat or deploy these jars into a host running tomcat to consume these jars. -------------------------------------------------------------------------------- /Middleware/commons/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | -------------------------------------------------------------------------------- /Middleware/commons/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | commons 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.wst.common.project.facet.core.builder 10 | 11 | 12 | 13 | 14 | org.eclipse.jdt.core.javabuilder 15 | 16 | 17 | 18 | 19 | org.eclipse.wst.validation.validationbuilder 20 | 21 | 22 | 23 | 24 | org.eclipse.m2e.core.maven2Builder 25 | 26 | 27 | 28 | 29 | 30 | org.eclipse.jem.workbench.JavaEMFNature 31 | org.eclipse.wst.common.modulecore.ModuleCoreNature 32 | org.eclipse.jdt.core.javanature 33 | org.eclipse.m2e.core.maven2Nature 34 | org.eclipse.wst.common.project.facet.core.nature 35 | 36 | 37 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/constants/HealthCheckConstants.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.constants; 2 | 3 | public class HealthCheckConstants { 4 | 5 | public static final String NoOfBootstraps = "NoOfBootstraps"; 6 | 7 | public static final String NoOfDefaultSteps = "NoOfDefaultSteps"; 8 | 9 | public static final String Connectivity = "Connectivity"; 10 | 11 | public static final String AutoScaling ="AutoScaling"; 12 | } 13 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/constants/Roles.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.constants; 2 | 3 | public class Roles { 4 | public static final String AMI_ROTATION = "rotateami"; 5 | public static final String CREATE_CLUSTER = "createcluster"; 6 | public static final String TERMINATE_CLUSTER = "terminatecluster"; 7 | public static final String ADD_STEP = "addstep"; 8 | public static final String ADMIN = "admin"; 9 | public static final String SUPER_ADMIN = "superadmin"; 10 | public static final String FLIP_DNS = "flipdns"; 11 | } 12 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/constants/WorkflowConstants.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.constants; 2 | 3 | public class WorkflowConstants { 4 | public static final String CREATE_CLUSTER = "CREATECLUSTER"; 5 | public static final String ROTATE_AMI_NON_HA = "RotateAMI-NonHA"; 6 | public static final String ROTATE_AMI_HA = "RotateAMI-HA"; 7 | public static final String ROTATE_AMI = "ROTATEAMI"; 8 | } 9 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/dao/ConfigDao.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.dao; 2 | 3 | import java.util.List; 4 | 5 | import com.intuit.quickfabric.commons.vo.ConfigVO; 6 | import com.intuit.quickfabric.commons.vo.ConfigDefinitionVO; 7 | 8 | public interface ConfigDao { 9 | 10 | List getAllConfigurationDefinitions(); 11 | 12 | ConfigDefinitionVO getConfigDefinition(String configName); 13 | 14 | List getAccountConfigs(String accountId); 15 | 16 | void addAccountConfig(int configId, String accountId, String value, boolean isEncrypted); 17 | 18 | void updateAccountConfig(int configId, String accountId, String value, boolean isEncrypted); 19 | 20 | List getApplicationConfigs(); 21 | 22 | void addApplicationConfig(int configId, String value, boolean isEncrypted); 23 | 24 | void updateApplicationConfig(int configId, String value, boolean isEncrypted); 25 | } 26 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/dao/LoginRolesDao.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.dao; 2 | 3 | import java.util.List; 4 | 5 | import org.springframework.stereotype.Component; 6 | 7 | import com.intuit.quickfabric.commons.domain.UserAccess; 8 | import com.intuit.quickfabric.commons.vo.LoginRolesVO; 9 | 10 | @Component 11 | public interface LoginRolesDao { 12 | 13 | LoginRolesVO getUserByEmail(String emailID); 14 | 15 | List getUserAccessList(String email); 16 | 17 | void resetPassword(int userId, String newEncryptedPassword) ; 18 | } 19 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/domain/Role.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.domain; 2 | 3 | public class Role { 4 | 5 | private int roleId; 6 | private String roleName; 7 | private int serviceId; 8 | 9 | public Role(int roleId, String roleName, int serviceId) { 10 | 11 | this.roleId = roleId; 12 | this.roleName = roleName; 13 | this.serviceId = serviceId; 14 | } 15 | 16 | public int getRoleId() { 17 | return roleId; 18 | } 19 | 20 | public void setRoleId(int roleId) { 21 | this.roleId = roleId; 22 | } 23 | 24 | public String getRoleName() { 25 | return roleName; 26 | } 27 | 28 | public void setRoleName(String roleName) { 29 | this.roleName = roleName; 30 | } 31 | 32 | public int getServiceId() { 33 | return serviceId; 34 | } 35 | 36 | public void setServiceId(int serviceId) { 37 | this.serviceId = serviceId; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/exceptions/QuickFabricBaseException.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.exceptions; 2 | 3 | import org.springframework.http.HttpStatus; 4 | 5 | public abstract class QuickFabricBaseException extends RuntimeException { 6 | 7 | public abstract HttpStatus getHttpStatusCode(); 8 | 9 | public QuickFabricBaseException(String message) { 10 | super(message); 11 | } 12 | 13 | public QuickFabricBaseException(Throwable e) { 14 | super(e); 15 | } 16 | 17 | public QuickFabricBaseException(String message, Throwable cause) { 18 | super(message, cause); 19 | } 20 | } 21 | 22 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/exceptions/QuickFabricClientException.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.exceptions; 2 | 3 | import org.springframework.http.HttpStatus; 4 | 5 | public class QuickFabricClientException extends QuickFabricBaseException { 6 | 7 | public QuickFabricClientException(String message) { 8 | super(message); 9 | } 10 | 11 | public QuickFabricClientException(String message, Throwable cause) { 12 | super(message, cause); 13 | } 14 | 15 | @Override 16 | public HttpStatus getHttpStatusCode() { 17 | return HttpStatus.BAD_REQUEST; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/exceptions/QuickFabricJsonException.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.exceptions; 2 | 3 | import org.json.JSONException; 4 | import org.springframework.http.HttpStatus; 5 | 6 | public class QuickFabricJsonException extends QuickFabricBaseException { 7 | 8 | public QuickFabricJsonException(Exception e) { 9 | this("Error happen during json manipulation", e); 10 | } 11 | 12 | public QuickFabricJsonException(String message, Exception e) { 13 | super(message, e); 14 | } 15 | 16 | @Override 17 | public HttpStatus getHttpStatusCode() { 18 | return HttpStatus.INTERNAL_SERVER_ERROR; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/exceptions/QuickFabricRestHandlerException.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.exceptions; 2 | 3 | import org.springframework.http.HttpStatus; 4 | 5 | public class QuickFabricRestHandlerException extends QuickFabricBaseException { 6 | 7 | public QuickFabricRestHandlerException(String message) { 8 | super(message); 9 | } 10 | 11 | public QuickFabricRestHandlerException(Throwable e) { 12 | super(e); 13 | } 14 | 15 | public QuickFabricRestHandlerException(String message, Throwable cause) { 16 | super(message, cause); 17 | } 18 | 19 | @Override 20 | public HttpStatus getHttpStatusCode() { 21 | return HttpStatus.INTERNAL_SERVER_ERROR; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/exceptions/QuickFabricSQLException.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.exceptions; 2 | 3 | import org.springframework.http.HttpStatus; 4 | 5 | public class QuickFabricSQLException extends QuickFabricBaseException { 6 | 7 | public QuickFabricSQLException(String message) { 8 | super(message); 9 | } 10 | 11 | public QuickFabricSQLException(String message, Throwable cause) { 12 | super(message, cause); 13 | } 14 | 15 | public QuickFabricSQLException(Exception e) { 16 | super(e); 17 | } 18 | 19 | @Override 20 | public HttpStatus getHttpStatusCode() { 21 | return HttpStatus.INTERNAL_SERVER_ERROR; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/exceptions/QuickFabricServerException.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.exceptions; 2 | 3 | import org.springframework.http.HttpStatus; 4 | 5 | public class QuickFabricServerException extends QuickFabricBaseException { 6 | 7 | public QuickFabricServerException(String message) { 8 | super(message); 9 | } 10 | 11 | public QuickFabricServerException(String message, Throwable cause) { 12 | super(message, cause); 13 | } 14 | 15 | public QuickFabricServerException(Exception e) { 16 | super(e); 17 | } 18 | 19 | @Override 20 | public HttpStatus getHttpStatusCode() { 21 | return HttpStatus.INTERNAL_SERVER_ERROR; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/exceptions/QuickFabricUnauthenticatedException.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.exceptions; 2 | 3 | import org.springframework.http.HttpStatus; 4 | 5 | public class QuickFabricUnauthenticatedException extends QuickFabricBaseException { 6 | public QuickFabricUnauthenticatedException(String message) { 7 | super(message); 8 | } 9 | 10 | public QuickFabricUnauthenticatedException(Throwable e) { 11 | super(e); 12 | } 13 | 14 | public QuickFabricUnauthenticatedException(String message, Throwable cause) { 15 | super(message, cause); 16 | } 17 | 18 | @Override 19 | public HttpStatus getHttpStatusCode() { 20 | return HttpStatus.UNAUTHORIZED; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/exceptions/QuickFabricUnauthorizedException.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.exceptions; 2 | 3 | import org.springframework.http.HttpStatus; 4 | 5 | public class QuickFabricUnauthorizedException extends QuickFabricBaseException { 6 | 7 | public QuickFabricUnauthorizedException(){ 8 | this("Access is denied."); 9 | } 10 | 11 | public QuickFabricUnauthorizedException(String message) { 12 | super(message); 13 | } 14 | 15 | public QuickFabricUnauthorizedException(String message, Throwable cause) { 16 | super(message, cause); 17 | } 18 | 19 | @Override 20 | public HttpStatus getHttpStatusCode() { 21 | return HttpStatus.FORBIDDEN; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/mapper/LoginRolesMapper.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.mapper; 2 | 3 | import org.springframework.dao.DataAccessException; 4 | import org.springframework.jdbc.core.ResultSetExtractor; 5 | 6 | import com.intuit.quickfabric.commons.vo.LoginRolesVO; 7 | 8 | import java.sql.ResultSet; 9 | import java.sql.SQLException; 10 | 11 | public class LoginRolesMapper implements ResultSetExtractor { 12 | 13 | 14 | public LoginRolesVO extractData(ResultSet rs) throws SQLException, 15 | DataAccessException { 16 | 17 | LoginRolesVO loginRolesVO = null; 18 | while (rs.next()) { 19 | loginRolesVO = new LoginRolesVO(); 20 | loginRolesVO.setEmailId(rs.getString("email_id")); 21 | loginRolesVO.setFirstName(rs.getString("first_name")); 22 | loginRolesVO.setLastName(rs.getString("last_name")); 23 | loginRolesVO.setCreationDate(rs.getString("creation_date")); 24 | loginRolesVO.setPasscode(rs.getString("passcode")); 25 | loginRolesVO.setSuperAdmin(rs.getBoolean("super_admin")); 26 | loginRolesVO.setUserId(rs.getInt("user_id")); 27 | } 28 | return loginRolesVO; 29 | } 30 | } -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/security/JwtAuthenticationSuccessHandler.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.security; 2 | 3 | import javax.servlet.http.HttpServletRequest; 4 | import javax.servlet.http.HttpServletResponse; 5 | 6 | import org.springframework.security.core.Authentication; 7 | import org.springframework.security.web.authentication.AuthenticationSuccessHandler; 8 | 9 | public class JwtAuthenticationSuccessHandler implements AuthenticationSuccessHandler { 10 | 11 | @Override 12 | public void onAuthenticationSuccess(HttpServletRequest request, HttpServletResponse response, Authentication authentication) { 13 | // We do not need to do anything extra on REST authentication success, because there is no page to redirect to 14 | } 15 | 16 | } -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/security/JwtTokenFilterConfigurer.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.security; 2 | 3 | import org.springframework.security.config.annotation.SecurityConfigurerAdapter; 4 | import org.springframework.security.config.annotation.web.builders.HttpSecurity; 5 | import org.springframework.security.web.DefaultSecurityFilterChain; 6 | import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; 7 | 8 | import com.intuit.quickfabric.commons.security.JWTTokenProvider; 9 | import com.intuit.quickfabric.commons.security.JwtTokenFilter; 10 | 11 | public class JwtTokenFilterConfigurer extends SecurityConfigurerAdapter { 12 | 13 | private JWTTokenProvider jwtTokenProvider; 14 | 15 | public JwtTokenFilterConfigurer(JWTTokenProvider jwtTokenProvider) { 16 | this.jwtTokenProvider = jwtTokenProvider; 17 | } 18 | 19 | @Override 20 | public void configure(HttpSecurity http) throws Exception { 21 | JwtTokenFilter customFilter = new JwtTokenFilter(jwtTokenProvider); 22 | http.addFilterBefore(customFilter, UsernamePasswordAuthenticationFilter.class); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/security/RestAuthenticationEntryPoint.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.security; 2 | 3 | import java.io.IOException; 4 | 5 | import javax.servlet.http.HttpServletRequest; 6 | import javax.servlet.http.HttpServletResponse; 7 | 8 | import org.springframework.security.core.AuthenticationException; 9 | import org.springframework.security.web.AuthenticationEntryPoint; 10 | 11 | public class RestAuthenticationEntryPoint implements AuthenticationEntryPoint { 12 | 13 | @Override 14 | public void commence(HttpServletRequest request, HttpServletResponse response, AuthenticationException authException) throws IOException { 15 | // This is invoked when user tries to access a secured REST resource without supplying any credentials 16 | // We should just send a 401 Unauthorized response because there is no 'login page' to redirect to 17 | response.sendError(HttpServletResponse.SC_UNAUTHORIZED, "Unauthorized"); 18 | } 19 | } 20 | 21 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/utils/EnableMethod.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.utils; 2 | 3 | import java.lang.annotation.Retention; 4 | import java.lang.annotation.Target; 5 | import java.lang.annotation.RetentionPolicy; 6 | import java.lang.annotation.ElementType; 7 | 8 | @Retention(RetentionPolicy.RUNTIME) 9 | @Target(ElementType.METHOD) 10 | public @interface EnableMethod { 11 | 12 | public String configName(); 13 | } 14 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/utils/RestTemplateResponseErrorHandler.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.utils; 2 | 3 | import org.apache.logging.log4j.LogManager; 4 | import org.apache.logging.log4j.Logger; 5 | import org.springframework.http.HttpStatus; 6 | import org.springframework.http.client.ClientHttpResponse; 7 | import org.springframework.stereotype.Component; 8 | import org.springframework.web.client.ResponseErrorHandler; 9 | 10 | import java.io.IOException; 11 | 12 | import static org.springframework.http.HttpStatus.Series.CLIENT_ERROR; 13 | 14 | @Component 15 | public class RestTemplateResponseErrorHandler implements ResponseErrorHandler { 16 | 17 | private final Logger logger = LogManager.getLogger(RestTemplateResponseErrorHandler.class); 18 | 19 | @Override 20 | public boolean hasError(ClientHttpResponse httpResponse) throws IOException { 21 | logger.info("hasError httpResponse:" + httpResponse); 22 | return httpResponse.getStatusCode().series() == CLIENT_ERROR 23 | || httpResponse.getStatusCode().series() == HttpStatus.Series.SERVER_ERROR; 24 | } 25 | 26 | @Override 27 | public void handleError(ClientHttpResponse httpResponse) throws IOException { 28 | logger.info("handleError httpResponse:" + httpResponse); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/ApiErrorVO.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | import java.util.UUID; 4 | 5 | public class ApiErrorVO { 6 | 7 | private String errorMessage; 8 | 9 | private String messageDetails; 10 | 11 | private String errorId; 12 | 13 | public ApiErrorVO(String errorMessage) { 14 | this.errorMessage = errorMessage; 15 | errorId = UUID.randomUUID().toString(); 16 | } 17 | 18 | public ApiErrorVO(String errorMessage, String messageDetails) { 19 | this.errorMessage = errorMessage; 20 | this.messageDetails = messageDetails; 21 | errorId = UUID.randomUUID().toString(); 22 | } 23 | 24 | public String getErrorMessage() { 25 | return errorMessage; 26 | } 27 | 28 | public void setErrorMessage(String errorMessage) { 29 | this.errorMessage = errorMessage; 30 | } 31 | 32 | public String getMessageDetails() { 33 | return messageDetails; 34 | } 35 | 36 | public void setMessageDetails(String messageDetails) { 37 | this.messageDetails = messageDetails; 38 | } 39 | 40 | public String getErrorId() { 41 | return errorId; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/AppSeverity.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | public enum AppSeverity { 4 | NONE("NONE"), 5 | LOW("LOW"), 6 | MODERATE("MODERATE"), 7 | SEVERE("SEVERE"), 8 | CRITICAL("CRITICAL"); 9 | 10 | private final String value; 11 | 12 | AppSeverity(String severity) { 13 | this.value = severity; 14 | } 15 | 16 | public String toString() { 17 | return this.value; 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/AppStatus.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | 5 | public enum AppStatus { 6 | RUNNING("RUNNING"), 7 | SUCCEEDED("SUCCEEDED"), 8 | FAILED("FAILED"); 9 | 10 | // the value used to match json node string value with an enum constant 11 | @JsonProperty("status") 12 | private final String value; 13 | 14 | AppStatus(final String type) { 15 | value = type; 16 | } 17 | 18 | @Override 19 | public String toString() { 20 | return value; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/AwsAccountProfile.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | public class AwsAccountProfile { 4 | 5 | private String accountId; 6 | 7 | private String accountEnv; 8 | 9 | private String accountOwner; 10 | 11 | public String getAccountId() { 12 | return accountId; 13 | } 14 | 15 | public void setAccountId(String accountId) { 16 | this.accountId = accountId; 17 | } 18 | 19 | public String getAccountEnv() { 20 | return accountEnv; 21 | } 22 | 23 | public void setAccountEnv(String accountEnv) { 24 | this.accountEnv = accountEnv; 25 | } 26 | 27 | public String getAccountOwner() { 28 | return accountOwner; 29 | } 30 | 31 | public void setAccountOwner(String accountOwner) { 32 | this.accountOwner = accountOwner; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/BootstrapActionVO.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | public class BootstrapActionVO { 4 | 5 | private String bootstrapName; 6 | 7 | private String bootstrapScript; 8 | 9 | public String getBootstrapName() { 10 | return bootstrapName; 11 | } 12 | 13 | public void setBootstrapName(String bootstrapName) { 14 | this.bootstrapName = bootstrapName; 15 | } 16 | 17 | public String getBootstrapScript() { 18 | return bootstrapScript; 19 | } 20 | 21 | public void setBootstrapScript(String bootstrapScript) { 22 | this.bootstrapScript = bootstrapScript; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/BootstrapParametersRequest.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | public class BootstrapParametersRequest{ 4 | 5 | private String dnsName; 6 | private String headlessUser; 7 | 8 | public String getDnsName() { 9 | return dnsName; 10 | } 11 | 12 | public void setDnsName(String dnsName) { 13 | this.dnsName = dnsName; 14 | } 15 | 16 | public String getHeadlessUser() { 17 | return headlessUser; 18 | } 19 | 20 | public void setHeadlessUser(String headlessUser) { 21 | this.headlessUser = headlessUser; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/BootstrapTestResponse.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | public class BootstrapTestResponse { 4 | 5 | private String statusCode; 6 | private String result; 7 | private Integer bootstrapCount; 8 | 9 | public Integer getBootstrapCount() { 10 | return bootstrapCount; 11 | } 12 | 13 | public void setBootstrapCount(Integer bootstrapCount) { 14 | this.bootstrapCount = bootstrapCount; 15 | } 16 | 17 | public String getStatusCode() { 18 | return statusCode; 19 | } 20 | 21 | public void setStatusCode(String statusCode) { 22 | this.statusCode = statusCode; 23 | } 24 | 25 | public String getResult() { 26 | return result; 27 | } 28 | 29 | public void setResult(String result) { 30 | this.result = result; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/ClusterError.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | public class ClusterError { 4 | String errorType; 5 | String errorMessage; 6 | 7 | 8 | public String getErrorType() { 9 | return errorType; 10 | } 11 | public void setErrorType(String errorType) { 12 | this.errorType = errorType; 13 | } 14 | public String getErrorMessage() { 15 | return errorMessage; 16 | } 17 | public void setErrorMessage(String errorMessage) { 18 | this.errorMessage = errorMessage; 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/ClusterHealthCheckStatusType.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | public enum ClusterHealthCheckStatusType { 4 | NEW, 5 | INITIATED, 6 | INPROGRESS, 7 | FAILED, 8 | SUCCESS 9 | } 10 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/ClusterHealthCheckStatusUpdate.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | import javax.validation.constraints.Min; 4 | import javax.validation.constraints.NotNull; 5 | import javax.validation.constraints.Size; 6 | 7 | public class ClusterHealthCheckStatusUpdate { 8 | 9 | @NotNull(message = "execution Id is not valid") 10 | @Min(1) 11 | private int executionId; 12 | 13 | @NotNull(message = "status cannot be empty") 14 | @Size(min = 1) 15 | private String status; 16 | 17 | private String message; 18 | 19 | public String getMessage() { 20 | return message; 21 | } 22 | 23 | public void setMessage(String message) { 24 | this.message = message; 25 | } 26 | 27 | public int getExecutionId() { 28 | return executionId; 29 | } 30 | 31 | public void setExecutionId(int executionId) { 32 | this.executionId = executionId; 33 | } 34 | 35 | public String getStatus() { 36 | return status; 37 | } 38 | 39 | public void setStatus(String status) { 40 | this.status = status; 41 | } 42 | 43 | @Override 44 | public String toString() { 45 | return "ClusterHealthCheckStatusUpdate executionId:" + executionId + " status:" + status; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/ClusterMessageResponseVO.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | public class ClusterMessageResponseVO { 4 | private String clusterId; 5 | 6 | private String clusterName; 7 | 8 | private String message; 9 | 10 | public String getClusterId() { 11 | return clusterId; 12 | } 13 | 14 | public void setClusterId(String clusterId) { 15 | this.clusterId = clusterId; 16 | } 17 | 18 | public String getClusterName() { 19 | return clusterName; 20 | } 21 | 22 | public void setClusterName(String clusterName) { 23 | this.clusterName = clusterName; 24 | } 25 | 26 | public String getMessage() { 27 | return message; 28 | } 29 | 30 | public void setMessage(String message) { 31 | this.message = message; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/ClusterStatus.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | import com.fasterxml.jackson.annotation.JsonInclude; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | @JsonInclude(JsonInclude.Include.NON_EMPTY) 6 | 7 | public enum ClusterStatus { 8 | // Defining constants like this so we can use this enum to convert string 9 | // values from JSON to enum constants 10 | STARTING("STARTING"), 11 | INITIATED("INITIATED"), 12 | FAILED("FAILED"), 13 | RUNNING("RUNNING"), 14 | BOOTSTRAPPING("BOOTSTRAPPING"), 15 | TERMINATION_INITIATED("TERMINATION_INITIATED"), 16 | TERMINATED("TERMINATED"), 17 | TERMINATING("TERMINATING"), 18 | ClusterAlreadyExists("ClusterAlreadyExists"), 19 | ClusterNotPresent("ClusterNotPresent"), 20 | HEALTHY("HEALTHY"), 21 | unhealthy("unhealthy"), 22 | WAITING("WAITING"), 23 | Completed("Completed"), 24 | TERMINATED_WITH_ERRORS("TERMINATED_WITH_ERRORS"); 25 | 26 | // the value used to match json node string value with an enum constant 27 | @JsonProperty("status") 28 | private final String value; 29 | 30 | ClusterStatus(final String type) { 31 | value = type; 32 | } 33 | 34 | @Override 35 | public String toString() { 36 | return value; 37 | } 38 | } -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/ClusterType.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | 5 | public enum ClusterType { 6 | @JsonProperty("exploratory") 7 | EXPLORATORY("exploratory"), 8 | @JsonProperty("scheduled") 9 | SCHEDULED("scheduled"), 10 | @JsonProperty("transient") 11 | TRANSIENT("transient"); 12 | 13 | // the value which is used for matching 14 | // the json node value with this enum 15 | @JsonProperty("type") 16 | private final String value; 17 | 18 | 19 | public String getValue() { 20 | return value; 21 | } 22 | 23 | ClusterType(final String type) 24 | { 25 | value = type; 26 | } 27 | 28 | @Override 29 | public String toString() { 30 | return value; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/ConfigDataType.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | public enum ConfigDataType { 4 | String, 5 | Int, 6 | Boolean, 7 | Date, 8 | DateTime, 9 | Long, 10 | Decimal 11 | } 12 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/ConfigType.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | public enum ConfigType { 4 | Application, 5 | Account 6 | } 7 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/HadoopJarStep.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | public class HadoopJarStep { 7 | 8 | private String mainClass=""; 9 | 10 | private String jar; 11 | 12 | private List stepArgs = new ArrayList(); 13 | 14 | public String getMainClass() { 15 | return mainClass; 16 | } 17 | 18 | public void setMainClass(String mainClass) { 19 | this.mainClass = mainClass; 20 | } 21 | 22 | public String getJar() { 23 | return jar; 24 | } 25 | 26 | public void setJar(String jar) { 27 | this.jar = jar; 28 | } 29 | 30 | public List getStepArgs() { 31 | return stepArgs; 32 | } 33 | 34 | public void setStepArgs(List stepArgs) { 35 | this.stepArgs = stepArgs; 36 | } 37 | 38 | 39 | } 40 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/JobSchedulingAdviceVO.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | import java.util.List; 4 | 5 | import org.springframework.stereotype.Component; 6 | import com.fasterxml.jackson.annotation.JsonInclude; 7 | 8 | @Component 9 | @JsonInclude(JsonInclude.Include.NON_NULL) 10 | public class JobSchedulingAdviceVO { 11 | private String advice; 12 | private List leastUsed; 13 | private List mostUsed; 14 | 15 | 16 | public String getAdvice() { 17 | return advice; 18 | } 19 | 20 | public void setAdvice(String advice) { 21 | this.advice = advice; 22 | } 23 | 24 | public List getLeastUsed() { 25 | return leastUsed; 26 | } 27 | 28 | public void setLeastUsed(List leastUsed) { 29 | this.leastUsed = leastUsed; 30 | } 31 | 32 | public List getMostUsed() { 33 | return mostUsed; 34 | } 35 | 36 | public void setMostUsed(List mostUsed) { 37 | this.mostUsed = mostUsed; 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/LoginRequest.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | public class LoginRequest { 4 | 5 | private String emailId; 6 | 7 | private String passcode; 8 | 9 | public String getEmailId() { 10 | return emailId; 11 | } 12 | 13 | public void setEmailId(String emailId) { 14 | this.emailId = emailId; 15 | } 16 | 17 | public String getPasscode() { 18 | return passcode; 19 | } 20 | 21 | public void setPasscode(String passcode) { 22 | this.passcode = passcode; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/MonthlyCostVO.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | public class MonthlyCostVO { 4 | private String billMonth; 5 | private long cost; 6 | 7 | public String getBillMonth() { 8 | return billMonth; 9 | } 10 | 11 | public void setBillMonth(String billMonth) { 12 | this.billMonth = billMonth; 13 | } 14 | 15 | public long getCost() { 16 | return cost; 17 | } 18 | 19 | public void setCost(long cost) { 20 | this.cost = cost; 21 | } 22 | 23 | } -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/ResetPasswordRequest.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | import javax.validation.constraints.Email; 4 | import javax.validation.constraints.NotBlank; 5 | import javax.validation.constraints.NotNull; 6 | import javax.validation.constraints.Size; 7 | 8 | public class ResetPasswordRequest { 9 | 10 | @NotNull 11 | @Size(min = 1) 12 | @Email 13 | private String email; 14 | 15 | @NotBlank 16 | @Size(min = 5) 17 | private String newPassword; 18 | 19 | public String getEmail() { 20 | return email; 21 | } 22 | 23 | public void setEmail(String email) { 24 | this.email = email; 25 | } 26 | 27 | public String getNewPassword() { 28 | return newPassword; 29 | } 30 | 31 | public void setNewPassword(String newPassword) { 32 | this.newPassword = newPassword.trim(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/ServiceType.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | public enum ServiceType { 4 | 5 | EMR, 6 | SCHEDULERS; 7 | } 8 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/ServiceVO.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnore; 4 | import com.fasterxml.jackson.annotation.JsonInclude; 5 | 6 | import java.util.List; 7 | 8 | @JsonInclude(JsonInclude.Include.NON_EMPTY) 9 | public class ServiceVO { 10 | 11 | @JsonIgnore 12 | private int id; 13 | 14 | private ServiceType serviceType; 15 | 16 | private List roles; 17 | 18 | public int getId() { 19 | return id; 20 | } 21 | 22 | public void setId(int id) { 23 | this.id = id; 24 | } 25 | 26 | public ServiceType getServiceType() { 27 | return serviceType; 28 | } 29 | 30 | public void setServiceType(ServiceType serviceType) { 31 | this.serviceType = serviceType; 32 | } 33 | 34 | public List getRoles() { 35 | return roles; 36 | } 37 | 38 | public void setRoles(List roles) { 39 | this.roles = roles; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/StepStatus.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | 5 | public enum StepStatus { 6 | 7 | NEW("NEW"), 8 | INTIATED("INTIATED"), 9 | VALIDATED("VALIDATED"), 10 | COMPLETED("COMPLETED"), 11 | PENDING("PENDING"), 12 | FAILED("FAILED"), 13 | CANCELLED("CANCELLED"), 14 | RUNNING("RUNNING"), 15 | TERMINATED("TERMINATED"), 16 | INTERRUPTED("INTERRUPTED"), 17 | CANCEL_PENDING("CANCEL_PENDING"), 18 | COMPLETED_TERMINATEDCLUSTER("COMPLETED_TERMINATEDCLUSTER"); 19 | @JsonProperty("status") 20 | private final String value; 21 | 22 | StepStatus(final String type) { 23 | value = type; 24 | } 25 | 26 | @Override 27 | public String toString() { 28 | return value; 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/StepsValidateRequest.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import com.fasterxml.jackson.annotation.JsonInclude; 7 | @JsonInclude(JsonInclude.Include.NON_NULL) 8 | public class StepsValidateRequest { 9 | String clusterId; 10 | String clusterName; 11 | private List stepIds= new ArrayList(); 12 | 13 | public StepsValidateRequest() { 14 | 15 | } 16 | 17 | public StepsValidateRequest(String clusterName, String clusterId, List asList) { 18 | this.clusterId=clusterId; 19 | this.clusterName=clusterName; 20 | this.stepIds=asList; 21 | } 22 | public String getClusterId() { 23 | return clusterId; 24 | } 25 | public void setClusterId(String clusterId) { 26 | this.clusterId = clusterId; 27 | } 28 | public String getClusterName() { 29 | return clusterName; 30 | } 31 | public void setClusterName(String clusterName) { 32 | this.clusterName = clusterName; 33 | } 34 | public List getStepIds() { 35 | return stepIds; 36 | } 37 | public void setStepIds(List stepids) { 38 | this.stepIds = stepids; 39 | } 40 | 41 | 42 | } 43 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/SubscriptionVO.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | 4 | 5 | public class SubscriptionVO { 6 | 7 | private long reportId; 8 | private String reportName; 9 | 10 | private boolean subscribed; 11 | 12 | public long getReportId() { 13 | return reportId; 14 | } 15 | 16 | public void setReportId(long reportId) { 17 | this.reportId = reportId; 18 | } 19 | 20 | public String getReportName() { 21 | return reportName; 22 | } 23 | 24 | public void setReportName(String reportName) { 25 | this.reportName = reportName; 26 | } 27 | 28 | public boolean isSubscribed() { 29 | return subscribed; 30 | } 31 | 32 | public void setSubscribed(boolean subscribed) { 33 | this.subscribed = subscribed; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/UserAccountSegmentMapping.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | public class UserAccountSegmentMapping { 4 | private int mappingId; 5 | private int segmentId; 6 | private int userId; 7 | private int awsAccountId; 8 | 9 | public void setMappingId(int mappingId) { 10 | this.mappingId = mappingId; 11 | } 12 | 13 | public int getMappingId() { 14 | return mappingId; 15 | } 16 | 17 | public void setSegmentId(int segmentId) { 18 | this.segmentId = segmentId; 19 | } 20 | 21 | public int getSegmentId() { 22 | return segmentId; 23 | } 24 | 25 | public void setUserId(int userId) { 26 | this.userId = userId; 27 | } 28 | 29 | public int getUserId() { 30 | return userId; 31 | } 32 | 33 | public void setAwsAccountId(int awsAccountId) { 34 | this.awsAccountId = awsAccountId; 35 | } 36 | 37 | public int getAwsAccountId() { 38 | return awsAccountId; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/UserRole.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnore; 4 | 5 | import java.util.List; 6 | 7 | public class UserRole { 8 | 9 | @JsonIgnore 10 | private int id; 11 | 12 | private String name; 13 | 14 | private List segments; 15 | 16 | public int getId() { 17 | return id; 18 | } 19 | 20 | public void setId(int id) { 21 | this.id = id; 22 | } 23 | 24 | public String getName() { 25 | return name; 26 | } 27 | 28 | public void setName(String name) { 29 | this.name = name; 30 | } 31 | 32 | public List getSegments() { 33 | return segments; 34 | } 35 | 36 | public void setSegments(List segments) { 37 | this.segments = segments; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/UserVO.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | public class UserVO { 4 | private long userId; 5 | private String userEmail; 6 | private String firstName; 7 | private String lastName; 8 | 9 | public long getUserId() { 10 | return userId; 11 | } 12 | 13 | public void setUserId(long userId) { 14 | this.userId = userId; 15 | } 16 | 17 | public String getUserEmail() { 18 | return userEmail; 19 | } 20 | 21 | public void setUserEmail(String userEmail) { 22 | this.userEmail = userEmail; 23 | } 24 | 25 | public String getFirstName() { 26 | return firstName; 27 | } 28 | 29 | public void setFirstName(String firstName) { 30 | this.firstName = firstName; 31 | } 32 | 33 | public String getLastName() { 34 | return lastName; 35 | } 36 | 37 | public void setLastName(String lastName) { 38 | this.lastName = lastName; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/WorkflowStatus.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | public enum WorkflowStatus { 4 | NEW, 5 | IN_PROGRESS, 6 | FAILED, 7 | SUCCESS 8 | } 9 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/WorkflowType.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | public enum WorkflowType { 4 | CREATE_NEW_CLUSTER, 5 | CLUSTER_BOOTSTRAPS, 6 | CLUSTER_CUSTOM_STEPS, 7 | HEALTH_CHECK, 8 | TERMINATE_CURRENT_CLUSTER, 9 | MARK_CURRENT_CLUSTER_FOR_TERMINATION 10 | } 11 | -------------------------------------------------------------------------------- /Middleware/commons/src/main/java/com/intuit/quickfabric/commons/vo/YarnAppHeuristicVO.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.commons.vo; 2 | 3 | 4 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 5 | import com.fasterxml.jackson.annotation.JsonInclude; 6 | 7 | @JsonIgnoreProperties(ignoreUnknown = true) 8 | @JsonInclude(JsonInclude.Include.NON_EMPTY) 9 | public class YarnAppHeuristicVO { 10 | private String heuristicName; 11 | private String severity; 12 | private long score; 13 | private String advice; 14 | 15 | public String getHeuristicName() { 16 | return heuristicName; 17 | } 18 | 19 | public void setHeuristicName(String heuristicName) { 20 | this.heuristicName = heuristicName; 21 | } 22 | public String getSeverity() { 23 | return severity; 24 | } 25 | public void setSeverity(String severity) { 26 | this.severity = severity; 27 | } 28 | public long getScore() { 29 | return score; 30 | } 31 | public void setScore(long score) { 32 | this.score = score; 33 | } 34 | 35 | public String getAdvice() { 36 | return advice; 37 | } 38 | 39 | public void setAdvice(String advice) { 40 | this.advice = advice; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /Middleware/emr/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | emr 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.wst.common.project.facet.core.builder 10 | 11 | 12 | 13 | 14 | org.eclipse.jdt.core.javabuilder 15 | 16 | 17 | 18 | 19 | org.eclipse.wst.validation.validationbuilder 20 | 21 | 22 | 23 | 24 | org.eclipse.m2e.core.maven2Builder 25 | 26 | 27 | 28 | 29 | 30 | org.eclipse.jem.workbench.JavaEMFNature 31 | org.eclipse.wst.common.modulecore.ModuleCoreNature 32 | org.eclipse.jdt.core.javanature 33 | org.eclipse.m2e.core.maven2Nature 34 | org.eclipse.wst.common.project.facet.core.nature 35 | org.eclipse.wst.jsdt.core.jsNature 36 | 37 | 38 | -------------------------------------------------------------------------------- /Middleware/emr/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-jdk-alpine 2 | ARG JAR_FILE=target/*.jar 3 | COPY ${JAR_FILE} emr.jar 4 | ENTRYPOINT ["java","-jar","/emr.jar"] 5 | EXPOSE 8080 6 | -------------------------------------------------------------------------------- /Middleware/emr/src/main/java/com/intuit/quickfabric/emr/EMRMainApplication.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.emr; 2 | 3 | import org.apache.logging.log4j.LogManager; 4 | import org.apache.logging.log4j.Logger; 5 | import org.springframework.boot.SpringApplication; 6 | import org.springframework.boot.autoconfigure.SpringBootApplication; 7 | import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration; 8 | import org.springframework.boot.builder.SpringApplicationBuilder; 9 | import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; 10 | import org.springframework.retry.annotation.EnableRetry; 11 | 12 | @SpringBootApplication(exclude = {SecurityAutoConfiguration.class }, scanBasePackages = {"com.intuit.quickfabric"}) 13 | @EnableRetry 14 | public class EMRMainApplication extends SpringBootServletInitializer 15 | { 16 | private static final Logger logger = LogManager.getLogger(EMRMainApplication.class); 17 | 18 | public static void main (String args[]) 19 | { 20 | SpringApplication.run(EMRMainApplication.class,args); 21 | logger.info("----Application Started----"); 22 | 23 | } 24 | 25 | @Override 26 | protected SpringApplicationBuilder configure(SpringApplicationBuilder application) { 27 | 28 | return application; 29 | } 30 | } -------------------------------------------------------------------------------- /Middleware/emr/src/main/java/com/intuit/quickfabric/emr/dao/EMRClusterStepsDao.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.emr.dao; 2 | 3 | import com.intuit.quickfabric.commons.vo.BootstrapActionVO; 4 | import com.intuit.quickfabric.commons.vo.ClusterStep; 5 | import com.intuit.quickfabric.commons.vo.ClusterVO; 6 | import com.intuit.quickfabric.commons.vo.StepResponseVO; 7 | 8 | import java.util.List; 9 | 10 | 11 | public interface EMRClusterStepsDao { 12 | 13 | void saveStepRequestForCluster(ClusterVO clusterDetails); 14 | 15 | void saveStepRequestForCluster(String clusterName, ClusterStep step); 16 | 17 | StepResponseVO getStepsOfACluster(String clusterId); 18 | 19 | void updateStepIdsInDB(StepResponseVO stepResponse, List steps); 20 | 21 | List getStepsByStepIds(List stepIds); 22 | 23 | void saveBootstrapActionRequestForCluster(ClusterVO clusterDetails); 24 | 25 | List getBootstrapActionsByClusterId(String clusterId); 26 | 27 | } 28 | -------------------------------------------------------------------------------- /Middleware/emr/src/main/java/com/intuit/quickfabric/emr/dao/EmailReportSubscriptionsDao.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.emr.dao; 2 | 3 | import com.intuit.quickfabric.commons.vo.SubscriptionVO; 4 | 5 | import java.util.List; 6 | 7 | public interface EmailReportSubscriptionsDao { 8 | 9 | List getSubscriptionsForUser(String userEmail); 10 | 11 | void updateSubscriptionsForUser(List reports, String email); 12 | 13 | } 14 | -------------------------------------------------------------------------------- /Middleware/emr/src/main/java/com/intuit/quickfabric/emr/dao/WorkflowDao.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.emr.dao; 2 | 3 | import com.intuit.quickfabric.commons.vo.Workflow; 4 | 5 | public interface WorkflowDao { 6 | 7 | Workflow getWorkflow(String name); 8 | } 9 | -------------------------------------------------------------------------------- /Middleware/emr/src/main/java/com/intuit/quickfabric/emr/helper/EmailReportSubscriptionsHelper.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.emr.helper; 2 | 3 | import com.intuit.quickfabric.commons.vo.SubscriptionVO; 4 | import com.intuit.quickfabric.emr.dao.EmailReportSubscriptionsDao; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.stereotype.Component; 7 | 8 | import java.util.List; 9 | import java.util.stream.Collectors; 10 | 11 | @Component 12 | public class EmailReportSubscriptionsHelper { 13 | 14 | @Autowired 15 | private EmailReportSubscriptionsDao emailReportSubscriptionsDao; 16 | 17 | public List getSubscriptionsForUser(String email) { 18 | return this.emailReportSubscriptionsDao.getSubscriptionsForUser(email); 19 | } 20 | 21 | public List updateSubscriptions(List subscriptions, String email) { 22 | 23 | //report ids for reports the user wants to subscribe to 24 | List reportsToSubscribe = subscriptions.stream() 25 | .filter(s -> s.isSubscribed()) 26 | .map(s -> s.getReportId()).collect(Collectors.toList()); 27 | 28 | this.emailReportSubscriptionsDao.updateSubscriptionsForUser(reportsToSubscribe, email); 29 | return getSubscriptionsForUser(email); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /Middleware/emr/src/main/java/com/intuit/quickfabric/emr/mapper/AWSAccountProfileMapper.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.emr.mapper; 2 | 3 | import java.sql.ResultSet; 4 | import java.sql.SQLException; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | import org.springframework.dao.DataAccessException; 9 | import org.springframework.jdbc.core.ResultSetExtractor; 10 | 11 | import com.intuit.quickfabric.commons.vo.AwsAccountProfile; 12 | 13 | 14 | public class AWSAccountProfileMapper implements ResultSetExtractor> { 15 | 16 | public List extractData(ResultSet rs) throws SQLException, DataAccessException { 17 | 18 | List lst = new ArrayList<>(); 19 | 20 | while(rs.next()) { 21 | AwsAccountProfile account = new AwsAccountProfile(); 22 | account.setAccountId(rs.getString("account_id")); 23 | account.setAccountOwner(rs.getString("account_owner")); 24 | account.setAccountEnv(rs.getString("account_env")); 25 | lst.add(account); 26 | } 27 | 28 | return lst; 29 | 30 | 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /Middleware/emr/src/main/java/com/intuit/quickfabric/emr/mapper/BootstrapActionMapper.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.emr.mapper; 2 | 3 | import java.sql.ResultSet; 4 | import java.sql.SQLException; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | import org.springframework.dao.DataAccessException; 9 | import org.springframework.jdbc.core.ResultSetExtractor; 10 | 11 | import com.intuit.quickfabric.commons.vo.BootstrapActionVO; 12 | 13 | public class BootstrapActionMapper implements ResultSetExtractor> { 14 | public List extractData(ResultSet rs) throws SQLException, 15 | DataAccessException { 16 | 17 | List vo = new ArrayList(); 18 | while (rs.next()) { 19 | 20 | BootstrapActionVO bootstrapAction= new BootstrapActionVO(); 21 | 22 | bootstrapAction.setBootstrapName(rs.getString("name")); 23 | bootstrapAction.setBootstrapScript(rs.getString("step_arg")); 24 | vo.add(bootstrapAction); 25 | 26 | } 27 | 28 | return vo; 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /Middleware/emr/src/main/java/com/intuit/quickfabric/emr/mapper/EMRClusterMetricsRmUrlMapper.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.emr.mapper; 2 | 3 | import java.sql.ResultSet; 4 | import java.sql.SQLException; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | import org.springframework.dao.DataAccessException; 9 | import org.springframework.jdbc.core.ResultSetExtractor; 10 | 11 | import com.intuit.quickfabric.commons.vo.EMRClusterMetricsVO; 12 | 13 | public class EMRClusterMetricsRmUrlMapper implements ResultSetExtractor> { 14 | 15 | public List extractData(ResultSet rs) throws SQLException, 16 | DataAccessException { 17 | 18 | List eMRClusterMetricsList = new ArrayList(); 19 | EMRClusterMetricsVO vo = null; 20 | while (rs.next()) { 21 | vo = new EMRClusterMetricsVO(); 22 | vo.setEmrId(rs.getString("emr_id")); 23 | vo.setRmUrl(rs.getString("rm_url")); 24 | vo.setEmrName(rs.getString("emr_name")); 25 | vo.setAccount(rs.getString("account")); 26 | 27 | eMRClusterMetricsList.add(vo); 28 | } 29 | 30 | return eMRClusterMetricsList; 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /Middleware/emr/src/main/java/com/intuit/quickfabric/emr/mapper/SegmentsMapper.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.emr.mapper; 2 | 3 | import org.springframework.dao.DataAccessException; 4 | import org.springframework.jdbc.core.ResultSetExtractor; 5 | 6 | import com.intuit.quickfabric.commons.vo.SegmentVO; 7 | 8 | import java.sql.ResultSet; 9 | import java.sql.SQLException; 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | 13 | public class SegmentsMapper implements ResultSetExtractor> { 14 | 15 | public List extractData(ResultSet rs) throws SQLException, DataAccessException { 16 | 17 | List segments = new ArrayList<>(); 18 | while (rs.next()) { 19 | SegmentVO segment = new SegmentVO(); 20 | segment.setSegmentId(rs.getInt("segment_id")); 21 | segment.setSegmentName(rs.getString("segment_name")); 22 | segment.setBusinessOwner(rs.getString("business_owner")); 23 | segment.setBusinessOwnerEmail(rs.getString("business_owner_email")); 24 | segments.add(segment); 25 | } 26 | return segments; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /Middleware/emr/src/main/java/com/intuit/quickfabric/emr/mapper/WorkflowMapper.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.emr.mapper; 2 | 3 | import org.springframework.dao.DataAccessException; 4 | import org.springframework.jdbc.core.ResultSetExtractor; 5 | 6 | import com.intuit.quickfabric.commons.vo.Workflow; 7 | import com.intuit.quickfabric.commons.vo.WorkflowStep; 8 | import com.intuit.quickfabric.commons.vo.WorkflowType; 9 | 10 | import java.sql.ResultSet; 11 | import java.sql.SQLException; 12 | 13 | 14 | public class WorkflowMapper implements ResultSetExtractor { 15 | 16 | @Override 17 | public Workflow extractData(ResultSet resultSet) throws SQLException, DataAccessException { 18 | Workflow workflow = new Workflow(); 19 | 20 | while (resultSet.next()) { 21 | workflow.setName(resultSet.getString("workflow_name")); 22 | WorkflowStep workflowStep = new WorkflowStep(); 23 | workflowStep.setWorkflowType(WorkflowType.valueOf(resultSet.getString("workflow_step").toUpperCase())); 24 | workflowStep.setTableName(resultSet.getString("lookup_table")); 25 | workflow.addStep(workflowStep); 26 | } 27 | 28 | return workflow; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /Middleware/emr/src/main/java/com/intuit/quickfabric/emr/model/EMRAppsModel.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.emr.model; 2 | 3 | import java.util.List; 4 | 5 | import com.intuit.quickfabric.commons.vo.EMRAppVO; 6 | 7 | public class EMRAppsModel { 8 | 9 | private String emrId; 10 | private String emrName; 11 | private List apps; 12 | 13 | public List getApps() { 14 | return apps; 15 | } 16 | 17 | public void setApps(List apps) { 18 | this.apps = apps; 19 | } 20 | 21 | public String getEmrId() { 22 | return emrId; 23 | } 24 | 25 | public void setEmrId(String emrId) { 26 | this.emrId = emrId; 27 | } 28 | 29 | public String getEmrName() { 30 | return emrName; 31 | } 32 | 33 | public void setEmrName(String emrName) { 34 | this.emrName = emrName; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /Middleware/emr/src/main/java/com/intuit/quickfabric/emr/model/EMRClusterCostModel.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.emr.model; 2 | import java.util.List; 3 | 4 | import com.fasterxml.jackson.annotation.JsonInclude; 5 | import com.intuit.quickfabric.commons.vo.EMRGroupCostVO; 6 | import com.intuit.quickfabric.commons.vo.EMRTimeSeriesReportVO; 7 | 8 | @JsonInclude(JsonInclude.Include.NON_NULL) 9 | public class EMRClusterCostModel { 10 | private String clusterId; 11 | private List clusterCost; 12 | private List emrGroupCost; 13 | 14 | public String getClusterId() { 15 | return clusterId; 16 | } 17 | 18 | public void setClusterId(String clusterId) { 19 | this.clusterId = clusterId; 20 | } 21 | 22 | public List getClusterCost() { 23 | return clusterCost; 24 | } 25 | 26 | public void setClusterCost(List clusterCost) { 27 | this.clusterCost = clusterCost; 28 | } 29 | 30 | public List getEmrGroupCost() { 31 | return emrGroupCost; 32 | } 33 | 34 | public void setEmrGroupCost(List emrGroupCost) { 35 | this.emrGroupCost = emrGroupCost; 36 | } 37 | 38 | 39 | } -------------------------------------------------------------------------------- /Middleware/emr/src/main/java/com/intuit/quickfabric/emr/model/EMRClusterMetadataModel.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.emr.model; 2 | 3 | import java.util.List; 4 | 5 | import org.springframework.stereotype.Component; 6 | 7 | import com.intuit.quickfabric.commons.vo.ClusterVO; 8 | 9 | @Component 10 | public class EMRClusterMetadataModel { 11 | 12 | private List emrClusterMetadataReport; 13 | 14 | public List getEmrClusterMetadataReport() { 15 | return emrClusterMetadataReport; 16 | } 17 | 18 | public void setEmrClusterMetadataReport(List emrClusterMetadataReport) { 19 | this.emrClusterMetadataReport = emrClusterMetadataReport; 20 | } 21 | } -------------------------------------------------------------------------------- /Middleware/emr/src/main/java/com/intuit/quickfabric/emr/model/EMRClusterMetricsModel.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.emr.model; 2 | 3 | import java.util.List; 4 | 5 | import org.springframework.stereotype.Component; 6 | 7 | import com.intuit.quickfabric.commons.vo.EMRClusterMetricsVO; 8 | 9 | @Component 10 | public class EMRClusterMetricsModel { 11 | 12 | private List emrClusterMetricsReport; 13 | 14 | public List getEmrClusterMetricsReport() { 15 | return emrClusterMetricsReport; 16 | } 17 | 18 | public void setEmrClusterMetricsReport(List emrClusterMetricsReport) { 19 | this.emrClusterMetricsReport = emrClusterMetricsReport; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /Middleware/emr/src/main/java/com/intuit/quickfabric/emr/model/UIPopulationModel.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.emr.model; 2 | 3 | import java.util.List; 4 | 5 | import org.springframework.stereotype.Component; 6 | 7 | import com.fasterxml.jackson.annotation.JsonInclude; 8 | import com.intuit.quickfabric.commons.vo.AwsAccountProfile; 9 | import com.intuit.quickfabric.commons.vo.SegmentVO; 10 | 11 | @Component 12 | @JsonInclude(JsonInclude.Include.NON_ABSENT) 13 | public class UIPopulationModel { 14 | private List segments; 15 | private List accounts; 16 | private List actions; 17 | 18 | 19 | public List getSegments() { 20 | return segments; 21 | } 22 | 23 | public void setSegments(List segments) { 24 | this.segments = segments; 25 | } 26 | 27 | public List getAccounts() { 28 | return accounts; 29 | } 30 | 31 | public void setAccounts(List accounts) { 32 | this.accounts = accounts; 33 | } 34 | 35 | public List getActions() { 36 | return actions; 37 | } 38 | 39 | public void setActions(List actions) { 40 | this.actions = actions; 41 | } 42 | 43 | 44 | } 45 | -------------------------------------------------------------------------------- /Middleware/emr/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.application.name=emr 2 | 3 | spring.datasource.url=jdbc:mysql://db:3306/quickfabric?allowPublicKeyRetrieval=true&useSSL=false&useLegacyDatetimeCode=false 4 | spring.datasource.username=qf_admin 5 | spring.datasource.password=${MYSQL_PASSWORD} 6 | 7 | server.servlet.context-path=/quickfabric/services 8 | 9 | logging.level.root=INFO 10 | logging.level.org.springframework.web=ERROR 11 | logging.level.org.hibernate=ERROR 12 | 13 | management.endpoints.web.exposure.include=* 14 | management.endpoint.shutdown.enabled=false 15 | management.endpoint.health.show-details=always 16 | 17 | aesencryptor.secret=${AES_SECRET_KEY} 18 | -------------------------------------------------------------------------------- /Middleware/emr/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{yyyy-MM-dd HH:mm:ss.SSS} %5p ${hostName} --- tid:%-5.5T [%15.15t] %-40.40c{1.} : %m%n%ex 5 | 6 | ./logs 7 | 8 | 9 | 10 | 11 | 12 | 13 | 16 | 17 | ${LOG_PATTERN} 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /Middleware/emr/src/test/java/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.application.name=emr 2 | server.servlet.context-path=/quickfabric/services 3 | spring.datasource.url=jdbc:mysql://nourl 4 | spring.datasource.username=fake_user 5 | spring.datasource.password=fake_password 6 | aesencryptor.secret=fake_key 7 | -------------------------------------------------------------------------------- /Middleware/pom.xml: -------------------------------------------------------------------------------- 1 | 4 | 4.0.0 5 | com.intuit 6 | quickfabric-application 7 | 0.0.1-SNAPSHOT 8 | 9 | commons 10 | emr 11 | schedulers 12 | 13 | pom 14 | 15 | 16 | 17 | org.springframework.boot 18 | spring-boot-starter-parent 19 | 2.2.0.RELEASE 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /Middleware/schedulers/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /schedulers.iml 3 | -------------------------------------------------------------------------------- /Middleware/schedulers/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | schedulers 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.wst.common.project.facet.core.builder 10 | 11 | 12 | 13 | 14 | org.eclipse.jdt.core.javabuilder 15 | 16 | 17 | 18 | 19 | org.eclipse.wst.validation.validationbuilder 20 | 21 | 22 | 23 | 24 | org.eclipse.m2e.core.maven2Builder 25 | 26 | 27 | 28 | 29 | 30 | org.eclipse.jem.workbench.JavaEMFNature 31 | org.eclipse.wst.common.modulecore.ModuleCoreNature 32 | org.eclipse.jdt.core.javanature 33 | org.eclipse.m2e.core.maven2Nature 34 | org.eclipse.wst.common.project.facet.core.nature 35 | org.eclipse.wst.jsdt.core.jsNature 36 | 37 | 38 | -------------------------------------------------------------------------------- /Middleware/schedulers/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-jdk-alpine 2 | ARG JAR_FILE=target/*.jar 3 | COPY ${JAR_FILE} scheduler.jar 4 | ENTRYPOINT ["java","-jar","/scheduler.jar"] 5 | -------------------------------------------------------------------------------- /Middleware/schedulers/src/main/java/com/intuit/quickfabric/schedulers/dao/ClusterHealthCheckDao.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.schedulers.dao; 2 | 3 | import com.intuit.quickfabric.commons.vo.ClusterHealthCheckStatusType; 4 | import com.intuit.quickfabric.commons.vo.ClusterHealthStatus; 5 | import com.intuit.quickfabric.commons.vo.EMRClusterHealthTestCase; 6 | 7 | import java.util.List; 8 | 9 | public interface ClusterHealthCheckDao { 10 | List getEMRClusterHealthStatus(String clusterId); 11 | 12 | List getNewHealthTestCases(); 13 | 14 | List getFunctionalTestSuites(String testName); 15 | 16 | void updateTestCaseStatus(int executionId, ClusterHealthCheckStatusType status, String remark); 17 | 18 | List getEMRClusterTestSuites(String clusterRole, String clusterType); 19 | 20 | void createHealthCheckTestCases(List testCases, String clusterId, String clusterName, String executedBy); 21 | 22 | } 23 | -------------------------------------------------------------------------------- /Middleware/schedulers/src/main/java/com/intuit/quickfabric/schedulers/dao/EMRClusterMetricsDao.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.schedulers.dao; 2 | 3 | import com.intuit.quickfabric.commons.vo.*; 4 | 5 | import java.sql.SQLException; 6 | import java.sql.Timestamp; 7 | import java.util.List; 8 | 9 | public interface EMRClusterMetricsDao { 10 | 11 | void updateEMRClusterMetricsList(List metrics); 12 | 13 | void updateEMRClusterMetricsCost(List costs); 14 | 15 | public List getDistinctEMRBillingComponent(); 16 | 17 | List getClusterMetricsReport(Timestamp from, Timestamp to, String segmentName); 18 | 19 | } 20 | -------------------------------------------------------------------------------- /Middleware/schedulers/src/main/java/com/intuit/quickfabric/schedulers/dao/EMRClusterStepsDao.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.schedulers.dao; 2 | 3 | import com.intuit.quickfabric.commons.vo.*; 4 | 5 | import java.util.List; 6 | import java.util.Set; 7 | 8 | public interface EMRClusterStepsDao { 9 | 10 | void updateStepIdsInDB(StepResponseVO stepResponse, List steps); 11 | 12 | List getStepsForNewSucceededClusters(Set clusterstatuses, 13 | List stepStatuses); 14 | 15 | public void updateBootstrapActionStatus(ClusterVO clusterDetails, String bootstrapStatus); 16 | 17 | public void saveBootstrapActionRequestForCluster(ClusterVO clusterDetails); 18 | 19 | List getBootstrapActionsByClusterId(String clusterId); 20 | 21 | } 22 | -------------------------------------------------------------------------------- /Middleware/schedulers/src/main/java/com/intuit/quickfabric/schedulers/helpers/EMRClusterMetadataHelper.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.schedulers.helpers; 2 | 3 | import com.intuit.quickfabric.commons.vo.ClusterVO; 4 | import com.intuit.quickfabric.schedulers.dao.EMRClusterMetadataDao; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.stereotype.Component; 7 | 8 | @Component 9 | public class EMRClusterMetadataHelper { 10 | 11 | @Autowired 12 | EMRClusterMetadataDao emrClusterMetadataDao; 13 | 14 | public ClusterVO getAllEMRClusterDetailsByClusterID(String clusterId) { 15 | return emrClusterMetadataDao.getAllEMRClusterDataForAMI(clusterId); 16 | } 17 | } -------------------------------------------------------------------------------- /Middleware/schedulers/src/main/java/com/intuit/quickfabric/schedulers/mappers/AddUserRequestMapper.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.schedulers.mappers; 2 | 3 | import java.sql.ResultSet; 4 | import java.sql.SQLException; 5 | 6 | import org.springframework.dao.DataAccessException; 7 | import org.springframework.jdbc.core.ResultSetExtractor; 8 | 9 | import com.intuit.quickfabric.commons.vo.ClusterVO; 10 | 11 | 12 | public class AddUserRequestMapper implements ResultSetExtractor { 13 | 14 | @Override 15 | public ClusterVO extractData(ResultSet rs) throws SQLException, DataAccessException { 16 | if(rs.next()) { 17 | ClusterVO clusterMetadata = new ClusterVO(); 18 | clusterMetadata.setClusterId(rs.getString("cluster_id")); 19 | clusterMetadata.setClusterName(rs.getString("cluster_name")); 20 | clusterMetadata.setAccount(rs.getString("account")); 21 | 22 | return clusterMetadata; 23 | } else { 24 | return null; 25 | } 26 | 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /Middleware/schedulers/src/main/java/com/intuit/quickfabric/schedulers/mappers/BootstrapActionMapper.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.schedulers.mappers; 2 | 3 | import java.sql.ResultSet; 4 | import java.sql.SQLException; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | import org.springframework.dao.DataAccessException; 9 | import org.springframework.jdbc.core.ResultSetExtractor; 10 | 11 | import com.intuit.quickfabric.commons.vo.BootstrapActionVO; 12 | 13 | public class BootstrapActionMapper implements ResultSetExtractor> { 14 | public List extractData(ResultSet rs) throws SQLException, 15 | DataAccessException { 16 | 17 | List vo = new ArrayList(); 18 | while (rs.next()) { 19 | 20 | BootstrapActionVO bootstrapAction= new BootstrapActionVO(); 21 | 22 | bootstrapAction.setBootstrapName(rs.getString("name")); 23 | bootstrapAction.setBootstrapScript(rs.getString("step_arg")); 24 | vo.add(bootstrapAction); 25 | 26 | } 27 | 28 | return vo; 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /Middleware/schedulers/src/main/java/com/intuit/quickfabric/schedulers/mappers/EMRClusterDetailsMapper.java: -------------------------------------------------------------------------------- 1 | package com.intuit.quickfabric.schedulers.mappers; 2 | 3 | import org.springframework.dao.DataAccessException; 4 | import org.springframework.jdbc.core.ResultSetExtractor; 5 | 6 | import com.intuit.quickfabric.commons.vo.ClusterRequest; 7 | 8 | import java.sql.ResultSet; 9 | import java.sql.SQLException; 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | 13 | public class EMRClusterDetailsMapper implements ResultSetExtractor> { 14 | 15 | public List extractData(ResultSet rs) throws SQLException, DataAccessException { 16 | 17 | List reqList= new ArrayList(); 18 | 19 | while (rs.next()) { 20 | ClusterRequest req= new ClusterRequest(); 21 | req.setAccount(rs.getString("account")); 22 | req.setClusterId(rs.getString("cluster_id")); 23 | req.setClusterName(rs.getString("cluster_name")); 24 | req.setCreationTimestamp(rs.getTimestamp("creation_request_timestamp").toString()); 25 | 26 | if(!reqList.contains(req)){ 27 | reqList.add(req); 28 | } 29 | } 30 | 31 | return reqList; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /Middleware/schedulers/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{yyyy-MM-dd HH:mm:ss.SSS} %5p ${hostName} --- tid:%-5.5T [%15.15t] %-40.40c{1.} : %m%n%ex 5 | 6 | ./logs 7 | 8 | 9 | 10 | 11 | 12 | 13 | 16 | 17 | ${LOG_PATTERN} 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /terraform/.gitignore: -------------------------------------------------------------------------------- 1 | **/.terraform/* 2 | backend.tpl 3 | *.lock.info 4 | *.tf.bk 5 | *.tfstate 6 | *.tfstate.* 7 | .DS_Store 8 | crash.log 9 | input.tfvars 10 | terraform.tfstate 11 | terraform.tfstate.backup 12 | terraform.tfstate.d 13 | terraform_wrapper_* 14 | -------------------------------------------------------------------------------- /terraform/input.tfvars.basic.example: -------------------------------------------------------------------------------- 1 | 2 | # Account Ids for QuickFabric 3 | account_ids = { 4 | "parent_account_id" : "", 5 | "child_account_ids" : [] 6 | } 7 | 8 | # AWS Region to deploy QuickFabric 9 | region = "us-east-1" 10 | 11 | -------------------------------------------------------------------------------- /terraform/modules/container/main.tf: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Start a container 4 | resource "docker_container" "container" { 5 | name = var.container_name 6 | image = var.image 7 | working_dir = var.working_dir 8 | command = var.command 9 | 10 | attach = var.attach 11 | rm = var.rm 12 | must_run = var.must_run 13 | 14 | env = var.env 15 | 16 | dynamic "mounts" { 17 | for_each = var.mounts 18 | content { 19 | target = mounts.key 20 | source = mounts.value 21 | type = "bind" 22 | } 23 | } 24 | dynamic "ports" { 25 | for_each = var.ports 26 | content { 27 | internal = ports.key 28 | external = ports.value 29 | } 30 | } 31 | dynamic "networks_advanced" { 32 | for_each = var.network 33 | content { 34 | name = networks_advanced.value 35 | } 36 | } 37 | 38 | } 39 | 40 | output "exit_code" { 41 | value = docker_container.container.exit_code 42 | } 43 | -------------------------------------------------------------------------------- /terraform/modules/container/variables.tf: -------------------------------------------------------------------------------- 1 | variable "container_name" {} 2 | variable "data_store" {} 3 | variable "command" { default = [] } 4 | variable "working_dir" { default = "/" } 5 | 6 | variable "image" {} 7 | variable "attach" { default = false } 8 | variable "rm" { default = false } 9 | variable "must_run" { default = false } 10 | variable "mounts" { default = {} } 11 | 12 | variable "env" { default = [] } 13 | variable "network" { default = {} } 14 | variable "ports" { default = {} } 15 | -------------------------------------------------------------------------------- /terraform/modules/ec2/files/instance_policy.txt: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Action": [ 7 | "s3:Get*", 8 | "s3:List*" 9 | ], 10 | "Resource": "*" 11 | }, 12 | { 13 | "Effect": "Allow", 14 | "Action": [ 15 | "ses:SendEmail", 16 | "ses:SendRawEmail" 17 | ], 18 | "Resource": "*" 19 | } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /terraform/modules/ec2/files/instance_role.txt: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Action": "sts:AssumeRole", 6 | "Principal": { 7 | "Service": "ec2.amazonaws.com" 8 | }, 9 | "Effect": "Allow", 10 | "Sid": "" 11 | } 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /terraform/modules/ec2/output.tf: -------------------------------------------------------------------------------- 1 | output "server_ip" { 2 | 3 | value = var.provision && length(aws_eip.eip) > 0 ? aws_eip.eip[0].public_ip : "" 4 | } 5 | output "private_ip" { 6 | 7 | value = var.provision && length(aws_instance.qf_server) > 0 ? aws_instance.qf_server[0].private_ip : "" 8 | } 9 | -------------------------------------------------------------------------------- /terraform/modules/ec2/variables.tf: -------------------------------------------------------------------------------- 1 | variable "region" {} 2 | variable "provision" { default = true } 3 | variable "user_data" { default = "" } 4 | variable "server_name" {} 5 | variable "sg_ids" { type = list } 6 | variable "device_mount_path" { default = "/dev/sdh" } 7 | variable "ebs" { type = bool } 8 | variable "instance_type" {} 9 | variable "subnet_id" {} 10 | variable "ami_id" {} 11 | variable "vpc_id" {} 12 | variable "cidr_admin_whitelist" { default = [] } 13 | variable "key_pair" {} 14 | variable "tags" {} 15 | -------------------------------------------------------------------------------- /terraform/modules/igw/main.tf: -------------------------------------------------------------------------------- 1 | 2 | locals { 3 | common_tags = { 4 | Name = "quickfabric-internet-gateway" 5 | } 6 | } 7 | 8 | # Creates Internet Gateway if no Subnets are provided 9 | resource "aws_internet_gateway" "main" { 10 | count = length(var.input_subnet_id) == 0 ? 1 : 0 11 | vpc_id = var.vpc_id 12 | tags = merge(local.common_tags, var.tags) 13 | } 14 | 15 | # Creates Route table if no Subnets are provided 16 | resource "aws_route_table" "public" { 17 | 18 | count = length(var.input_subnet_id) == 0 ? 1 : 0 19 | vpc_id = var.vpc_id 20 | 21 | route { 22 | cidr_block = "0.0.0.0/0" 23 | gateway_id = aws_internet_gateway.main[0].id 24 | } 25 | tags = merge(local.common_tags, var.tags) 26 | } 27 | 28 | # Creates Route table association rules if no Subnets are provided 29 | resource "aws_route_table_association" "public" { 30 | 31 | 32 | count = length(var.input_subnet_id) == 0 ? 1 : 0 33 | subnet_id = element(var.public_subnet_id, 0) 34 | route_table_id = aws_route_table.public[0].id 35 | } 36 | 37 | -------------------------------------------------------------------------------- /terraform/modules/igw/variables.tf: -------------------------------------------------------------------------------- 1 | variable "input_subnet_id" {} 2 | variable "public_subnet_id" {} 3 | variable "vpc_id" {} 4 | variable "tags" {} 5 | -------------------------------------------------------------------------------- /terraform/modules/nat/main.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | common_tags = { 3 | Name = "quickfabric-nat-gateway" 4 | } 5 | } 6 | 7 | # Creates a Elastic IP address 8 | resource "aws_eip" "nat" { 9 | 10 | count = length(var.input_subnet_id) == 0 ? 1 : 0 11 | vpc = true 12 | tags = merge(local.common_tags, var.tags) 13 | } 14 | 15 | resource "aws_nat_gateway" "main" { 16 | count = length(var.input_subnet_id) == 0 ? 1 : 0 17 | 18 | allocation_id = aws_eip.nat[0].id 19 | subnet_id = element(var.public_subnet_id, 0) 20 | tags = merge(local.common_tags, var.tags) 21 | } 22 | 23 | 24 | # Creates Route table if no Subnets are provided 25 | resource "aws_route_table" "private" { 26 | 27 | count = length(var.input_subnet_id) == 0 ? 1 : 0 28 | vpc_id = var.vpc_id 29 | 30 | route { 31 | cidr_block = "0.0.0.0/0" 32 | gateway_id = aws_nat_gateway.main[0].id 33 | } 34 | tags = merge(local.common_tags, var.tags) 35 | } 36 | 37 | # Creates Route table association rules if no Subnets are provided 38 | resource "aws_route_table_association" "private" { 39 | 40 | 41 | count = length(var.input_subnet_id) == 0 ? length(var.private_subnet_id) : 0 42 | subnet_id = element(var.private_subnet_id, count.index) 43 | route_table_id = aws_route_table.private[0].id 44 | } 45 | -------------------------------------------------------------------------------- /terraform/modules/nat/variables.tf: -------------------------------------------------------------------------------- 1 | variable "mod_depends_on" {} 2 | variable "input_subnet_id" {} 3 | variable "public_subnet_id" {} 4 | variable "private_subnet_id" {} 5 | variable "vpc_id" {} 6 | variable "tags" {} 7 | -------------------------------------------------------------------------------- /terraform/modules/r53/main.tf: -------------------------------------------------------------------------------- 1 | 2 | locals { 3 | common_tags = { 4 | Name = "quickfabric-dns-zone" 5 | } 6 | } 7 | 8 | data "aws_route53_zone" "quickfabric" { 9 | count = var.hosted_zone_name_exists ? 1 : 0 10 | name = var.zone_name 11 | } 12 | 13 | 14 | # Creates a Route53 zone record to access EMR cluster operations 15 | resource "aws_route53_zone" "quickfabric_zone" { 16 | 17 | count = var.hosted_zone_name_exists == false ? 1 : 0 18 | name = var.zone_name 19 | force_destroy = true 20 | 21 | tags = merge(local.common_tags, var.tags) 22 | } 23 | 24 | # Creates a Route53 zone record to access EMR cluster operations 25 | resource "aws_route53_record" "quickfabric_record" { 26 | 27 | zone_id = var.hosted_zone_name_exists ? data.aws_route53_zone.quickfabric[0].zone_id : aws_route53_zone.quickfabric_zone[0].zone_id 28 | name = var.www_domain_name 29 | type = "A" 30 | ttl = "3600" 31 | records = ["127.0.0.1"] 32 | 33 | } 34 | 35 | -------------------------------------------------------------------------------- /terraform/modules/r53/output.tf: -------------------------------------------------------------------------------- 1 | output "r53_hosted_zone" { 2 | 3 | value = var.hosted_zone_name_exists ? data.aws_route53_zone.quickfabric[0].name : aws_route53_zone.quickfabric_zone[0].name 4 | } 5 | -------------------------------------------------------------------------------- /terraform/modules/r53/variables.tf: -------------------------------------------------------------------------------- 1 | variable "hosted_zone_name_exists" {} 2 | variable "zone_name" {} 3 | variable "www_domain_name" {} 4 | variable "tags" {} 5 | -------------------------------------------------------------------------------- /terraform/modules/s3/main.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | common_tags = { 3 | Name = "quickfabric-s3-bucket" 4 | } 5 | } 6 | 7 | # Creates S3 bucket for quickfabric logs 8 | resource "aws_s3_bucket" "out_bucket" { 9 | 10 | bucket = var.bucket_name 11 | acl = "private" 12 | force_destroy = true 13 | tags = merge(local.common_tags, var.tags) 14 | 15 | } 16 | 17 | # Creates a logs directory under the S3 bucket 18 | resource "aws_s3_bucket_object" "directories" { 19 | 20 | count = length(var.directories) 21 | 22 | bucket = aws_s3_bucket.out_bucket.id 23 | key = element(var.directories, count.index) 24 | depends_on = [aws_s3_bucket.out_bucket] 25 | } 26 | 27 | 28 | -------------------------------------------------------------------------------- /terraform/modules/s3/output.tf: -------------------------------------------------------------------------------- 1 | output "bucket_name" { 2 | value = aws_s3_bucket.out_bucket.id 3 | } 4 | -------------------------------------------------------------------------------- /terraform/modules/s3/variables.tf: -------------------------------------------------------------------------------- 1 | variable "bucket_name" {} 2 | variable "tags" {} 3 | variable "directories" {} 4 | 5 | -------------------------------------------------------------------------------- /terraform/modules/s3_uploads/main.tf: -------------------------------------------------------------------------------- 1 | # Uploads a file to S3 2 | resource "aws_s3_bucket_object" "content" { 3 | 4 | for_each = var.provision ? var.content_uploads : {} 5 | 6 | bucket = var.bucket_name 7 | key = each.key 8 | content = each.value 9 | etag = md5(each.value) 10 | 11 | } 12 | 13 | # Uploads a file to S3 14 | resource "aws_s3_bucket_object" "file" { 15 | 16 | for_each = var.provision ? var.file_uploads : {} 17 | 18 | bucket = var.bucket_name 19 | key = each.key 20 | source = each.value.output_path 21 | etag = each.value.output_md5 22 | 23 | } 24 | -------------------------------------------------------------------------------- /terraform/modules/s3_uploads/variables.tf: -------------------------------------------------------------------------------- 1 | variable "bucket_name" {} 2 | variable "tags" {} 3 | 4 | variable "content_uploads" { default = {} } 5 | variable "file_uploads" { default = {} } 6 | 7 | variable "provision" { default = true } 8 | -------------------------------------------------------------------------------- /terraform/modules/sg/get_ip.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Get the public IP address of the system where you are running the 4 | # terrafrom scripts 5 | 6 | my_ip=`curl -s --retry 3 http://checkip.amazonaws.com` 7 | 8 | 9 | if [[ ! -z $my_ip ]]; 10 | then 11 | echo -n "{\"result\": \"$my_ip/32\"}" 12 | else 13 | echo -n "{\"no_result\": \"\"}" 14 | fi 15 | 16 | # returns a string in JSON format {"result" : ""} 17 | # for the terraform external to process 18 | -------------------------------------------------------------------------------- /terraform/modules/sg/output.tf: -------------------------------------------------------------------------------- 1 | output "sg_id" { 2 | value = aws_security_group.whitelist.id 3 | } 4 | -------------------------------------------------------------------------------- /terraform/modules/sg/variables.tf: -------------------------------------------------------------------------------- 1 | variable "sg_name" {} 2 | variable "vpc_id" {} 3 | variable "whitelist" {} 4 | variable "whitelist_sg" { default = { "inbound" : [] } } 5 | variable "tags" {} 6 | variable "cidr_admin_whitelist" { default = [] } 7 | variable "security_groups" { default = [] } 8 | 9 | -------------------------------------------------------------------------------- /terraform/modules/subnet/main.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | common_tags = { 3 | Name = "quickfabric-${var.name}" 4 | } 5 | } 6 | 7 | # Get the subnet details including VPC id. 8 | data "aws_subnet" "subnet" { 9 | 10 | count = length(var.subnet_id) > 0 ? 1 : 0 11 | 12 | id = var.subnet_id[0] 13 | } 14 | 15 | data "aws_availability_zones" "available" {} 16 | 17 | # Creates subnet if no Subnets are provided 18 | resource "aws_subnet" "main" { 19 | # Mentioning the availability region since the ebs volume lifecycle is related to this 20 | availability_zone = data.aws_availability_zones.available.names[0] 21 | count = length(var.subnet_id) > 0 ? 0 : length(var.subnet_cidr_block) 22 | vpc_id = var.vpc_id 23 | cidr_block = element(var.subnet_cidr_block, count.index) 24 | map_public_ip_on_launch = true 25 | tags = merge(local.common_tags, var.tags) 26 | } 27 | -------------------------------------------------------------------------------- /terraform/modules/subnet/output.tf: -------------------------------------------------------------------------------- 1 | output "subnet_id" { 2 | 3 | value = length(var.subnet_id) > 0 ? var.subnet_id : aws_subnet.main[*].id 4 | } 5 | 6 | output "subnet_az" { 7 | value = length(var.subnet_id) > 0 ? data.aws_subnet.subnet[0].availability_zone : aws_subnet.main[0].availability_zone 8 | } 9 | -------------------------------------------------------------------------------- /terraform/modules/subnet/variables.tf: -------------------------------------------------------------------------------- 1 | variable "subnet_cidr_block" {} 2 | variable "subnet_id" {} 3 | variable "vpc_id" {} 4 | variable "tags" {} 5 | variable "name" {} -------------------------------------------------------------------------------- /terraform/modules/vpc/main.tf: -------------------------------------------------------------------------------- 1 | 2 | locals { 3 | common_tags = { 4 | Name = "quickfabric-vpc" 5 | } 6 | } 7 | 8 | 9 | # Get the subnet details including VPC id. 10 | data "aws_subnet" "subnet" { 11 | 12 | count = length(var.public_subnet_id) == 0 && length(var.private_subnet_id) == 0 ? 0 : 1 13 | 14 | id = coalescelist(var.public_subnet_id, var.private_subnet_id)[0] 15 | } 16 | 17 | 18 | 19 | # Creates a VPC if no subnet ids are provided 20 | resource "aws_vpc" "main" { 21 | 22 | count = length(var.public_subnet_id) == 0 && length(var.private_subnet_id) == 0 ? 1 : 0 23 | 24 | cidr_block = var.cidr_block 25 | enable_dns_hostnames = true 26 | tags = merge(local.common_tags, var.tags) 27 | } 28 | 29 | -------------------------------------------------------------------------------- /terraform/modules/vpc/output.tf: -------------------------------------------------------------------------------- 1 | output "vpc_id" { 2 | 3 | value = length(var.public_subnet_id) == 0 && length(var.private_subnet_id) == 0 && length(aws_vpc.main) > 0 ? aws_vpc.main[0].id : length(data.aws_subnet.subnet) > 0 ? data.aws_subnet.subnet[0].vpc_id : "" 4 | } 5 | -------------------------------------------------------------------------------- /terraform/modules/vpc/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | variable "public_subnet_id" { type = list } 3 | variable "private_subnet_id" { type = list } 4 | variable "cidr_block" { type = string } 5 | variable "tags" { type = map } 6 | 7 | -------------------------------------------------------------------------------- /terraform/modules_backend/s3_backend/main.tf: -------------------------------------------------------------------------------- 1 | # Creates the config file in local 2 | resource "local_file" "backend_tf_file" { 3 | content = data.template_file.backend_tf[0].rendered 4 | filename = "./backend.tf" 5 | } 6 | 7 | # Identify the Caller account details 8 | data "aws_caller_identity" "current" {} 9 | 10 | # Create user data file using templates. 11 | data "template_file" "backend_tf" { 12 | count = fileexists("./modules/s3_backend/backend.tpl") ? 1 : 0 13 | template = file("./modules/s3_backend/backend.tpl") 14 | vars = { 15 | REGION = var.region 16 | BUCKET_NAME = aws_s3_bucket.state_bucket.bucket 17 | } 18 | } 19 | 20 | # Creates S3 bucket for terrafrom state 21 | resource "aws_s3_bucket" "state_bucket" { 22 | bucket = "${var.tf_s3_backend}-${data.aws_caller_identity.current.account_id}" 23 | acl = "private" 24 | 25 | server_side_encryption_configuration { 26 | rule { 27 | apply_server_side_encryption_by_default { 28 | sse_algorithm = "AES256" 29 | } 30 | } 31 | } 32 | 33 | versioning { 34 | enabled = true 35 | } 36 | 37 | lifecycle { 38 | prevent_destroy = true 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /terraform/modules_backend/s3_backend/variables.tf: -------------------------------------------------------------------------------- 1 | variable "tf_s3_backend" {} 2 | variable "region" {} 3 | 4 | -------------------------------------------------------------------------------- /terraform/output.tf: -------------------------------------------------------------------------------- 1 | output "quickfabric_url" { 2 | value = "\n\tQuickFabric will be accessible through the following URL. Please wait for few minutes for the docker containers to come up.\n\n http://${module.qf.server_ip}" 3 | } 4 | 5 | output "aws_account_id" { 6 | value = data.aws_caller_identity.current.account_id 7 | } 8 | 9 | output "bastion_ip" { 10 | value = module.bastion.server_ip 11 | } 12 | 13 | output "api_gw_credentials" { 14 | value = local.api_creds 15 | } 16 | -------------------------------------------------------------------------------- /terraform/provider.tf: -------------------------------------------------------------------------------- 1 | 2 | # Specify the provider and access details 3 | provider "aws" { 4 | region = var.region 5 | } 6 | 7 | 8 | -------------------------------------------------------------------------------- /terraform/scripts/env.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | 4 | data = {} 5 | 6 | 7 | def get_os_env(variable): 8 | return os.environ.get(variable.upper(),os.environ.get(variable.lower(),"")) 9 | 10 | data["HOME"] = get_os_env('HOME') 11 | data["AWS_PROFILE"] = get_os_env('AWS_PROFILE') 12 | data["AWS_ACCESS_KEY_ID"] = get_os_env('AWS_ACCESS_KEY_ID') 13 | data["AWS_SECRET_ACCESS_KEY"] = get_os_env('AWS_SECRET_ACCESS_KEY') 14 | data["AWS_SESSION_TOKEN"] = get_os_env('AWS_SESSION_TOKEN') 15 | 16 | print(json.dumps(data)) 17 | 18 | -------------------------------------------------------------------------------- /terraform/secrets.tfvars.example: -------------------------------------------------------------------------------- 1 | # Application encryption key 2 | AES_SECRET_KEY = "xxxxxxxxxxxx" 3 | 4 | # MySQL root password of the DB instance createtd by Quickfabric 5 | MYSQL_PASSWORD = "xxxxxxxxxxxx" 6 | -------------------------------------------------------------------------------- /terraform/templates/serverless/output.conf.tpl: -------------------------------------------------------------------------------- 1 | { 2 | "vpc_id": "${vpc_id}", 3 | "public_subnet_ids": ${public_subnet_ids}, 4 | "private_subnet_ids": ${private_subnet_ids}, 5 | "emr_version": "${emr_version}", 6 | "keypair": "${key_pair}", 7 | "artifacts_bucket": "s3://${bucket}", 8 | "emr_s3_log_path": "s3://${bucket}/${logs_path}", 9 | "region": "${region}", 10 | "r53_hosted_zone": "${r53_hosted_zone}", 11 | "bootstrap_actions": [ 12 | { 13 | "bootstrapScript": "scripts/install-gradle-bootstrap.sh", 14 | "bootstrapName": "Install Gradle" 15 | } 16 | ], 17 | "steps": [ 18 | { 19 | "Name": "InstallDrElephant", 20 | "OnBoot": true, 21 | "Type": "shell", 22 | "ActionOnFailure": "CONTINUE", 23 | "Script": "scripts/install-dr-elephant.sh", 24 | "Jar": "s3://elasticmapreduce/libs/script-runner/script-runner.jar", 25 | "Args": [] 26 | }], 27 | "env" : "${env}" 28 | } 29 | 30 | -------------------------------------------------------------------------------- /terraform_local/modules/container/main.tf: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Start a container 4 | resource "docker_container" "container" { 5 | name = var.container_name 6 | image = var.image 7 | working_dir = var.working_dir 8 | command = var.command 9 | 10 | attach = var.attach 11 | rm = var.rm 12 | must_run = var.must_run 13 | 14 | env = var.env 15 | 16 | dynamic "mounts" { 17 | for_each = var.mounts 18 | content { 19 | target = mounts.key 20 | source = mounts.value 21 | type = "bind" 22 | } 23 | } 24 | dynamic "ports" { 25 | for_each = var.ports 26 | content { 27 | internal = ports.key 28 | external = ports.value 29 | } 30 | } 31 | dynamic "networks_advanced" { 32 | for_each = var.network 33 | content { 34 | name = networks_advanced.value 35 | } 36 | } 37 | 38 | } 39 | 40 | output "exit_code" { 41 | value = docker_container.container.exit_code 42 | } 43 | -------------------------------------------------------------------------------- /terraform_local/modules/container/variables.tf: -------------------------------------------------------------------------------- 1 | variable "container_name" {} 2 | variable "data_store" {} 3 | variable "command" { default = [] } 4 | variable "working_dir" { default = "/" } 5 | 6 | variable "image" {} 7 | variable "attach" { default = false } 8 | variable "rm" { default = false } 9 | variable "must_run" { default = false } 10 | variable "mounts" { default = {} } 11 | 12 | variable "env" { default = [] } 13 | variable "network" { default = {} } 14 | variable "ports" { default = {} } 15 | -------------------------------------------------------------------------------- /terraform_local/output.tf: -------------------------------------------------------------------------------- 1 | output "quickfabric_url" { 2 | value = "http://127.0.0.1/" 3 | } 4 | -------------------------------------------------------------------------------- /terraform_local/provider.tf: -------------------------------------------------------------------------------- 1 | provider "docker" { 2 | alias = "docker" 3 | host = "unix:///var/run/docker.sock" 4 | } 5 | -------------------------------------------------------------------------------- /terraform_local/secrets.tfvars.example: -------------------------------------------------------------------------------- 1 | MYSQL_PASSWORD = "xxxxxxxxxxxx" 2 | AES_SECRET_KEY = "xxxxxxxxxxxx" 3 | 4 | 5 | -------------------------------------------------------------------------------- /terraform_local/variables.tf: -------------------------------------------------------------------------------- 1 | variable "MYSQL_PASSWORD" {} 2 | variable "AES_SECRET_KEY" {} 3 | variable "data_store_path" { default = "/tmp/data" } 4 | --------------------------------------------------------------------------------