├── .env ├── .env.common ├── .github └── workflows │ ├── go.yml │ └── golangci-lint.yml ├── .gitignore ├── Dockerfile-db ├── Dockerfile-kafka ├── LICENSE ├── README.md ├── api ├── .editorconfig ├── .env ├── .gitignore ├── Dockerfile ├── README.md ├── api │ ├── authenticated_http_handler.go │ ├── create_client.go │ ├── delete_client.go │ ├── generate_token.go │ ├── list_clients.go │ ├── pipeline │ │ └── run_pipeline.go │ └── stats │ │ └── stats_api.go ├── cli │ ├── app.go │ └── command │ │ ├── db.go │ │ ├── fixtures.go │ │ ├── migrate.go │ │ └── server.go ├── config │ └── config.go ├── db │ └── migration │ │ ├── 000001_init_schema.down.sql │ │ └── 000001_init_schema.up.sql ├── docker-compose.yml ├── go.mod ├── go.sum ├── helpers │ ├── decode_json_body.go │ ├── errors.go │ ├── gorm.go │ └── headers.go ├── main.go ├── model │ ├── command │ │ ├── create_oauth_client.go │ │ └── delete_oauth_client.go │ ├── entity │ │ └── oauth_client.go │ ├── persister │ │ └── persister.go │ └── repository │ │ ├── error.go │ │ ├── oauth_client_repository.go │ │ └── oauth_token_repository.go ├── service │ ├── authorization_validator.go │ ├── oauth │ │ ├── client_store.go │ │ ├── scopes.go │ │ ├── server.go │ │ └── token_store.go │ ├── permission_validator.go │ ├── proxying_client.go │ └── ylem_users │ │ └── client.go └── tests │ ├── .env │ └── service │ └── oauth │ ├── .env │ └── scopes_test.go ├── backend ├── integrations │ ├── .editorconfig │ ├── .env │ ├── .gitignore │ ├── Dockerfile │ ├── README.md │ ├── api │ │ ├── AuthorizeHubspot.go │ │ ├── AuthorizeJira.go │ │ ├── AuthorizeSalesforce.go │ │ ├── AuthorizeSlack.go │ │ ├── ChangeIntegrationStatus.go │ │ ├── Common.go │ │ ├── ConfirmEmailIntegration.go │ │ ├── ConfirmSmsIntegration.go │ │ ├── CreateApiIntegration.go │ │ ├── CreateEmailIntegration.go │ │ ├── CreateGoogleSheetsIntegration.go │ │ ├── CreateHubspotAuthorization.go │ │ ├── CreateHubspotIntegration.go │ │ ├── CreateIncidentIoIntegration.go │ │ ├── CreateJenkinsIntegration.go │ │ ├── CreateJiraAuthorization.go │ │ ├── CreateJiraIntegration.go │ │ ├── CreateOpsgenieIntegration.go │ │ ├── CreateSQLIntegration.go │ │ ├── CreateSalesforceAuthorization.go │ │ ├── CreateSalesforceIntegration.go │ │ ├── CreateSlackAuthorization.go │ │ ├── CreateSlackIntegration.go │ │ ├── CreateSmsIntegration.go │ │ ├── CreateTableauIntegration.go │ │ ├── CreateWhatsAppIntegration.go │ │ ├── DeleteIntegration.go │ │ ├── DescribeSQLIntegrationTable.go │ │ ├── GetAllHubspotAuthorizations.go │ │ ├── GetAllIntegrations.go │ │ ├── GetAllJiraAuthorizations.go │ │ ├── GetAllSalesforceAuthorizations.go │ │ ├── GetAllSlackAuthorizations.go │ │ ├── GetApiIntegration.go │ │ ├── GetApiIntegrationPrivate.go │ │ ├── GetEmailIntegration.go │ │ ├── GetEmailIntegrationPrivate.go │ │ ├── GetGoogleSheetsIntegration.go │ │ ├── GetGoogleSheetsIntegrationPrivate.go │ │ ├── GetHubspotAuthorization.go │ │ ├── GetHubspotIntegration.go │ │ ├── GetHubspotIntegrationPrivate.go │ │ ├── GetIncidentIoIntegration.go │ │ ├── GetIncidentIoIntegrationPrivate.go │ │ ├── GetIncidentIoSeverities.go │ │ ├── GetJenkinsIntegration.go │ │ ├── GetJenkinsIntegrationPrivate.go │ │ ├── GetJiraAuthorization.go │ │ ├── GetJiraIntegration.go │ │ ├── GetJiraIntegrationPrivate.go │ │ ├── GetOpsgenieIntegration.go │ │ ├── GetOpsgenieIntegrationPrivate.go │ │ ├── GetSQLIntegration.go │ │ ├── GetSQLIntegrationPrivate.go │ │ ├── GetSalesforceAuthorization.go │ │ ├── GetSalesforceIntegration.go │ │ ├── GetSalesforceIntegrationPrivate.go │ │ ├── GetSlackAuthorization.go │ │ ├── GetSlackIntegration.go │ │ ├── GetSlackIntegrationPrivate.go │ │ ├── GetSmsIntegration.go │ │ ├── GetSmsIntegrationPrivate.go │ │ ├── GetTableauIntegration.go │ │ ├── GetTableauIntegrationPrivate.go │ │ ├── GetWhatsAppIntegration.go │ │ ├── GetWhatsAppIntegrationPrivate.go │ │ ├── ReauthorizeSlackAuthorization.go │ │ ├── ResendConfirmationEmail.go │ │ ├── ResendConfirmationSms.go │ │ ├── ShowSQLIntegrationDatabases.go │ │ ├── ShowSQLIntegrationTables.go │ │ ├── TestExistingSQLIntegrationConnection.go │ │ ├── TestSQLIntegrationConnection.go │ │ ├── UpdateApiIntegration.go │ │ ├── UpdateEmailIntegration.go │ │ ├── UpdateGoogleSheetsIntegration.go │ │ ├── UpdateHubspotAuthorization.go │ │ ├── UpdateHubspotIntegration.go │ │ ├── UpdateIncidentIoIntegration.go │ │ ├── UpdateJenkinsIntegration.go │ │ ├── UpdateJiraAuthorization.go │ │ ├── UpdateJiraIntegration.go │ │ ├── UpdateOpsgenieIntegration.go │ │ ├── UpdateSQLIntegration.go │ │ ├── UpdateSalesforceAuthorization.go │ │ ├── UpdateSalesforceIntegration.go │ │ ├── UpdateSlackAuthorization.go │ │ ├── UpdateSlackIntegration.go │ │ ├── UpdateSmsIntegration.go │ │ ├── UpdateTableauIntegration.go │ │ └── UpdateWhatsAppIntegration.go │ ├── cli │ │ ├── app.go │ │ └── command │ │ │ ├── db │ │ │ ├── db.go │ │ │ └── migrate.go │ │ │ ├── kafka │ │ │ └── kafka_consumer.go │ │ │ └── server │ │ │ └── server.go │ ├── config │ │ ├── config.go │ │ └── keys │ │ │ ├── .gitignore │ │ │ └── .gitkeep │ ├── db │ │ └── migration │ │ │ ├── 000001_init_schema.down.sql │ │ │ ├── 000001_init_schema.up.sql │ │ │ ├── 000002_add_whatsapp.down.sql │ │ │ └── 000002_add_whatsapp.up.sql │ ├── docker-compose.yml │ ├── entities │ │ ├── Api.go │ │ ├── Email.go │ │ ├── GoogleSheets.go │ │ ├── Hubspot.go │ │ ├── HubspotAuthorization.go │ │ ├── HubspotAuthorizationCollection.go │ │ ├── IncidentIo.go │ │ ├── Integration.go │ │ ├── IntegrationCollection.go │ │ ├── Jenkins.go │ │ ├── Jira.go │ │ ├── JiraAuthorization.go │ │ ├── JiraAuthorizationCollection.go │ │ ├── Opsgenie.go │ │ ├── SQLIntegration.go │ │ ├── Salesforce.go │ │ ├── SalesforceAuthorization.go │ │ ├── SalesforceAuthorizationCollection.go │ │ ├── Slack.go │ │ ├── SlackAuthorization.go │ │ ├── SlackAuthorizationCollection.go │ │ ├── Sms.go │ │ ├── Tableau.go │ │ └── WhatsApp.go │ ├── go.mod │ ├── go.sum │ ├── helpers │ │ ├── apiDestinationAuthentication.go │ │ ├── db.go │ │ ├── decodeJsonBody.go │ │ ├── httpErrors.go │ │ ├── postgresql │ │ │ └── driver.go │ │ ├── rand.go │ │ ├── sqlToJson.go │ │ └── ssh.go │ ├── main.go │ ├── repositories │ │ ├── ApiRepository.go │ │ ├── EmailRepository.go │ │ ├── GoogleSheetsRepository.go │ │ ├── HubspotAuthorizationRepository.go │ │ ├── HubspotRepository.go │ │ ├── IncidentIoRepository.go │ │ ├── IntegrationRepository.go │ │ ├── JenkinsRepository.go │ │ ├── JiraAuthorizationRepository.go │ │ ├── JiraRepository.go │ │ ├── OpsgenieRepository.go │ │ ├── SQLRepository.go │ │ ├── SalesforceAuthorizationRepository.go │ │ ├── SalesforceRepository.go │ │ ├── SlackAuthorizationRepository.go │ │ ├── SlackRepository.go │ │ ├── SmsRepository.go │ │ ├── TableauRepository.go │ │ └── WhatsAppRepository.go │ ├── resources │ │ ├── embedded.go │ │ └── templates │ │ │ └── html │ │ │ └── confirmation_email.gohtml │ ├── services │ │ ├── ConfirmUsersEmail.go │ │ ├── EMailSender.go │ │ ├── JIraService.go │ │ ├── SlackService.go │ │ ├── SmsSender.go │ │ ├── UpdateIntegrationConnection.go │ │ ├── YlemUsers.go │ │ ├── aws │ │ │ └── kms │ │ │ │ ├── box.go │ │ │ │ └── kms.go │ │ ├── bigquery │ │ │ └── connection.go │ │ ├── es │ │ │ └── es.go │ │ ├── hubspot.go │ │ ├── incidentio │ │ │ └── client.go │ │ ├── runner.go │ │ ├── salesforce.go │ │ ├── server │ │ │ └── server.go │ │ └── sql │ │ │ ├── MySQLIntegrationConnection.go │ │ │ ├── PostgreSqlIntegrationConnection.go │ │ │ ├── RedshiftIntegrationConnection.go │ │ │ ├── SQLIntegrationConnection.go │ │ │ ├── SnowflakeIntegrationConnection.go │ │ │ └── init.go │ └── tests │ │ └── .env ├── pipelines │ ├── .editorconfig │ ├── .env │ ├── .gitignore │ ├── Dockerfile │ ├── README.md │ ├── app │ │ ├── dashboard │ │ │ ├── api_handler.go │ │ │ ├── dashboard.go │ │ │ └── data.go │ │ ├── envvariable │ │ │ ├── api_handler.go │ │ │ ├── data.go │ │ │ ├── env_variable.go │ │ │ └── env_variables.go │ │ ├── folder │ │ │ ├── Folders.go │ │ │ ├── api_handler.go │ │ │ ├── data.go │ │ │ └── folder.go │ │ ├── pipeline │ │ │ ├── Pipelines.go │ │ │ ├── api_handler.go │ │ │ ├── common │ │ │ │ └── common.go │ │ │ ├── data.go │ │ │ ├── pipeline.go │ │ │ └── run │ │ │ │ └── pipeline_run_config.go │ │ ├── pipelinetemplate │ │ │ ├── api_handler_share_link.go │ │ │ ├── api_handler_template.go │ │ │ ├── data.go │ │ │ └── shared_workflow.go │ │ ├── schedule │ │ │ ├── data.go │ │ │ └── scheduled_run.go │ │ ├── task │ │ │ ├── Tasks.go │ │ │ ├── api_handler.go │ │ │ ├── data.go │ │ │ ├── data_aggregator.go │ │ │ ├── data_api_call.go │ │ │ ├── data_code.go │ │ │ ├── data_condition.go │ │ │ ├── data_external_trigger.go │ │ │ ├── data_filter.go │ │ │ ├── data_for_each.go │ │ │ ├── data_gpt.go │ │ │ ├── data_merge.go │ │ │ ├── data_notification.go │ │ │ ├── data_processor.go │ │ │ ├── data_query.go │ │ │ ├── data_run_pipeline.go │ │ │ ├── data_transformer.go │ │ │ ├── http_types.go │ │ │ ├── implementation_router.go │ │ │ ├── result │ │ │ │ ├── api_handler.go │ │ │ │ ├── data.go │ │ │ │ └── task_run_result.go │ │ │ └── task.go │ │ ├── tasktrigger │ │ │ ├── api_handler.go │ │ │ ├── data.go │ │ │ ├── task_trigger.go │ │ │ ├── task_triggers.go │ │ │ └── types │ │ │ │ └── types.go │ │ └── trial │ │ │ ├── api_handler.go │ │ │ └── creator.go │ ├── cli │ │ ├── app.go │ │ └── command │ │ │ ├── db.go │ │ │ ├── fixtures.go │ │ │ ├── migrate.go │ │ │ ├── schedule_generator.go │ │ │ ├── schedule_publisher.go │ │ │ ├── server.go │ │ │ └── trigger_listener.go │ ├── config │ │ └── config.go │ ├── db │ │ └── migration │ │ │ ├── 000001_init_schema.down.sql │ │ │ ├── 000001_init_schema.up.sql │ │ │ ├── 000002_pipeline_templates.down.sql │ │ │ └── 000002_pipeline_templates.up.sql │ ├── docker-compose.yml │ ├── go.mod │ ├── go.sum │ ├── helpers │ │ ├── db.go │ │ ├── decode_headers.go │ │ ├── decode_json_body.go │ │ ├── http.go │ │ ├── misc.go │ │ ├── slice.go │ │ └── ui │ │ │ └── element_to_ui.go │ ├── main.go │ ├── services │ │ ├── authorization_validator.go │ │ ├── kafka │ │ │ └── goka.go │ │ ├── messaging │ │ │ ├── aggregator.go │ │ │ ├── api.go │ │ │ ├── code.go │ │ │ ├── condition.go │ │ │ ├── errors.go │ │ │ ├── external_trigger.go │ │ │ ├── factory.go │ │ │ ├── filter.go │ │ │ ├── for_each.go │ │ │ ├── gpt.go │ │ │ ├── merge.go │ │ │ ├── notification.go │ │ │ ├── processor.go │ │ │ ├── query.go │ │ │ ├── run_pipeline.go │ │ │ └── transformer.go │ │ ├── permission_validator.go │ │ ├── pipeline_connection.go │ │ ├── provider │ │ │ ├── pipeline_provider.go │ │ │ └── task_provider.go │ │ ├── schedule │ │ │ └── publisher.go │ │ ├── taskrunner │ │ │ └── pipeline_run_initiator.go │ │ ├── trigger │ │ │ └── listener │ │ │ │ └── trigger_listener.go │ │ └── ylem_integrations │ │ │ └── client.go │ └── tests │ │ └── services │ │ └── messaging │ │ ├── .env │ │ ├── aggregator_test.go │ │ ├── api_test.go │ │ ├── code_test.go │ │ ├── condition_test.go │ │ ├── external_trigger_test.go │ │ ├── filter_test.go │ │ ├── for_each_test.go │ │ ├── gpt_test.go │ │ ├── merge_test.go │ │ ├── notification_test.go │ │ ├── processor_test.go │ │ ├── query_test.go │ │ ├── run_pipeline_test.go │ │ └── transformer_test.go ├── statistics │ ├── .editorconfig │ ├── .env │ ├── .gitignore │ ├── Dockerfile │ ├── Dockerfile-db │ ├── README.md │ ├── api │ │ ├── get_aggregated_task_stats.go │ │ ├── get_avg_workflow_value.go │ │ ├── get_last_metric_values.go │ │ ├── get_last_run_stats.go │ │ ├── get_last_workflow_runs_log.go │ │ ├── get_metric_values.go │ │ ├── get_slow_task_runs.go │ │ ├── get_stats.go │ │ ├── get_workflow_duration_stats_quantile.go │ │ └── get_workflow_value_quantile.go │ ├── cli │ │ ├── app.go │ │ └── command │ │ │ ├── db │ │ │ ├── db.go │ │ │ ├── fixtures.go │ │ │ └── migrate.go │ │ │ ├── server │ │ │ └── server.go │ │ │ └── taskrun │ │ │ └── result_listener.go │ ├── config │ │ └── config.go │ ├── database │ │ └── .gitignore │ ├── docker-compose.yml │ ├── domain │ │ ├── entity │ │ │ ├── persister │ │ │ │ └── entity_persister.go │ │ │ └── task_run.go │ │ └── readmodel │ │ │ ├── common.go │ │ │ ├── dto │ │ │ └── stats.go │ │ │ ├── pipeline.go │ │ │ └── task_run.go │ ├── go.mod │ ├── go.sum │ ├── helpers │ │ ├── decodeJSONBody.go │ │ ├── errors.go │ │ └── time.go │ ├── main.go │ ├── services │ │ ├── AuthenticationValidator.go │ │ ├── PermissionValidator.go │ │ ├── db │ │ │ ├── db.go │ │ │ ├── migration.go │ │ │ └── migration │ │ │ │ └── 20231113203511.go │ │ ├── server │ │ │ └── server.go │ │ └── taskrun │ │ │ └── result_listener.go │ └── tests │ │ └── .env └── users │ ├── .editorconfig │ ├── .env │ ├── .gitignore │ ├── Dockerfile │ ├── README.md │ ├── api │ ├── ActivateUser.go │ ├── AssignRoleToUser.go │ ├── AuthorizationCheck.go │ ├── ConfirmEmail.go │ ├── ConfirmEmailInternal.go │ ├── DeleteUser.go │ ├── ExternalAuth.go │ ├── ExternalAuthCallback.go │ ├── GetMyOrganization.go │ ├── GetOrganizationDataKey.go │ ├── GetPendingInvitationsInOrganization.go │ ├── GetUsersInOrganization.go │ ├── IsExternalAuthAvailable.go │ ├── IssueJWTPrivate.go │ ├── LoginUser.go │ ├── PermissionCheck.go │ ├── PostInvitations.go │ ├── PostUser.go │ ├── UpdateMe.go │ ├── UpdateMyPassword.go │ ├── UpdateOrganization.go │ ├── UpdateOrganizationConnections.go │ └── ValidateInvitation.go │ ├── cli │ ├── app.go │ └── command │ │ ├── db.go │ │ ├── encrypt │ │ └── encrypt.go │ │ ├── fixtures.go │ │ ├── migrate.go │ │ └── server │ │ └── server.go │ ├── config │ ├── config.go │ └── jwt │ │ └── .gitkeep │ ├── db │ └── migration │ │ ├── 000001_init_schema.down.sql │ │ └── 000001_init_schema.up.sql │ ├── docker-compose.yml │ ├── entities │ ├── Action.go │ ├── Invitations.go │ ├── Organization.go │ ├── Resources.go │ └── User.go │ ├── go.mod │ ├── go.sum │ ├── helpers │ ├── db.go │ ├── decodeJsonBody.go │ ├── errors.go │ ├── randSeq.go │ ├── sqlToJson.go │ └── tsHumanizer.go │ ├── main.go │ ├── repositories │ ├── InvitationRepository.go │ ├── OrganizationRepository.go │ └── UserRepository.go │ ├── services │ ├── Authorization.go │ ├── CreateDefaultEmailDestination.go │ ├── CreateTrialDBDataSource.go │ ├── CreateTrialPipelines.go │ ├── Permissions.go │ ├── SignUpUser.go │ ├── TestTrialDBDataSource.go │ ├── Validators.go │ ├── kms │ │ ├── box.go │ │ └── kms.go │ └── server │ │ └── server.go │ └── tests │ ├── entities │ ├── .env │ ├── Action_test.go │ └── User_test.go │ └── services │ ├── .env │ ├── Permissions_test.go │ └── Validators_test.go ├── database ├── .gitignore ├── init.sql └── redis │ └── redis.conf ├── docker-compose.yml ├── processor ├── python_processor │ ├── .editorconfig │ ├── .gitignore │ ├── Dockerfile │ ├── README.md │ ├── docker-compose.yml │ ├── main.py │ └── requirements.txt └── taskrunner │ ├── .editorconfig │ ├── .env │ ├── .gitignore │ ├── Dockerfile │ ├── README.md │ ├── api │ └── example_handler.go │ ├── cli │ ├── app.go │ └── command │ │ ├── fixtures.go │ │ ├── load_balancer.go │ │ ├── server │ │ └── server.go │ │ └── task_runner.go │ ├── config │ ├── config.go │ └── keys │ │ └── .gitkeep │ ├── docker-compose.yml │ ├── domain │ └── runner │ │ ├── aggregate_data.go │ │ ├── call_api.go │ │ ├── check_condition.go │ │ ├── code.go │ │ ├── external_trigger.go │ │ ├── filter.go │ │ ├── gpt.go │ │ ├── merge.go │ │ ├── process_data.go │ │ ├── run_for_each.go │ │ ├── run_pipeline.go │ │ ├── run_query.go │ │ ├── runner.go │ │ ├── send_notification.go │ │ └── transform_data.go │ ├── go.mod │ ├── go.sum │ ├── helpers │ ├── aws.go │ ├── decode_json.go │ ├── errors.go │ ├── evaluate.go │ ├── evaluate │ │ ├── arithmetic.go │ │ ├── date.go │ │ ├── funcs.go │ │ ├── language.go │ │ ├── text.go │ │ └── variable.go │ ├── kafka │ │ └── decode_input.go │ ├── postgresql │ │ └── driver.go │ ├── sqlToJson.go │ └── time.go │ ├── internal │ └── loadbalancer │ │ └── partitioner.go │ ├── main.go │ ├── services │ ├── api │ │ └── api.go │ ├── aws │ │ ├── kms │ │ │ ├── box.go │ │ │ ├── kms.go │ │ │ └── source.go │ │ └── ses.go │ ├── bigquery │ │ └── connection.go │ ├── es │ │ └── es.go │ ├── evaluate │ │ └── evaluate.go │ ├── gemini │ │ └── gemini.go │ ├── google │ │ └── sheets │ │ │ └── client.go │ ├── gopyk │ │ └── client.go │ ├── hubspot │ │ └── hubspot.go │ ├── incidentio │ │ └── client.go │ ├── jenkins │ │ └── client.go │ ├── jira │ │ └── jira.go │ ├── openai │ │ ├── gpt.go │ │ └── openai.go │ ├── opsgenie │ │ └── client.go │ ├── redis │ │ └── client.go │ ├── salesforce │ │ └── salesforce.go │ ├── server │ │ └── server.go │ ├── slack │ │ └── slack.go │ ├── sqlIntegrations │ │ ├── IntegrationConnection.go │ │ ├── MySQLIntegrationConnection.go │ │ ├── PostgreSqlIntegrationConnection.go │ │ ├── RedshiftIntegrationConnection.go │ │ └── SnowflakeIntegrationConnection.go │ ├── tableau │ │ ├── client.go │ │ └── misc.go │ ├── templater │ │ └── templater.go │ ├── transformers │ │ └── transformers.go │ ├── twilio │ │ └── twilio.go │ └── ylem_statistics │ │ └── client.go │ └── tests │ └── services │ ├── evaluate │ ├── .env │ ├── config │ │ └── keys │ │ │ └── id_rsa │ └── evaluate_test.go │ └── transformers │ ├── .env │ ├── config │ └── keys │ │ └── id_rsa │ └── transformers_test.go ├── server ├── .gitignore ├── README.md ├── docker-compose.yml ├── logs │ └── nginx │ │ └── .gitignore └── nginx │ ├── Dockerfile │ ├── conf.d │ └── default.conf │ ├── nginx.conf │ └── sites │ └── default.conf └── ui ├── .env ├── .gitignore ├── Dockerfile ├── Dockerfile.dev ├── README.md ├── docker-compose-dev.yml ├── docker-compose.yml ├── docker └── nginx │ └── default.conf ├── eslint.config.js ├── jsconfig.json ├── package-lock.json ├── package.json ├── public ├── favicon.ico ├── images │ ├── info.png │ ├── logo-s-dark.png │ ├── logo-s.png │ ├── logo2-dark.png │ ├── logo2.png │ ├── release-notes │ │ └── r17.png │ ├── template-previews │ │ ├── 0.jpeg │ │ ├── 1.jpeg │ │ ├── 2.jpeg │ │ ├── 3.jpeg │ │ ├── 4.jpeg │ │ ├── 5.jpeg │ │ ├── 6.jpeg │ │ ├── 7.jpeg │ │ ├── 8.jpeg │ │ └── 9.jpeg │ └── tour │ │ ├── dashboard.png │ │ ├── env-variables.png │ │ ├── metrics.png │ │ ├── oauth.png │ │ ├── pipeline.png │ │ ├── profiling.png │ │ └── welcome.png ├── index.html ├── logo192.png ├── manifest.json └── robots.txt └── src ├── App.js ├── App.scss ├── App.test.js ├── Modal.scss ├── actions ├── OAuthClients.js ├── auth.js ├── envVariables.js ├── errors.js ├── folders.js ├── infoTexts.js ├── integrations.js ├── message.js ├── pipeline.js ├── pipelines.js ├── releaseNotes.js ├── roles.js ├── settings.js ├── taskTriggers.js ├── tasks.js ├── tourSteps.js └── types.js ├── components ├── avatar.component.js ├── breadcrumbs.component.js ├── charts │ ├── barChart.component.js │ ├── lineChart.component.js │ └── polarAreaChart.component.js ├── confirmEmail.component.js ├── datepicker.component.js ├── emailChips.component.js ├── externalSignIn.component.js ├── formControls │ ├── input.component.js │ ├── inputChips.component.js │ ├── queryUI.component.js │ ├── textarea.component.js │ ├── textareaEditor.component.js │ └── validations.js ├── forms │ ├── OAuthClientForm.component.js │ ├── envVariableForm.component.js │ ├── folderForm.component.js │ ├── hubspotAuthorizationForm.component.js │ ├── integrationForm.component.js │ ├── jiraAuthorizationForm.component.js │ ├── metricForm.component.js │ ├── salesforceAuthorizationForm.component.js │ ├── scheduleForm.component.js │ ├── slackAuthorizationForm.component.js │ └── sqlIntegrationForm.component.js ├── invitation.component.js ├── login.component.js ├── modals │ ├── confirmationModal.component.js │ ├── fullScreenModal.component.js │ ├── fullScreenWithMenusModal.component.js │ ├── infoModal.component.js │ ├── rightModal.component.js │ └── taskModal.component.js ├── pendingInvitations.component.js ├── pipelines │ ├── forms │ │ ├── aggregatorForm.component.js │ │ ├── apiCallForm.component.js │ │ ├── codeForm.component.js │ │ ├── conditionForm.component.js │ │ ├── externalTriggerForm.component.js │ │ ├── filterForm.component.js │ │ ├── forEachForm.component.js │ │ ├── gptForm.component.js │ │ ├── mergeForm.component.js │ │ ├── notificationForm.component.js │ │ ├── pipelineRunForm.component.js │ │ ├── processorForm.component.js │ │ ├── queryForm.component.js │ │ ├── taskForm.component.js │ │ └── transformerForm.component.js │ ├── initial-elements.js │ ├── miniPipeline.component.js │ ├── nodes │ │ ├── APICallNode.component.js │ │ ├── aggregatorNode.component.js │ │ ├── codeNode.component.js │ │ ├── conditionNode.component.js │ │ ├── externalTriggerNode.component.js │ │ ├── filterNode.component.js │ │ ├── forEachNode.component.js │ │ ├── gptNode.component.js │ │ ├── mergeNode.component.js │ │ ├── notificationNode.component.js │ │ ├── pipelineRunNode.component.js │ │ ├── processorNode.component.js │ │ ├── queryNode.component.js │ │ └── transformerNode.component.js │ ├── pipeline.component.js │ ├── pipelineBreadcrumbs.component.js │ ├── pipelineLogs.component.js │ ├── pipelineRun.component.js │ ├── pipelineRunOutput.component.js │ ├── pipelineSidebar.component.js │ ├── pipelineSlowTaskRuns.component.js │ ├── pipelineTabs.component.js │ ├── pipelineTemplates.component.js │ ├── pipelineTriggers.component.js │ └── statistic │ │ ├── pipelineStatistic.component.js │ │ └── pipelineStatisticSummary.component.js ├── register.component.js ├── timeAgo.component.js └── widgets │ ├── integrationWidget.component.js │ └── pipelineWidget.component.js ├── containers ├── content.js ├── footer.js ├── header.js ├── headerDropdown.js ├── headerSearch.js ├── index.js ├── integrationsTabs.js ├── layout.js └── sidebar.js ├── helpers └── history.js ├── images ├── api-i.png ├── api.png ├── aws-rds-i.png ├── clickhouse-i.png ├── elasticsearch-i.png ├── email-i.png ├── folder.png ├── google-bigquery-i.png ├── google-cloud-sql-i.png ├── google-generic-i.png ├── google-sheets-i.png ├── hubspot-i.png ├── immuta-i.png ├── incidentio-i.png ├── info.png ├── jenkins-i.png ├── jira-i.png ├── microsoft-azure-sql-i.png ├── mysql-i.png ├── opsgenie-i.png ├── planet-scale-i.png ├── postgresql-i.png ├── postgresql.png ├── question-i.png ├── redshift-i.png ├── salesforce-i.png ├── slack-i.png ├── slack.png ├── sms-i.png ├── snowflake-i.png ├── snowflake.png ├── sql-i.png ├── tableau-i.png ├── taskFailed.png ├── taskPaused.png ├── taskPending.png ├── taskRunning.gif ├── taskRunning.png ├── taskSuccess.png ├── tour │ └── welcome.png ├── visualhomepage.png └── whatsapp-i.png ├── index.css ├── index.js ├── logo.svg ├── polyill.js ├── reducers ├── auth.js ├── index.js └── message.js ├── reportWebVitals.js ├── routes.js ├── serviceWorker.js ├── services ├── auth.service.js ├── envVariable.service.js ├── folder.service.js ├── integration.service.js ├── invitation.service.js ├── oauth.service.js ├── organization.service.js ├── pipeline.service.js ├── stat.service.js ├── task.service.js ├── taskTrigger.service.js ├── template.service.js └── user.service.js ├── setupTests.js ├── store.js └── views ├── dashboard └── Dashboard.js └── pages ├── api-clients └── APIClients.js ├── env-variables └── EnvVariables.js ├── integrations ├── HubspotAuthorizations.js ├── Integrations.js ├── JiraAuthorizations.js ├── SalesforceAuthorizations.js └── SlackAuthorizations.js ├── metrics └── Metrics.js ├── page404 └── Page404.js ├── pipelines └── Pipelines.js ├── profiling └── SlowTasks.js ├── settings └── Settings.js └── users └── Users.js /.env: -------------------------------------------------------------------------------- 1 | YLEM_DATABASE_NAME=ylem 2 | YLEM_DATABASE_ROOT_PASSWORD=dtmnsecret 3 | YLEM_DATABASE_USER=dtmnuser 4 | YLEM_DATABASE_PASSWORD=dtmnpassword 5 | 6 | YLEM_REDIS_PASSWORD=dtmnpassword 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # If you prefer the allow list template instead of the deny list, see community template: 2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 3 | # 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Dependency directories (remove the comment below to include it) 18 | # vendor/ 19 | 20 | # Go workspace file 21 | go.work 22 | .DS_Store 23 | /database/data/ 24 | the-container 25 | .idea/ 26 | upd_ui 27 | -------------------------------------------------------------------------------- /Dockerfile-db: -------------------------------------------------------------------------------- 1 | FROM mariadb:latest 2 | 3 | EXPOSE 3306 4 | -------------------------------------------------------------------------------- /Dockerfile-kafka: -------------------------------------------------------------------------------- 1 | FROM apache/kafka:3.7.0 2 | 3 | EXPOSE 9092 4 | -------------------------------------------------------------------------------- /api/.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | insert_final_newline = true 3 | indent_style = tab 4 | -------------------------------------------------------------------------------- /api/.env: -------------------------------------------------------------------------------- 1 | API_DATABASE_NAME=api 2 | -------------------------------------------------------------------------------- /api/.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | cover.html 14 | 15 | # Dependency directories (remove the comment below to include it) 16 | # vendor/ 17 | 18 | # Others 19 | .DS_Store 20 | .idea 21 | ylem_api 22 | .env.local 23 | -------------------------------------------------------------------------------- /api/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.23.2-alpine AS builder 2 | 3 | ENV CGO_ENABLED=0 4 | 5 | RUN apk add --no-cache ca-certificates git curl 6 | 7 | RUN mkdir /user && \ 8 | echo 'nobody:x:65534:65534:nobody:/:' > /user/passwd && \ 9 | echo 'nobody:x:65534:' > /user/group 10 | 11 | WORKDIR /opt/ylem_api 12 | 13 | COPY go.mod ./ 14 | COPY go.sum ./ 15 | 16 | RUN go mod download 17 | 18 | COPY . . 19 | 20 | RUN go build . 21 | 22 | FROM golang:1.23.2-alpine AS final 23 | 24 | COPY --from=builder /usr/local/bin /usr/local/bin 25 | 26 | COPY --from=builder /user/group /user/passwd /etc/ 27 | 28 | COPY --from=builder /opt /opt 29 | 30 | #USER root 31 | 32 | EXPOSE 7339 33 | 34 | WORKDIR /opt/ylem_api 35 | 36 | #CMD ["/opt/ylem_api/ylem_api", "server", "serve"] 37 | -------------------------------------------------------------------------------- /api/api/generate_token.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "net/http" 5 | "ylem_api/service/oauth" 6 | 7 | log "github.com/sirupsen/logrus" 8 | ) 9 | 10 | func GenerateToken(w http.ResponseWriter, r *http.Request) { 11 | srv, err := oauth.NewServer() 12 | if err != nil { 13 | log.Error(err) 14 | w.WriteHeader(http.StatusInternalServerError) 15 | return 16 | } 17 | 18 | err = srv.HandleTokenRequest(w, r) 19 | if err != nil { 20 | log.Error(err) 21 | w.WriteHeader(http.StatusInternalServerError) 22 | return 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /api/cli/app.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "ylem_api/cli/command" 5 | 6 | "github.com/urfave/cli/v2" 7 | ) 8 | 9 | func NewApplication() *cli.App { 10 | return &cli.App{ 11 | Commands: []*cli.Command{ 12 | command.ServerCommands, 13 | command.DbCommands, 14 | }, 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /api/cli/command/db.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import "github.com/urfave/cli/v2" 4 | 5 | var DbCommands = &cli.Command{ 6 | Name: "db", 7 | Usage: "Database management commands", 8 | Subcommands: []*cli.Command{ 9 | FixturesCommand, 10 | MigrationsCommand, 11 | }, 12 | } 13 | -------------------------------------------------------------------------------- /api/cli/command/fixtures.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | log "github.com/sirupsen/logrus" 5 | "github.com/urfave/cli/v2" 6 | ) 7 | 8 | var fixtureLoadHandler cli.ActionFunc = func(c *cli.Context) error { 9 | log.Info("Loading fixtures...") 10 | log.Info("Done.") 11 | 12 | return nil 13 | } 14 | 15 | var FixtureLoadCommand = &cli.Command{ 16 | Name: "load", 17 | Usage: "Load fixtures into database", 18 | Action: fixtureLoadHandler, 19 | } 20 | 21 | var FixturesCommand = &cli.Command{ 22 | Name: "fixtures", 23 | Usage: "Fixtures", 24 | Subcommands: []*cli.Command{ 25 | FixtureLoadCommand, 26 | }, 27 | } 28 | -------------------------------------------------------------------------------- /api/db/migration/000001_init_schema.down.sql: -------------------------------------------------------------------------------- 1 | DO NOT ROLL MIGRATIONS BACK; -------------------------------------------------------------------------------- /api/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | ylem_api_migrations: 3 | env_file: 4 | - .env 5 | - ../.env.common 6 | build: 7 | context: . 8 | dockerfile: Dockerfile 9 | command: /opt/ylem_api/ylem_api db migrations migrate 10 | container_name: ylem_api_migrations 11 | networks: 12 | - ylem_network 13 | links: 14 | - ylem_database 15 | depends_on: 16 | ylem_database: 17 | condition: service_healthy 18 | volumes: 19 | - .:/go/src/ylem_api 20 | working_dir: /go/src/ylem_api 21 | stdin_open: true 22 | tty: true 23 | 24 | ylem_api: 25 | env_file: 26 | - .env 27 | - ../.env.common 28 | build: 29 | context: . 30 | dockerfile: Dockerfile 31 | command: /opt/ylem_api/ylem_api server serve 32 | container_name: ylem_api 33 | networks: 34 | - ylem_network 35 | depends_on: 36 | - ylem_api_migrations 37 | ports: 38 | - "7339:7339" 39 | volumes: 40 | - .:/go/src/ylem_api 41 | working_dir: /go/src/ylem_api 42 | stdin_open: true 43 | tty: true 44 | 45 | networks: 46 | default: 47 | name: ylem_network 48 | external: true 49 | -------------------------------------------------------------------------------- /api/helpers/gorm.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "ylem_api/config" 7 | 8 | "github.com/sirupsen/logrus" 9 | "gorm.io/driver/mysql" 10 | "gorm.io/gorm" 11 | "gorm.io/gorm/logger" 12 | ) 13 | 14 | var gormInstance *gorm.DB 15 | 16 | func init() { 17 | if testing.Testing() { 18 | return 19 | } 20 | 21 | cfg := config.Cfg() 22 | 23 | dsn := fmt.Sprintf( 24 | "%s:%s@tcp(%s:%s)/%s?parseTime=true&multiStatements=true&loc=UTC", 25 | cfg.DB.User, 26 | cfg.DB.Password, 27 | cfg.DB.Host, 28 | cfg.DB.Port, 29 | cfg.DB.Name); 30 | 31 | db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{ 32 | Logger: logger.New( 33 | logrus.New(), 34 | logger.Config{ 35 | IgnoreRecordNotFoundError: true, 36 | }), 37 | }) 38 | if err != nil { 39 | panic(err) 40 | } 41 | gormInstance = db 42 | } 43 | 44 | func GormInstance() *gorm.DB { 45 | return gormInstance 46 | } 47 | -------------------------------------------------------------------------------- /api/helpers/headers.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | ) 7 | 8 | func DecodeHeaders(hjson string) (map[string]string, error) { 9 | headers := make(map[string]string) 10 | if hjson == "" { 11 | return headers, nil 12 | } 13 | 14 | err := json.Unmarshal([]byte(hjson), &headers) 15 | 16 | return headers, err 17 | } 18 | 19 | func SetHeaders(w http.ResponseWriter, headers http.Header) { 20 | for header, vals := range headers { 21 | for _, v := range vals { 22 | w.Header().Add(header, v) 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /api/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "os/signal" 7 | "ylem_api/cli" 8 | "syscall" 9 | "time" 10 | 11 | "github.com/asaskevich/govalidator" 12 | log "github.com/sirupsen/logrus" 13 | ) 14 | 15 | func main() { 16 | loc, _ := time.LoadLocation("UTC") 17 | time.Local = loc 18 | 19 | govalidator.SetFieldsRequiredByDefault(true) 20 | 21 | ctx, cancel := context.WithCancel(context.Background()) 22 | 23 | done := make(chan bool) 24 | go func() { 25 | defer close(done) 26 | err := cli.NewApplication().RunContext(ctx, os.Args) 27 | if err != nil { 28 | log.Fatalf("error running application: %v", err) 29 | } else { 30 | log.Info("Graceful shutdown") 31 | } 32 | }() 33 | 34 | wait := make(chan os.Signal, 1) 35 | signal.Notify(wait, syscall.SIGINT, syscall.SIGTERM) 36 | select { 37 | case <-wait: // wait for SIGINT/SIGTERM 38 | signal.Reset(syscall.SIGINT, syscall.SIGTERM) // resetting signal listener, so that repeated Ctrl+C will exit immediately 39 | cancel() // graceful stop 40 | <-done 41 | 42 | case <-done: 43 | cancel() // graceful stop 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /api/model/command/create_oauth_client.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "crypto/rand" 5 | "encoding/hex" 6 | "ylem_api/model/entity" 7 | "ylem_api/model/persister" 8 | 9 | "github.com/google/uuid" 10 | ) 11 | 12 | type CreateOauthClientCommand struct { 13 | OrganizationUuid uuid.UUID `json:"organization_uuid"` 14 | UserUuid uuid.UUID `json:"user_uuid"` 15 | Name string `json:"name"` 16 | } 17 | 18 | type CreateOauthClientHandler struct { 19 | persister persister.EntityPersister 20 | } 21 | 22 | func (h *CreateOauthClientHandler) Handle(c CreateOauthClientCommand) (uuid.UUID, error) { 23 | s, err := generateSecureToken(32) 24 | if err != nil { 25 | return uuid.Nil, err 26 | } 27 | cl := &entity.OauthClient{ 28 | Uuid: uuid.New(), 29 | UserUuid: c.UserUuid, 30 | OrganizationUuid: c.OrganizationUuid, 31 | Name: c.Name, 32 | Secret: s, 33 | } 34 | 35 | return cl.Uuid, h.persister.SaveOauthClient(cl) 36 | } 37 | 38 | func generateSecureToken(length int) (string, error) { 39 | b := make([]byte, length) 40 | if _, err := rand.Read(b); err != nil { 41 | return "", err 42 | } 43 | return hex.EncodeToString(b), nil 44 | } 45 | 46 | func NewCreateOauthClientHandler() *CreateOauthClientHandler { 47 | h := &CreateOauthClientHandler{ 48 | persister: persister.Instance(), 49 | } 50 | 51 | return h 52 | } 53 | -------------------------------------------------------------------------------- /api/model/command/delete_oauth_client.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "ylem_api/model/persister" 5 | 6 | "github.com/google/uuid" 7 | ) 8 | 9 | type DeleteOauthClientCommand struct { 10 | OrganizationUuid uuid.UUID `json:"organization_uuid"` 11 | UserUuid uuid.UUID `json:"user_uuid"` 12 | } 13 | 14 | type DeleteOauthClientHandler struct { 15 | persister persister.EntityPersister 16 | } 17 | 18 | func (h *DeleteOauthClientHandler) Handle(uid string) (bool, error) { 19 | err := h.persister.DeleteOauthClientByUuid(uid) 20 | if err != nil { 21 | return false, err 22 | } 23 | 24 | err = h.persister.DeleteOauthTokensByClientUuid(uid) 25 | if err != nil { 26 | return false, err 27 | } 28 | 29 | return true, nil 30 | } 31 | 32 | func NewDeleteOauthClientHandler() *DeleteOauthClientHandler { 33 | h := &DeleteOauthClientHandler{ 34 | persister: persister.Instance(), 35 | } 36 | 37 | return h 38 | } 39 | -------------------------------------------------------------------------------- /api/model/repository/error.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import "errors" 4 | 5 | var ErrNotFound = errors.New("record not found") 6 | -------------------------------------------------------------------------------- /api/service/oauth/client_store.go: -------------------------------------------------------------------------------- 1 | package oauth 2 | 3 | import ( 4 | "context" 5 | "ylem_api/model/entity" 6 | "ylem_api/model/persister" 7 | "ylem_api/model/repository" 8 | 9 | "github.com/go-oauth2/oauth2/v4" 10 | "github.com/google/uuid" 11 | ) 12 | 13 | type GormOauthClientStore struct { 14 | repository repository.OauthClientRepository 15 | persister persister.EntityPersister 16 | } 17 | 18 | func (s *GormOauthClientStore) GetByID(ctx context.Context, id string) (oauth2.ClientInfo, error) { 19 | return s.repository.FindByUuid(uuid.MustParse(id)) 20 | } 21 | 22 | func (s *GormOauthClientStore) Create(ctx context.Context, client *entity.OauthClient) (oauth2.ClientInfo, error) { 23 | return client, s.persister.SaveOauthClient(client) 24 | } 25 | 26 | func NewOauthClientStore() oauth2.ClientStore { 27 | return &GormOauthClientStore{ 28 | repository: repository.NewOauthClientRepository(), 29 | persister: persister.Instance(), 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /api/service/oauth/scopes.go: -------------------------------------------------------------------------------- 1 | package oauth 2 | 3 | import "strings" 4 | 5 | const ( 6 | ScopePipelinesRun = "pipelines:run" 7 | ScopeStatsRead = "stats:read" 8 | ) 9 | 10 | func IsScopeGranted(scope, grantedScopes string) bool { 11 | for _, v := range strings.Split(grantedScopes, ",") { 12 | if scope == strings.TrimSpace(v) { 13 | return true 14 | } 15 | } 16 | 17 | return false 18 | } 19 | 20 | func NormalizeScopes(scopes string) []string { 21 | scopeArr := strings.Split(scopes, ",") 22 | for k, v := range scopeArr { 23 | scopeArr[k] = strings.TrimSpace(v) 24 | } 25 | 26 | return scopeArr 27 | } 28 | -------------------------------------------------------------------------------- /api/tests/.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/api/tests/.env -------------------------------------------------------------------------------- /api/tests/service/oauth/.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/api/tests/service/oauth/.env -------------------------------------------------------------------------------- /backend/integrations/.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | insert_final_newline = true 3 | indent_style = tab 4 | -------------------------------------------------------------------------------- /backend/integrations/.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | cover.html 14 | 15 | # Dependency directories (remove the comment below to include it) 16 | # vendor/ 17 | 18 | # Others 19 | .DS_Store 20 | .idea 21 | ylem_integrations 22 | .env.local 23 | migrate 24 | config/keys/id_rsa 25 | config/keys/id_rsa.pub 26 | -------------------------------------------------------------------------------- /backend/integrations/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/b5q6i6w4/ylem-public-images@sha256:c73b7d09874740f2c0df7003954fbbc46310ae30363e4a201d809aac5dff6afc AS builder 2 | 3 | RUN apt-get update && apt-get install -y ca-certificates git curl 4 | 5 | # install Golang-migrate tool 6 | RUN curl -L https://github.com/golang-migrate/migrate/releases/download/v4.15.1/migrate.linux-amd64.tar.gz | tar xvz 7 | RUN cp ./migrate /usr/local/bin 8 | 9 | RUN mkdir /user && \ 10 | echo 'nobody:x:65534:65534:nobody:/:' > /user/passwd && \ 11 | echo 'nobody:x:65534:' > /user/group 12 | 13 | WORKDIR /opt/ylem_integrations 14 | 15 | COPY go.mod ./ 16 | COPY go.sum ./ 17 | 18 | RUN go mod download 19 | 20 | COPY . . 21 | 22 | RUN go build . 23 | 24 | FROM public.ecr.aws/b5q6i6w4/ylem-public-images@sha256:c73b7d09874740f2c0df7003954fbbc46310ae30363e4a201d809aac5dff6afc AS final 25 | 26 | COPY --from=builder /usr/local/bin /usr/local/bin 27 | 28 | COPY --from=builder /user/group /user/passwd /etc/ 29 | 30 | COPY --from=builder /opt /opt 31 | 32 | #USER nobody:nobody 33 | 34 | EXPOSE 7337 35 | 36 | VOLUME /opt/ylem_integrations/config/keys 37 | 38 | WORKDIR /opt/ylem_integrations 39 | 40 | #CMD ["/opt/ylem_integrations/ylem_integrations", "server", "serve"] 41 | -------------------------------------------------------------------------------- /backend/integrations/cli/app.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "ylem_integrations/cli/command/db" 5 | "ylem_integrations/cli/command/kafka" 6 | "ylem_integrations/cli/command/server" 7 | 8 | "github.com/urfave/cli/v2" 9 | ) 10 | 11 | func NewApplication() *cli.App { 12 | return &cli.App{ 13 | Commands: []*cli.Command{ 14 | server.Command, 15 | kafka.Commands, 16 | db.DbCommands, 17 | }, 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /backend/integrations/cli/command/db/db.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import "github.com/urfave/cli/v2" 4 | 5 | var DbCommands = &cli.Command{ 6 | Name: "db", 7 | Usage: "Database management commands", 8 | Subcommands: []*cli.Command{ 9 | MigrationsCommand, 10 | }, 11 | } 12 | -------------------------------------------------------------------------------- /backend/integrations/cli/command/db/migrate.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "ylem_integrations/helpers" 5 | 6 | "github.com/golang-migrate/migrate/v4" 7 | "github.com/golang-migrate/migrate/v4/database/mysql" 8 | _ "github.com/golang-migrate/migrate/v4/source/file" 9 | log "github.com/sirupsen/logrus" 10 | "github.com/urfave/cli/v2" 11 | ) 12 | 13 | var migrateHandler cli.ActionFunc = func(c *cli.Context) error { 14 | log.Info("Applying migrations to database") 15 | 16 | driver, err := mysql.WithInstance(helpers.DbConn(), &mysql.Config{}) 17 | if err != nil { 18 | log.Info("migrate command failed: " + err.Error()) 19 | return err 20 | } 21 | 22 | m, err := migrate.NewWithDatabaseInstance( 23 | "file://db/migration", 24 | "mysql", 25 | driver, 26 | ) 27 | if err != nil { 28 | log.Info("migrate command failed: " + err.Error()) 29 | return err 30 | } 31 | 32 | err = m.Up() 33 | if err == migrate.ErrNoChange { 34 | log.Info("Nothing to migrate") 35 | return nil 36 | } 37 | 38 | return err 39 | } 40 | 41 | var MigrateCommand = &cli.Command{ 42 | Name: "migrate", 43 | Usage: "Migrate the DB to the latest version", 44 | Action: migrateHandler, 45 | } 46 | 47 | var MigrationsCommand = &cli.Command{ 48 | Name: "migrations", 49 | Usage: "Migration-related commands", 50 | Subcommands: []*cli.Command{ 51 | MigrateCommand, 52 | }, 53 | } 54 | -------------------------------------------------------------------------------- /backend/integrations/cli/command/server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "ylem_integrations/config" 5 | "ylem_integrations/services/server" 6 | 7 | log "github.com/sirupsen/logrus" 8 | "github.com/urfave/cli/v2" 9 | ) 10 | 11 | var serveHandler cli.ActionFunc = func(c *cli.Context) error { 12 | log.Debug("serve command called") 13 | return server.NewServer(config.Cfg().Listen).Run(c.Context) 14 | } 15 | 16 | var ServeCommand = &cli.Command{ 17 | Name: "serve", 18 | Description: "Start a HTTP(s) server", 19 | Usage: "Start a HTTP(s) server", 20 | Action: serveHandler, 21 | } 22 | 23 | var Command = &cli.Command{ 24 | Name: "server", 25 | Description: "HTTP(s) server commands", 26 | Usage: "HTTP(s) server commands", 27 | Subcommands: []*cli.Command{ 28 | ServeCommand, 29 | }, 30 | } 31 | -------------------------------------------------------------------------------- /backend/integrations/config/keys/.gitignore: -------------------------------------------------------------------------------- 1 | !.gitignore 2 | !.gitkeep 3 | -------------------------------------------------------------------------------- /backend/integrations/config/keys/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/backend/integrations/config/keys/.gitkeep -------------------------------------------------------------------------------- /backend/integrations/db/migration/000001_init_schema.down.sql: -------------------------------------------------------------------------------- 1 | DO NOT ROLL MIGRATIONS BACK; 2 | -------------------------------------------------------------------------------- /backend/integrations/db/migration/000002_add_whatsapp.down.sql: -------------------------------------------------------------------------------- 1 | DO NOT ROLL MIGRATIONS BACK; 2 | -------------------------------------------------------------------------------- /backend/integrations/db/migration/000002_add_whatsapp.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE `whatsapps` 2 | ( 3 | `id` int(10) unsigned NOT NULL AUTO_INCREMENT, 4 | `integration_id` int(10) unsigned NOT NULL, 5 | `content_sid` BLOB NOT NULL, 6 | `updated_at` timestamp NULL DEFAULT NULL ON UPDATE current_timestamp(), 7 | `created_at` timestamp NOT NULL DEFAULT current_timestamp(), 8 | PRIMARY KEY (`id`), 9 | KEY `integration_id` (`integration_id`), 10 | CONSTRAINT `whatsapps_integration_id` FOREIGN KEY (`integration_id`) REFERENCES `integrations` (`id`) 11 | ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_general_ci; 12 | -------------------------------------------------------------------------------- /backend/integrations/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | ylem_integrations_migrations: 3 | env_file: 4 | - .env 5 | - ../../.env.common 6 | build: 7 | context: . 8 | dockerfile: Dockerfile 9 | command: /opt/ylem_integrations/ylem_integrations db migrations migrate 10 | container_name: ylem_integrations_migrations 11 | networks: 12 | - ylem_network 13 | links: 14 | - ylem_database 15 | depends_on: 16 | ylem_database: 17 | condition: service_healthy 18 | volumes: 19 | - .:/go/src/ylem_integrations 20 | working_dir: /go/src/ylem_integrations 21 | stdin_open: true 22 | tty: true 23 | 24 | ylem_integrations: 25 | env_file: 26 | - .env 27 | - ../../.env.common 28 | build: 29 | context: . 30 | dockerfile: Dockerfile 31 | command: /opt/ylem_integrations/ylem_integrations server serve 32 | container_name: ylem_integrations 33 | networks: 34 | - ylem_network 35 | depends_on: 36 | - ylem_integrations_migrations 37 | ports: 38 | - "7337:7337" 39 | volumes: 40 | - .:/go/src/ylem_integrations 41 | working_dir: /go/src/ylem_integrations 42 | stdin_open: true 43 | tty: true 44 | 45 | networks: 46 | default: 47 | name: ylem_network 48 | external: true 49 | -------------------------------------------------------------------------------- /backend/integrations/entities/Email.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | import "time" 4 | 5 | type Email struct { 6 | Id int64 `json:"-"` 7 | Integration Integration `json:"integration"` 8 | Code string `json:"-"` 9 | IsConfirmed bool `json:"is_confirmed"` 10 | RequestedAt time.Time `json:"requested_at"` 11 | } 12 | 13 | const IntegrationTypeEmail = "email" 14 | 15 | func (e Email) CanResendEmail() bool { 16 | // @todo what criteria? 17 | return !e.IsConfirmed 18 | } 19 | -------------------------------------------------------------------------------- /backend/integrations/entities/GoogleSheets.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | import "ylem_integrations/services/aws/kms" 4 | 5 | const ( 6 | IntegrationTypeGoogleSheets = "google-sheets" 7 | 8 | GoogleSheetsModeOverwrite = "overwrite" 9 | GoogleSheetsModeAppend = "append" 10 | ) 11 | 12 | type GoogleSheets struct { 13 | Id int64 `json:"-"` 14 | Integration Integration `json:"integration"` 15 | SpreadsheetId string `json:"spreadsheet_id"` 16 | SheetId int64 `json:"sheet_id"` 17 | Mode string `json:"mode"` 18 | Credentials *kms.SecretBox `json:"credentials"` 19 | WriteHeader bool `json:"write_header"` 20 | } 21 | -------------------------------------------------------------------------------- /backend/integrations/entities/Hubspot.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | type Hubspot struct { 4 | Id int64 `json:"-"` 5 | Integration Integration `json:"integration"` 6 | HubspotAuthorization HubspotAuthorization `json:"authorization"` 7 | PipelineStageCode string `json:"pipeline_stage_code"` 8 | OwnerCode string `json:"owner_code"` 9 | } 10 | 11 | const IntegrationTypeHubspot = "hubspot" 12 | -------------------------------------------------------------------------------- /backend/integrations/entities/HubspotAuthorization.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | import ( 4 | "time" 5 | "ylem_integrations/services/aws/kms" 6 | ) 7 | 8 | type HubspotAuthorization struct { 9 | Id int64 `json:"-"` 10 | Uuid string `json:"uuid"` 11 | CreatorUuid string `json:"-"` 12 | OrganizationUuid string `json:"-"` 13 | Name string `json:"name"` 14 | State string `json:"-"` 15 | IsActive bool `json:"is_active"` 16 | AccessToken *kms.SecretBox `json:"-"` 17 | AccessTokenExpiresAt time.Time `json:"-"` 18 | RefreshToken *kms.SecretBox `json:"-"` 19 | Scopes *string `json:"-"` 20 | } 21 | -------------------------------------------------------------------------------- /backend/integrations/entities/HubspotAuthorizationCollection.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | type HubspotAuthorizationCollection struct { 4 | Items []HubspotAuthorization 5 | } 6 | -------------------------------------------------------------------------------- /backend/integrations/entities/IncidentIo.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | import "ylem_integrations/services/aws/kms" 4 | 5 | type IncidentIo struct { 6 | Id int64 `json:"-"` 7 | Integration Integration `json:"integration"` 8 | ApiKey *kms.SecretBox `json:"api_key"` 9 | Visibility string `json:"visibility"` 10 | } 11 | 12 | const IntegrationTypeIncidentIo = "incidentio" 13 | -------------------------------------------------------------------------------- /backend/integrations/entities/Integration.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | type Integration struct { 4 | Id int64 `json:"-"` 5 | Uuid string `json:"uuid"` 6 | CreatorUuid string `json:"-"` 7 | OrganizationUuid string `json:"-"` 8 | Status string `json:"status"` 9 | Type string `json:"type"` 10 | IoType string `json:"io_type"` 11 | Name string `json:"name"` 12 | Value string `json:"value"` 13 | UserUpdatedAt string `json:"user_updated_at"` 14 | } 15 | 16 | const IntegrationStatusNew = "new" 17 | const IntegrationStatusOnline = "online" 18 | const IntegrationStatusOffline = "offline" 19 | 20 | const IntegrationIoTypeAll = "all" 21 | const IntegrationIoTypeSQL = "sql" 22 | const IntegrationIoTypeRead = "read" 23 | const IntegrationIoTypeWrite = "write" 24 | const IntegrationIoTypeReadWrite = "read-write" 25 | 26 | func IsIntegrationStatusSupported(Status string) bool { 27 | return Status == IntegrationStatusOnline || Status == IntegrationStatusOffline 28 | } 29 | 30 | func IsIoTypeSupported(Type string) bool { 31 | return map[string]bool{ 32 | IntegrationIoTypeAll: true, 33 | IntegrationIoTypeSQL: true, 34 | IntegrationIoTypeRead: true, 35 | IntegrationIoTypeWrite: true, 36 | IntegrationIoTypeReadWrite: true, 37 | }[Type] 38 | } -------------------------------------------------------------------------------- /backend/integrations/entities/IntegrationCollection.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | type IntegrationCollection struct { 4 | Items []Integration 5 | } 6 | -------------------------------------------------------------------------------- /backend/integrations/entities/Jenkins.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | import "ylem_integrations/services/aws/kms" 4 | 5 | type Jenkins struct { 6 | Id int64 `json:"-"` 7 | Integration Integration `json:"integration"` 8 | BaseUrl string `json:"base_url"` 9 | Token *kms.SecretBox `json:"token"` 10 | } 11 | 12 | const IntegrationTypeJenkins = "jenkins" 13 | -------------------------------------------------------------------------------- /backend/integrations/entities/Jira.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | type Jira struct { 4 | Id int64 `json:"-"` 5 | Integration Integration `json:"integration"` 6 | JiraAuthorization JiraAuthorization `json:"authorization"` 7 | IssueType string `json:"issue_type"` 8 | } 9 | 10 | const IntegrationTypeJira = "jira" 11 | -------------------------------------------------------------------------------- /backend/integrations/entities/JiraAuthorization.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | import "ylem_integrations/services/aws/kms" 4 | 5 | type JiraAuthorization struct { 6 | Id int64 `json:"-"` 7 | Uuid string `json:"uuid"` 8 | CreatorUuid string `json:"-"` 9 | OrganizationUuid string `json:"-"` 10 | Name string `json:"name"` 11 | State string `json:"-"` 12 | IsActive bool `json:"is_active"` 13 | AccessToken *kms.SecretBox `json:"-"` 14 | Cloudid *string `json:"resource_id"` 15 | Scopes *string `json:"-"` 16 | } 17 | -------------------------------------------------------------------------------- /backend/integrations/entities/JiraAuthorizationCollection.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | type JiraAuthorizationCollection struct { 4 | Items []JiraAuthorization 5 | } 6 | -------------------------------------------------------------------------------- /backend/integrations/entities/Opsgenie.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | import "ylem_integrations/services/aws/kms" 4 | 5 | type Opsgenie struct { 6 | Id int64 `json:"-"` 7 | Integration Integration `json:"integration"` 8 | ApiKey *kms.SecretBox `json:"api_key"` 9 | } 10 | 11 | const IntegrationTypeOpsgenie = "opsgenie" 12 | -------------------------------------------------------------------------------- /backend/integrations/entities/Salesforce.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | type Salesforce struct { 4 | Id int64 `json:"-"` 5 | Integration Integration `json:"integration"` 6 | SalesforceAuthorization SalesforceAuthorization `json:"authorization"` 7 | } 8 | 9 | const IntegrationTypeSalesforce = "salesforce" 10 | -------------------------------------------------------------------------------- /backend/integrations/entities/SalesforceAuthorization.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | import ( 4 | "ylem_integrations/services/aws/kms" 5 | ) 6 | 7 | type SalesforceAuthorization struct { 8 | Id int64 `json:"-"` 9 | Uuid string `json:"uuid"` 10 | CreatorUuid string `json:"-"` 11 | OrganizationUuid string `json:"-"` 12 | Name string `json:"name"` 13 | State string `json:"-"` 14 | IsActive bool `json:"is_active"` 15 | AccessToken *kms.SecretBox `json:"-"` 16 | RefreshToken *kms.SecretBox `json:"-"` 17 | Scopes *string `json:"-"` 18 | Domain *string `json:"-"` 19 | } 20 | -------------------------------------------------------------------------------- /backend/integrations/entities/SalesforceAuthorizationCollection.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | type SalesforceAuthorizationCollection struct { 4 | Items []SalesforceAuthorization 5 | } 6 | -------------------------------------------------------------------------------- /backend/integrations/entities/Slack.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | type Slack struct { 4 | Id int64 `json:"-"` 5 | Integration Integration `json:"integration"` 6 | SlackAuthorization SlackAuthorization `json:"authorization"` 7 | SlackChannelId *string `json:"-"` 8 | } 9 | 10 | const IntegrationTypeSlack = "slack" 11 | -------------------------------------------------------------------------------- /backend/integrations/entities/SlackAuthorization.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | type SlackAuthorization struct { 4 | Id int64 `json:"-"` 5 | Name string `json:"name"` 6 | Uuid string `json:"uuid"` 7 | CreatorUuid string `json:"-"` 8 | OrganizationUuid string `json:"-"` 9 | State string `json:"-"` 10 | AccessToken *string `json:"-"` 11 | Scopes *string `json:"-"` 12 | BotUserId *string `json:"-"` 13 | IsActive bool `json:"is_active"` 14 | } 15 | -------------------------------------------------------------------------------- /backend/integrations/entities/SlackAuthorizationCollection.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | type SlackAuthorizationCollection struct { 4 | Items []SlackAuthorization 5 | } 6 | -------------------------------------------------------------------------------- /backend/integrations/entities/Sms.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | import ( 4 | regexp2 "regexp" 5 | "time" 6 | ) 7 | 8 | type Sms struct { 9 | Id int64 `json:"-"` 10 | Integration Integration `json:"integration"` 11 | Code string `json:"-"` 12 | IsConfirmed bool `json:"is_confirmed"` 13 | RequestedAt time.Time `json:"requested_at"` 14 | } 15 | 16 | func (s Sms) CanResendSms() bool { 17 | return !s.IsConfirmed 18 | } 19 | 20 | const IntegrationTypeSms = "sms" 21 | 22 | func IsMobilePhoneValid(Number string) bool { 23 | return regexp2. 24 | MustCompile(`^((\+[0-9]{1,3})|(\+?\([0-9]{1,3}\)))[\s-]?(?:\(0?[0-9]{1,5}\)|[0-9]{1,5})[-\s]?[0-9][\d\s-]{5,7}\s?(?:x[\d-]{0,4})?$`). 25 | MatchString(Number) 26 | } 27 | -------------------------------------------------------------------------------- /backend/integrations/entities/Tableau.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | import "ylem_integrations/services/aws/kms" 4 | 5 | const IntegrationTypeTableau = "tableau" 6 | 7 | const ( 8 | TableauModeOverwrite = "overwrite" 9 | TableauModeAppend = "append" 10 | ) 11 | 12 | type Tableau struct { 13 | Id int64 `json:"-"` 14 | Integration Integration `json:"integration"` 15 | Username *kms.SecretBox `json:"username"` 16 | Password *kms.SecretBox `json:"password"` 17 | Sitename string `json:"site_name"` 18 | ProjectName string `json:"project_name"` 19 | DatasourceName string `json:"datasource_name"` 20 | Mode string `json:"mode"` 21 | } 22 | -------------------------------------------------------------------------------- /backend/integrations/entities/WhatsApp.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | type WhatsApp struct { 4 | Id int64 `json:"-"` 5 | Integration Integration `json:"integration"` 6 | ContentSid string `json:"content_sid"` 7 | } 8 | 9 | const IntegrationTypeWhatsApp = "whatsapp" 10 | -------------------------------------------------------------------------------- /backend/integrations/helpers/db.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "ylem_integrations/config" 7 | ) 8 | 9 | const DB_TIME_TIMESTAMP = "2006-01-02 15:04:05" 10 | 11 | func DbConn() *sql.DB { 12 | config := config.Cfg() 13 | 14 | db, err := sql.Open( 15 | "mysql", 16 | fmt.Sprintf( 17 | "%s:%s@tcp(%s:%s)/%s?parseTime=true&multiStatements=true", 18 | config.DBConfig.User, 19 | config.DBConfig.Password, 20 | config.DBConfig.Host, 21 | config.DBConfig.Port, 22 | config.DBConfig.Name)) 23 | 24 | if err != nil { 25 | panic(err) 26 | } 27 | 28 | return db 29 | } 30 | 31 | func NumRows(rows *sql.Rows) (count int) { 32 | for rows.Next() { 33 | err := rows.Scan(&count) 34 | CheckDbErr(err) 35 | } 36 | return count 37 | } 38 | 39 | func CheckDbErr(err error) { 40 | if err != nil { 41 | panic(err) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /backend/integrations/helpers/rand.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | ) 7 | 8 | const charset = "0123456789" 9 | 10 | var seededRand *rand.Rand = rand.New( 11 | rand.NewSource(time.Now().UnixNano())) 12 | 13 | func StringWithCharset(length int, charset string) string { 14 | b := make([]byte, length) 15 | for i := range b { 16 | b[i] = charset[seededRand.Intn(len(charset))] 17 | } 18 | return string(b) 19 | } 20 | 21 | func CreateRandomNumericString(length int) string { 22 | return StringWithCharset(length, charset) 23 | } 24 | -------------------------------------------------------------------------------- /backend/integrations/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "os/signal" 7 | "syscall" 8 | "time" 9 | "ylem_integrations/cli" 10 | "ylem_integrations/config" 11 | 12 | log "github.com/sirupsen/logrus" 13 | ) 14 | 15 | func main() { 16 | lvl, err := log.ParseLevel(config.Cfg().LogLevel) 17 | if err != nil { 18 | log.Fatal(err) 19 | } 20 | log.SetLevel(lvl) 21 | 22 | loc, _ := time.LoadLocation("UTC") 23 | time.Local = loc 24 | 25 | ctx, cancel := context.WithCancel(context.Background()) 26 | done := make(chan bool) 27 | go func() { 28 | defer close(done) 29 | err = cli.NewApplication().RunContext(ctx, os.Args) 30 | if err != nil { 31 | log.Fatalf("error running application: %v", err) 32 | } else { 33 | log.Info("Graceful shutdown") 34 | } 35 | }() 36 | 37 | wait := make(chan os.Signal, 1) 38 | signal.Notify(wait, syscall.SIGINT, syscall.SIGTERM) 39 | select { 40 | case <-wait: // wait for SIGINT/SIGTERM 41 | signal.Reset(syscall.SIGINT, syscall.SIGTERM) // resetting signal listener, so that repeated Ctrl+C will exit immediately 42 | cancel() // graceful stop 43 | <-done 44 | 45 | case <-done: 46 | cancel() // graceful stop 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /backend/integrations/resources/embedded.go: -------------------------------------------------------------------------------- 1 | package resources 2 | 3 | import "embed" 4 | 5 | //go:embed templates/html 6 | var EmbeddedFileSystem embed.FS 7 | var EmbeddedHtmlTemplates = map[string]string{ 8 | "confirmation_email": "templates/html/confirmation_email.gohtml", 9 | } 10 | -------------------------------------------------------------------------------- /backend/integrations/resources/templates/html/confirmation_email.gohtml: -------------------------------------------------------------------------------- 1 | Thank you for creating a new E-mail integration with Ylem.

2 | 3 | Your email confirmation code {{ .code }} 4 | Please confirm your email by following the link {{ .link }} and entering the code.

5 | 6 | Sincerely yours, 7 | Ylem. 8 | -------------------------------------------------------------------------------- /backend/integrations/services/ConfirmUsersEmail.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "github.com/kelseyhightower/envconfig" 8 | "log" 9 | "net/http" 10 | "ylem_integrations/config" 11 | ) 12 | 13 | func ConfirmUsersEmail(userUuid string) error { 14 | var config config.Config 15 | err := envconfig.Process("", &config) 16 | if err != nil { 17 | log.Println(err.Error()) 18 | 19 | return err 20 | } 21 | 22 | url := config.NetworkConfig.YlemUsersBaseUrl + "private/user/" + userUuid + "/confirm-email"; 23 | 24 | rp, _ := json.Marshal(map[string]string{}) 25 | req, err := http.NewRequest("POST", url, bytes.NewBuffer(rp)) 26 | if err != nil { 27 | log.Println(err.Error()) 28 | return err 29 | } 30 | req.Header.Set("Content-Type", "application/json") 31 | 32 | client := &http.Client{} 33 | resp, err := client.Do(req) 34 | if err != nil { 35 | log.Println(err.Error()) 36 | 37 | return err 38 | } 39 | defer resp.Body.Close() 40 | 41 | if resp.StatusCode == http.StatusOK { 42 | return nil 43 | } 44 | 45 | return errors.New("Failed to confirm user's Email. Error: " + resp.Status) 46 | } 47 | -------------------------------------------------------------------------------- /backend/integrations/services/SmsSender.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "errors" 5 | "github.com/kelseyhightower/envconfig" 6 | "github.com/twilio/twilio-go" 7 | twilioApi "github.com/twilio/twilio-go/rest/api/v2010" 8 | "log" 9 | "ylem_integrations/config" 10 | ) 11 | 12 | func SendSms(ToPhoneNumber string, Text string) error { 13 | var config config.Config 14 | err := envconfig.Process("", &config) 15 | if err != nil { 16 | log.Println(err.Error()) 17 | 18 | return nil 19 | } 20 | 21 | client := twilio.NewRestClientWithParams(twilio.ClientParams{ 22 | Username: config.Twilio.AccountSid, 23 | Password: config.Twilio.AuthToken, 24 | }) 25 | 26 | if client == nil { 27 | log.Println("Could not create Twilio client") 28 | 29 | return errors.New("could not create Twilio client") 30 | } 31 | 32 | params := &twilioApi.CreateMessageParams{} 33 | params.SetTo(ToPhoneNumber) 34 | params.SetFrom(config.Twilio.NumberFrom) 35 | params.SetBody(Text) 36 | 37 | _, err = client.Api.CreateMessage(params) 38 | if err != nil { 39 | log.Println(err.Error()) 40 | 41 | return err 42 | } 43 | 44 | return nil 45 | } 46 | 47 | func SendPhoneNumberVerificationSms(ToPhoneNumber string, Code string) error { 48 | return SendSms(ToPhoneNumber, "Your Ylem verification code " + Code) 49 | } 50 | -------------------------------------------------------------------------------- /backend/integrations/services/es/es.go: -------------------------------------------------------------------------------- 1 | package es 2 | 3 | import ( 4 | "context" 5 | essqlclient "github.com/ylem-co/es-sql-client" 6 | "github.com/go-resty/resty/v2" 7 | ) 8 | 9 | type Connection struct { 10 | ctx context.Context 11 | client *essqlclient.ES 12 | } 13 | 14 | func (c *Connection) Open() error { 15 | return nil 16 | } 17 | 18 | func (c *Connection) Test() error { 19 | _, err := c.client.Version(nil) 20 | 21 | return err 22 | } 23 | 24 | func (c *Connection) Close() error { 25 | return nil 26 | } 27 | 28 | func (c *Connection) Version() (uint8, error) { 29 | return c.client.Version(nil) 30 | } 31 | 32 | func NewConnection(ctx context.Context, url string, user string, password string, version *uint8) (*Connection, error) { 33 | es := essqlclient.CreateWithBaseUrl(ctx, url, nil, func(c *resty.Client) { 34 | if password == "" { 35 | return 36 | } 37 | 38 | c.SetBasicAuth(user, password) 39 | }) 40 | 41 | if version != nil { 42 | _, err := es.Version(version) 43 | 44 | if err != nil { 45 | return nil, err 46 | } 47 | } 48 | 49 | return &Connection{ 50 | ctx: ctx, 51 | client: &es, 52 | }, nil 53 | } 54 | -------------------------------------------------------------------------------- /backend/integrations/services/hubspot.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "github.com/ylem-co/hubspot-client" 5 | "ylem_integrations/config" 6 | ) 7 | 8 | const HubspotTokenSubtractTime = -10 9 | 10 | func init() { 11 | hs := config.Cfg().Hubspot 12 | hubspotclient.Initiate(hubspotclient.Config{ 13 | ClientID: hs.OauthClientId, 14 | ClientSecret: hs.OauthClientSecret, 15 | RedirectUrl: hs.OauthRedirectUri, 16 | Scopes: []string{"tickets"}, 17 | }) 18 | } 19 | -------------------------------------------------------------------------------- /backend/integrations/services/incidentio/client.go: -------------------------------------------------------------------------------- 1 | package incidentio 2 | 3 | import ( 4 | "fmt" 5 | "github.com/go-resty/resty/v2" 6 | log "github.com/sirupsen/logrus" 7 | "net/http" 8 | ) 9 | 10 | var srv *IncidentIo 11 | 12 | type IncidentIo struct { 13 | client *resty.Client 14 | } 15 | 16 | type Severity struct { 17 | Id string `json:"id"` 18 | Name string `json:"name"` 19 | Rank int `json:"rank"` 20 | } 21 | 22 | type ioseverities struct { 23 | Severities []Severity `json:"severities"` 24 | } 25 | 26 | func (i *IncidentIo) GetSeverities(ApiKey string) ([]Severity, error){ 27 | log.Tracef("incident.io: getting severities") 28 | var result ioseverities 29 | response, err := i.client. 30 | R(). 31 | SetAuthToken(ApiKey). 32 | SetResult(&result). 33 | Get("v1/severities") 34 | 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | if response.StatusCode() != http.StatusOK { 40 | log.Debug(string(response.Body())) 41 | 42 | return nil, fmt.Errorf("incident.io: getting severities, expected http 200, got %s", response.Status()) 43 | } 44 | 45 | return result.Severities, nil 46 | } 47 | 48 | func Instance() *IncidentIo { 49 | if srv == nil { 50 | srv = &IncidentIo{ 51 | client: resty.New().SetBaseURL("https://api.incident.io/"), 52 | } 53 | } 54 | 55 | return srv 56 | } 57 | -------------------------------------------------------------------------------- /backend/integrations/services/runner.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | log "github.com/sirupsen/logrus" 5 | "reflect" 6 | 7 | messaging "github.com/ylem-co/shared-messaging" 8 | ) 9 | 10 | func ProcessMessage(task interface{}) { 11 | switch task := task.(type) { 12 | case *messaging.TaskRunResult: 13 | // do something 14 | default: 15 | log.Debugf(`Ignoring the message of type "%s"`, reflect.TypeOf(task).String()) 16 | } 17 | } 18 | 19 | /*func runMeasured(f func() *messaging.TaskRunResult) *messaging.TaskRunResult { 20 | start := time.Now() 21 | tr := f() 22 | tr.ExecutedAt = time.Now() 23 | tr.Duration = time.Since(start) / time.Millisecond 24 | 25 | return tr 26 | }*/ 27 | -------------------------------------------------------------------------------- /backend/integrations/services/salesforce.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "github.com/ylem-co/salesforce-client" 5 | "ylem_integrations/config" 6 | ) 7 | 8 | func init() { 9 | sf := config.Cfg().Salesforce 10 | salesforceclient.Initiate(salesforceclient.Config{ 11 | ClientID: sf.OauthClientId, 12 | ClientSecret: sf.OauthClientSecret, 13 | RedirectUrl: sf.OauthRedirectUri, 14 | Scopes: []string{"api", "chatter_api", "refresh_token"}, 15 | }) 16 | } 17 | -------------------------------------------------------------------------------- /backend/integrations/services/sql/init.go: -------------------------------------------------------------------------------- 1 | package sql 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/go-sql-driver/mysql" 6 | "ylem_integrations/helpers" 7 | "ylem_integrations/helpers/postgresql" 8 | ) 9 | 10 | func init() { 11 | mysql.RegisterDialContext(helpers.MySqlSSHTCPNetName, helpers.SSHDialContextFunc) 12 | 13 | driver := &postgresql.Driver{} 14 | sql.Register("postgres+ssh", driver) 15 | 16 | postgresql.InitSshPool() 17 | } 18 | -------------------------------------------------------------------------------- /backend/integrations/tests/.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/backend/integrations/tests/.env -------------------------------------------------------------------------------- /backend/pipelines/.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | insert_final_newline = true 3 | indent_style = tab 4 | -------------------------------------------------------------------------------- /backend/pipelines/.env: -------------------------------------------------------------------------------- 1 | PIPELINES_DATABASE_NAME=pipelines 2 | 3 | PIPELINES_SYSTEM_ORGANIZATION_UUID=10c45d4a-e909-4946-86b2-c2165241cfde 4 | -------------------------------------------------------------------------------- /backend/pipelines/.gitignore: -------------------------------------------------------------------------------- 1 | # If you prefer the allow list template instead of the deny list, see community template: 2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 3 | # 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | cover.html 17 | 18 | # Dependency directories (remove the comment below to include it) 19 | # vendor/ 20 | 21 | # Go workspace file 22 | .DS_Store 23 | .idea 24 | ylem_pipelines 25 | .env.local 26 | -------------------------------------------------------------------------------- /backend/pipelines/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.23.2-alpine AS builder 2 | 3 | ENV CGO_ENABLED=0 4 | 5 | RUN apk add --no-cache ca-certificates git curl 6 | 7 | # install Golang-migrate tool 8 | # install Golang-migrate tool 9 | RUN curl -L https://github.com/golang-migrate/migrate/releases/download/v4.15.1/migrate.linux-amd64.tar.gz | tar xvz 10 | RUN cp ./migrate /usr/local/bin 11 | 12 | RUN mkdir /user && \ 13 | echo 'nobody:x:65534:65534:nobody:/:' > /user/passwd && \ 14 | echo 'nobody:x:65534:' > /user/group 15 | 16 | WORKDIR /opt/ylem_pipelines 17 | 18 | COPY go.mod ./ 19 | COPY go.sum ./ 20 | 21 | RUN go mod download 22 | 23 | COPY . . 24 | 25 | RUN go build . 26 | 27 | FROM golang:1.23.2-alpine AS final 28 | 29 | COPY --from=builder /usr/local/bin /usr/local/bin 30 | 31 | COPY --from=builder /user/group /user/passwd /etc/ 32 | 33 | COPY --from=builder /opt /opt 34 | 35 | #USER root 36 | 37 | EXPOSE 7336 38 | 39 | WORKDIR /opt/ylem_pipelines 40 | 41 | #CMD ["/opt/ylem_pipelines/ylem_pipelines", "server", "serve"] 42 | -------------------------------------------------------------------------------- /backend/pipelines/app/dashboard/dashboard.go: -------------------------------------------------------------------------------- 1 | package dashboard 2 | 3 | type Dashboard struct { 4 | NumActivePipelines int `json:"num_active_pipelines"` 5 | NumActiveMetrics int `json:"num_active_metrics"` 6 | NumNewPipelines int `json:"num_new_pipelines"` 7 | NumNewMetrics int `json:"num_new_metrics"` 8 | NumRecentlyUpdatedPipelines int `json:"num_recently_updated_pipelines"` 9 | NumRecentlyUpdatedMetrics int `json:"num_recently_updated_metrics"` 10 | NumMyPipelines int `json:"num_my_pipelines"` 11 | NumMyMetrics int `json:"num_my_metrics"` 12 | NumScheduledPipelines int `json:"num_scheduled_pipelines"` 13 | NumExtTriggeredPipelines int `json:"num_externally_triggered_pipelines"` 14 | NumPipelineTemplates int `json:"num_pipeline_templates"` 15 | NumMetricTemplates int `json:"num_metric_templates"` 16 | } 17 | 18 | type GroupedItems struct { 19 | Items []GroupedItem `json:"items"` 20 | } 21 | 22 | type GroupedItem struct { 23 | Count int `json:"count"` 24 | Year int `json:"year"` 25 | Month string `json:"month"` 26 | Week int `json:"week"` 27 | } 28 | 29 | const GroupByMonth = "month" 30 | const GroupByWeek = "week" 31 | -------------------------------------------------------------------------------- /backend/pipelines/app/envvariable/env_variable.go: -------------------------------------------------------------------------------- 1 | package envvariable 2 | 3 | import "regexp" 4 | 5 | type EnvVariable struct { 6 | Id int64 `json:"-"` 7 | Uuid string `json:"uuid"` 8 | Name string `json:"name"` 9 | OrganizationUuid string `json:"organization_uuid"` 10 | Value string `json:"value"` 11 | CreatedAt string `json:"created_at"` 12 | UpdatedAt string `json:"updated_at"` 13 | IsActive int8 `json:"-"` 14 | } 15 | 16 | func IsEnvVariableValValid(envVariableVal string) bool { 17 | regExp, _ := regexp.Compile(`^[0-9_-]*[0-9.]*[a-zA-Z]*[a-zA-Z0-9_-]*$`) 18 | return regExp.MatchString(envVariableVal) 19 | } 20 | 21 | func IsEnvVariableNameValid(envVariableName string) bool { 22 | regExp, _ := regexp.Compile(`^[0-9_-]*[a-zA-Z]*[a-zA-Z0-9_-]*$`) 23 | return regExp.MatchString(envVariableName) 24 | } 25 | -------------------------------------------------------------------------------- /backend/pipelines/app/envvariable/env_variables.go: -------------------------------------------------------------------------------- 1 | package envvariable 2 | 3 | type EnvVariables struct { 4 | Items []EnvVariable `json:"items"` 5 | } 6 | -------------------------------------------------------------------------------- /backend/pipelines/app/folder/Folders.go: -------------------------------------------------------------------------------- 1 | package folder 2 | 3 | type Folders struct { 4 | Items []Folder `json:"items"` 5 | } 6 | -------------------------------------------------------------------------------- /backend/pipelines/app/folder/folder.go: -------------------------------------------------------------------------------- 1 | package folder 2 | 3 | const FolderIsActive = 1 4 | const FolderIsNotActive = 0 5 | 6 | type Folder struct { 7 | Id int64 `json:"-"` 8 | Uuid string `json:"uuid"` 9 | Name string `json:"name"` 10 | Type string `json:"type"` 11 | OrganizationUuid string `json:"organization_uuid"` 12 | ParentUuid string `json:"parent_uuid"` 13 | ParentId int64 `json:"-"` 14 | IsActive int8 `json:"-"` 15 | CreatedAt string `json:"created_at"` 16 | UpdatedAt string `json:"updated_at"` 17 | } 18 | -------------------------------------------------------------------------------- /backend/pipelines/app/pipeline/Pipelines.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | type Pipelines struct { 4 | Items []Pipeline `json:"items"` 5 | } 6 | 7 | type SearchedPipelines struct { 8 | Items []SearchedPipeline `json:"items"` 9 | } 10 | 11 | type PipelineRunsPerMonths struct { 12 | Items []PipelineRunsPerMonth `json:"items"` 13 | } 14 | -------------------------------------------------------------------------------- /backend/pipelines/app/pipeline/common/common.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | const ( 4 | PipelineTypeGeneric = "generic" 5 | PipelineTypeMetric = "metric" 6 | ) 7 | 8 | func IsTypeSupported(Type string) bool { 9 | return map[string]bool{ 10 | PipelineTypeGeneric: true, 11 | PipelineTypeMetric: true, 12 | }[Type] 13 | } 14 | -------------------------------------------------------------------------------- /backend/pipelines/app/pipeline/run/pipeline_run_config.go: -------------------------------------------------------------------------------- 1 | package run 2 | 3 | const ( 4 | IdListTypeEnabled = "enabled" 5 | IdListTypeDisabled = "disabled" 6 | ) 7 | 8 | type PipelineRunConfig struct { 9 | TaskIds IdList `json:"task_ids"` 10 | TaskTriggerIds IdList `json:"task_trigger_ids"` 11 | } 12 | 13 | type IdList struct { 14 | Type string `json:"type"` 15 | Ids []string `json:"ids"` 16 | } 17 | -------------------------------------------------------------------------------- /backend/pipelines/app/pipelinetemplate/shared_workflow.go: -------------------------------------------------------------------------------- 1 | package pipelinetemplate 2 | 3 | type SharedPipeline struct { 4 | Id int64 `json:"-"` 5 | PipelineUuid string `json:"pipeline_uuid"` 6 | OrganizationUuid string `json:"organization_uuid"` 7 | CreatorUuid string `json:"creator_uuid"` 8 | ShareLink string `json:"share_link"` 9 | IsActive int8 `json:"-"` 10 | IsLinkPublished int8 `json:"is_link_published"` 11 | CreatedAt string `json:"-"` 12 | UpdatedAt *string `json:"-"` 13 | } 14 | -------------------------------------------------------------------------------- /backend/pipelines/app/schedule/scheduled_run.go: -------------------------------------------------------------------------------- 1 | package schedule 2 | 3 | import ( 4 | "time" 5 | "ylem_pipelines/app/pipeline/run" 6 | 7 | "github.com/google/uuid" 8 | ) 9 | 10 | type ScheduledRun struct { 11 | Id int64 12 | PipelineRunUuid uuid.UUID 13 | PipelineId int64 14 | Schedule string 15 | Input []byte 16 | EnvVars map[string]interface{} 17 | Config run.PipelineRunConfig 18 | ExecuteAt *time.Time 19 | } 20 | -------------------------------------------------------------------------------- /backend/pipelines/app/task/Tasks.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | type Tasks struct { 4 | Items []Task `json:"items"` 5 | } 6 | 7 | type SearchedTasks struct { 8 | Items []SearchedTask `json:"items"` 9 | } 10 | -------------------------------------------------------------------------------- /backend/pipelines/app/task/result/task_run_result.go: -------------------------------------------------------------------------------- 1 | package result 2 | 3 | import "github.com/google/uuid" 4 | 5 | const ( 6 | StatePending = "pending" 7 | StateExecuted = "executed" 8 | ) 9 | 10 | type TaskRunResult struct { 11 | Id int64 `json:"id"` 12 | State string `json:"state"` 13 | TaskId int64 `json:"-"` 14 | TaskUuid uuid.UUID `json:"task_uuid"` 15 | TaskRunUuid uuid.UUID `json:"task_run_uuid"` 16 | PipelineRunUuid uuid.UUID `json:"pipeline_run_uuid"` 17 | IsSuccessful bool `json:"is_successful"` 18 | Output []byte `json:"output"` 19 | Errors []TaskRunError `json:"errors"` 20 | CreatedAt string `json:"created_at"` 21 | UpdatedAt string `json:"updated_at"` 22 | } 23 | 24 | type TaskRunError struct { 25 | Id int64 `json:"id"` 26 | TaskRunResultId int64 `json:"task_run_result_id"` 27 | Code uint `json:"code"` 28 | Severity string `json:"severity"` 29 | Message string `json:"message"` 30 | } 31 | -------------------------------------------------------------------------------- /backend/pipelines/app/tasktrigger/task_trigger.go: -------------------------------------------------------------------------------- 1 | package tasktrigger 2 | 3 | import ( 4 | "regexp" 5 | "ylem_pipelines/app/tasktrigger/types" 6 | ) 7 | 8 | type TaskTrigger struct { 9 | Id int64 `json:"-"` 10 | Uuid string `json:"uuid"` 11 | PipelineId int64 `json:"-"` 12 | PipelineUuid string `json:"pipeline_uuid"` 13 | TriggerType string `json:"trigger_type"` 14 | Schedule string `json:"schedule"` 15 | TriggerTaskUuid string `json:"trigger_task_uuid"` 16 | TriggeredTaskUuid string `json:"triggered_task_uuid"` 17 | TriggerTaskId int64 `json:"-"` 18 | TriggeredTaskId int64 `json:"-"` 19 | IsActive int8 `json:"-"` 20 | CreatedAt string `json:"created_at"` 21 | UpdatedAt string `json:"updated_at"` 22 | } 23 | 24 | func IsTriggerTypeSupported(Type string) bool { 25 | return map[string]bool{ 26 | types.TriggerTypeSchedule: true, 27 | types.TriggerTypeConditionTrue: true, 28 | types.TriggerTypeConditionFalse: true, 29 | types.TriggerTypeOutput: true, 30 | }[Type] 31 | } 32 | 33 | func IsScheduleValid(schedule string) bool { 34 | scheduleRegex, _ := regexp.Compile(`^(@(annually|yearly|monthly|weekly|daily|hourly|reboot))|(@every (\d+(ns|us|µs|ms|s|m|h))+)|((((\d+,)+\d+|([\d\*]+(\/|-)\d+)|\d+|\*) ?){5,7})$`) 35 | return scheduleRegex.MatchString(schedule) 36 | } 37 | -------------------------------------------------------------------------------- /backend/pipelines/app/tasktrigger/task_triggers.go: -------------------------------------------------------------------------------- 1 | package tasktrigger 2 | 3 | type TaskTriggers struct { 4 | Items []TaskTrigger `json:"items"` 5 | } 6 | -------------------------------------------------------------------------------- /backend/pipelines/app/tasktrigger/types/types.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | const TaskTriggerIsActive = 1 4 | const TaskTriggerIsNotActive = 0 5 | 6 | const TriggerTypeSchedule = "schedule" 7 | const TriggerTypeConditionTrue = "condition_true" 8 | const TriggerTypeConditionFalse = "condition_false" 9 | const TriggerTypeOutput = "output" 10 | -------------------------------------------------------------------------------- /backend/pipelines/cli/app.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "ylem_pipelines/cli/command" 5 | 6 | "github.com/urfave/cli/v2" 7 | ) 8 | 9 | func NewApplication() *cli.App { 10 | return &cli.App{ 11 | Commands: []*cli.Command{ 12 | command.ServerCommands, 13 | command.ScheduleGeneratorCommands, 14 | command.SchedulePublisherCommands, 15 | command.DbCommands, 16 | command.TriggerListenerCommands, 17 | }, 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /backend/pipelines/cli/command/db.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import "github.com/urfave/cli/v2" 4 | 5 | var DbCommands = &cli.Command{ 6 | Name: "db", 7 | Usage: "Database management commands", 8 | Subcommands: []*cli.Command{ 9 | FixturesCommand, 10 | MigrationsCommand, 11 | }, 12 | } 13 | -------------------------------------------------------------------------------- /backend/pipelines/cli/command/migrate.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "ylem_pipelines/helpers" 5 | 6 | "github.com/golang-migrate/migrate/v4" 7 | "github.com/golang-migrate/migrate/v4/database/mysql" 8 | _ "github.com/golang-migrate/migrate/v4/source/file" 9 | "github.com/urfave/cli/v2" 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | var migrateHandler cli.ActionFunc = func(c *cli.Context) error { 14 | log.Info("Applying migrations to database") 15 | 16 | driver, err := mysql.WithInstance(helpers.DbConn(), &mysql.Config{}) 17 | if err != nil { 18 | log.Info("migrate command failed: " + err.Error()) 19 | return err 20 | } 21 | 22 | m, err := migrate.NewWithDatabaseInstance( 23 | "file://db/migration", 24 | "mysql", 25 | driver, 26 | ) 27 | if err != nil { 28 | log.Info("migrate command failed: " + err.Error()) 29 | return err 30 | } 31 | 32 | err = m.Up() 33 | if err == migrate.ErrNoChange { 34 | log.Info("Nothing to migrate") 35 | return nil 36 | } 37 | 38 | return err 39 | } 40 | 41 | var MigrateCommand = &cli.Command{ 42 | Name: "migrate", 43 | Usage: "Migrate the DB to the latest version", 44 | Action: migrateHandler, 45 | } 46 | 47 | var MigrationsCommand = &cli.Command{ 48 | Name: "migrations", 49 | Usage: "Migration-related commands", 50 | Subcommands: []*cli.Command{ 51 | MigrateCommand, 52 | }, 53 | } 54 | -------------------------------------------------------------------------------- /backend/pipelines/cli/command/schedule_publisher.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "ylem_pipelines/services/schedule" 5 | 6 | "github.com/urfave/cli/v2" 7 | ) 8 | 9 | var schedulePublisherStartHandler cli.ActionFunc = func(ctx *cli.Context) error { 10 | p, err := schedule.NewPublisher(ctx.Context) 11 | if err != nil { 12 | return err 13 | } 14 | 15 | return p.Start() 16 | } 17 | 18 | var SchedulePublisherStart = &cli.Command{ 19 | Name: "start", 20 | Usage: "Start schedule publisher", 21 | Action: schedulePublisherStartHandler, 22 | } 23 | 24 | var SchedulePublisherCommands = &cli.Command{ 25 | Name: "schedulepub", 26 | Usage: "Schedule publisher commands", 27 | Subcommands: []*cli.Command{ 28 | SchedulePublisherStart, 29 | }, 30 | } 31 | -------------------------------------------------------------------------------- /backend/pipelines/db/migration/000001_init_schema.down.sql: -------------------------------------------------------------------------------- 1 | DO NOT ROLL MIGRATIONS BACK; 2 | -------------------------------------------------------------------------------- /backend/pipelines/db/migration/000002_pipeline_templates.down.sql: -------------------------------------------------------------------------------- 1 | DO NOT ROLL MIGRATIONS BACK; 2 | -------------------------------------------------------------------------------- /backend/pipelines/helpers/decode_headers.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import "encoding/json" 4 | 5 | func DecodeHeaders(hjson string) (map[string]string, error) { 6 | headers := make(map[string]string) 7 | if hjson == "" { 8 | return headers, nil 9 | } 10 | 11 | err := json.Unmarshal([]byte(hjson), &headers) 12 | 13 | return headers, err 14 | } 15 | -------------------------------------------------------------------------------- /backend/pipelines/helpers/misc.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "fmt" 5 | "encoding/json" 6 | ) 7 | 8 | func DumpFormatted(val interface{}) { 9 | str, _ := json.MarshalIndent(val, "", " ") 10 | fmt.Println(string(str)) 11 | } 12 | -------------------------------------------------------------------------------- /backend/pipelines/helpers/slice.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | func ChunkSlice(sliceLen int, chunkSize int) [][]int { 4 | min := func(a, b int) int { 5 | if a <= b { 6 | return a 7 | } 8 | return b 9 | } 10 | 11 | makeRange := func(min, max int) []int { 12 | a := make([]int, max-min+1) 13 | for i := range a { 14 | a[i] = min + i 15 | } 16 | return a 17 | } 18 | 19 | batches := make([][]int, 0) 20 | for i := 0; i < sliceLen; i += chunkSize { 21 | batches = append(batches, makeRange(i, min(i+chunkSize, sliceLen)-1)) 22 | } 23 | return batches 24 | } 25 | -------------------------------------------------------------------------------- /backend/pipelines/services/kafka/goka.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | const MaxTaskInputLength = 15 * 1024 * 1024 4 | -------------------------------------------------------------------------------- /backend/pipelines/services/messaging/aggregator.go: -------------------------------------------------------------------------------- 1 | package messaging 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "ylem_pipelines/app/task" 7 | 8 | messaging "github.com/ylem-co/shared-messaging" 9 | ) 10 | 11 | type AggregatorTaskMessageFactory struct { 12 | } 13 | 14 | func (f *AggregatorTaskMessageFactory) CreateMessage(trc TaskRunContext) (*messaging.Envelope, error) { 15 | t := trc.Task 16 | impl, ok := t.Implementation.(*task.Aggregator) 17 | if !ok { 18 | return nil, fmt.Errorf( 19 | "wrong task type. Expected %s, got %s", 20 | reflect.TypeOf(&task.Aggregator{}).String(), 21 | reflect.TypeOf(t.Implementation).String(), 22 | ) 23 | } 24 | 25 | task, err := createTaskMessage(trc) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | msg := &messaging.AggregateDataTask{ 31 | Task: task, 32 | Expression: impl.Expression, 33 | VariableName: impl.VariableName, 34 | } 35 | 36 | return messaging.NewEnvelope(msg), nil 37 | } 38 | -------------------------------------------------------------------------------- /backend/pipelines/services/messaging/code.go: -------------------------------------------------------------------------------- 1 | package messaging 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "ylem_pipelines/app/task" 7 | 8 | messaging "github.com/ylem-co/shared-messaging" 9 | ) 10 | 11 | type CodeMessageFactory struct { 12 | } 13 | 14 | func (f *CodeMessageFactory) CreateMessage(trc TaskRunContext) (*messaging.Envelope, error) { 15 | t := trc.Task 16 | impl, ok := t.Implementation.(*task.Code) 17 | if !ok { 18 | return nil, fmt.Errorf( 19 | "wrong task type. Expected %s, got %s", 20 | reflect.TypeOf(&task.Code{}).String(), 21 | reflect.TypeOf(t.Implementation).String(), 22 | ) 23 | } 24 | 25 | task, err := createTaskMessage(trc) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | msg := &messaging.ExecuteCodeTask{ 31 | Task: task, 32 | Code: impl.Code, 33 | } 34 | 35 | return messaging.NewEnvelope(msg), nil 36 | } 37 | -------------------------------------------------------------------------------- /backend/pipelines/services/messaging/condition.go: -------------------------------------------------------------------------------- 1 | package messaging 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "ylem_pipelines/app/task" 7 | 8 | messaging "github.com/ylem-co/shared-messaging" 9 | ) 10 | 11 | type ConditionTaskMessageFactory struct { 12 | } 13 | 14 | func (f *ConditionTaskMessageFactory) CreateMessage(trc TaskRunContext) (*messaging.Envelope, error) { 15 | t := trc.Task 16 | impl, ok := t.Implementation.(*task.Condition) 17 | if !ok { 18 | return nil, fmt.Errorf( 19 | "wrong task type. Expected %s, got %s", 20 | reflect.TypeOf(&task.Condition{}).String(), 21 | reflect.TypeOf(t.Implementation).String(), 22 | ) 23 | } 24 | 25 | task, err := createTaskMessage(trc) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | msg := &messaging.CheckConditionTask{ 31 | Task: task, 32 | Expression: impl.Expression, 33 | } 34 | 35 | return messaging.NewEnvelope(msg), nil 36 | } 37 | -------------------------------------------------------------------------------- /backend/pipelines/services/messaging/errors.go: -------------------------------------------------------------------------------- 1 | package messaging 2 | 3 | type ErrorNonRepeatable struct { 4 | message string 5 | } 6 | 7 | func (e ErrorNonRepeatable) Error() string { 8 | return e.message 9 | } 10 | 11 | type ErrorRepeatable struct { 12 | message string 13 | } 14 | 15 | func (e ErrorRepeatable) Error() string { 16 | return e.message 17 | } 18 | 19 | func NewErrorNonRepeatable(message string) ErrorNonRepeatable { 20 | return ErrorNonRepeatable{ 21 | message: message, 22 | } 23 | } 24 | func NewErrorRepeatable(message string) ErrorRepeatable { 25 | return ErrorRepeatable{ 26 | message: message, 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /backend/pipelines/services/messaging/external_trigger.go: -------------------------------------------------------------------------------- 1 | package messaging 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "ylem_pipelines/app/task" 7 | 8 | messaging "github.com/ylem-co/shared-messaging" 9 | ) 10 | 11 | type ExternalTriggerMessageFactory struct { 12 | } 13 | 14 | func (f *ExternalTriggerMessageFactory) CreateMessage(trc TaskRunContext) (*messaging.Envelope, error) { 15 | t := trc.Task 16 | impl, ok := t.Implementation.(*task.ExternalTrigger) 17 | if !ok { 18 | return nil, fmt.Errorf( 19 | "wrong task type. Expected %s, got %s", 20 | reflect.TypeOf(&task.ExternalTrigger{}).String(), 21 | reflect.TypeOf(t.Implementation).String(), 22 | ) 23 | } 24 | 25 | task, err := createTaskMessage(trc) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | msg := &messaging.ExternalTriggerTask{ 31 | Task: task, 32 | Input: trc.Input, 33 | } 34 | 35 | if string(trc.Input) == "{}" { 36 | msg.Input = []byte(impl.TestData) 37 | } 38 | 39 | return messaging.NewEnvelope(msg), nil 40 | } 41 | -------------------------------------------------------------------------------- /backend/pipelines/services/messaging/filter.go: -------------------------------------------------------------------------------- 1 | package messaging 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "ylem_pipelines/app/task" 7 | 8 | messaging "github.com/ylem-co/shared-messaging" 9 | ) 10 | 11 | type FilterTaskMessageFactory struct { 12 | } 13 | 14 | func (f *FilterTaskMessageFactory) CreateMessage(trc TaskRunContext) (*messaging.Envelope, error) { 15 | t := trc.Task 16 | impl, ok := t.Implementation.(*task.Filter) 17 | if !ok { 18 | return nil, fmt.Errorf( 19 | "wrong task type. Expected %s, got %s", 20 | reflect.TypeOf(&task.Filter{}).String(), 21 | reflect.TypeOf(t.Implementation).String(), 22 | ) 23 | } 24 | 25 | task, err := createTaskMessage(trc) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | msg := &messaging.FilterTask{ 31 | Task: task, 32 | Expression: impl.Expression, 33 | } 34 | 35 | return messaging.NewEnvelope(msg), nil 36 | } 37 | -------------------------------------------------------------------------------- /backend/pipelines/services/messaging/for_each.go: -------------------------------------------------------------------------------- 1 | package messaging 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "ylem_pipelines/app/task" 7 | 8 | messaging "github.com/ylem-co/shared-messaging" 9 | ) 10 | 11 | type ForEachTaskMessageFactory struct { 12 | } 13 | 14 | func (f *ForEachTaskMessageFactory) CreateMessage(trc TaskRunContext) (*messaging.Envelope, error) { 15 | t := trc.Task 16 | _, ok := t.Implementation.(*task.ForEach) 17 | if !ok { 18 | return nil, fmt.Errorf( 19 | "wrong task type. Expected %s, got %s", 20 | reflect.TypeOf(&task.ForEach{}).String(), 21 | reflect.TypeOf(t.Implementation).String(), 22 | ) 23 | } 24 | 25 | task, err := createTaskMessage(trc) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | msg := &messaging.RunForEachTask{ 31 | Task: task, 32 | } 33 | 34 | return messaging.NewEnvelope(msg), nil 35 | } 36 | -------------------------------------------------------------------------------- /backend/pipelines/services/messaging/gpt.go: -------------------------------------------------------------------------------- 1 | package messaging 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "ylem_pipelines/app/task" 7 | 8 | messaging "github.com/ylem-co/shared-messaging" 9 | ) 10 | 11 | type GptMessageFactory struct { 12 | } 13 | 14 | func (f *GptMessageFactory) CreateMessage(trc TaskRunContext) (*messaging.Envelope, error) { 15 | t := trc.Task 16 | impl, ok := t.Implementation.(*task.Gpt) 17 | if !ok { 18 | return nil, fmt.Errorf( 19 | "wrong task type. Expected %s, got %s", 20 | reflect.TypeOf(&task.Gpt{}).String(), 21 | reflect.TypeOf(t.Implementation).String(), 22 | ) 23 | } 24 | 25 | task, err := createTaskMessage(trc) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | msg := &messaging.CallOpenapiGptTask{ 31 | Task: task, 32 | Prompt: impl.Prompt, 33 | } 34 | 35 | return messaging.NewEnvelope(msg), nil 36 | } 37 | -------------------------------------------------------------------------------- /backend/pipelines/services/messaging/merge.go: -------------------------------------------------------------------------------- 1 | package messaging 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "database/sql" 7 | "ylem_pipelines/app/task" 8 | "ylem_pipelines/app/tasktrigger" 9 | 10 | messaging "github.com/ylem-co/shared-messaging" 11 | ) 12 | 13 | type MergeTaskMessageFactory struct { 14 | db *sql.DB 15 | } 16 | 17 | func (f *MergeTaskMessageFactory) CreateMessage(trc TaskRunContext) (*messaging.Envelope, error) { 18 | t := trc.Task 19 | impl, ok := t.Implementation.(*task.Merge) 20 | if !ok { 21 | return nil, fmt.Errorf( 22 | "wrong task type. Expected %s, got %s", 23 | reflect.TypeOf(&task.Merge{}).String(), 24 | reflect.TypeOf(t.Implementation).String(), 25 | ) 26 | } 27 | 28 | task, err := createTaskMessage(trc) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | msg := &messaging.MergeTask{ 34 | Task: task, 35 | FieldNames: impl.FieldNames, 36 | } 37 | 38 | inputCount, err := tasktrigger.GetInputCount(f.db, trc.Task.Id) 39 | msg.Task.Meta.InputCount = inputCount 40 | 41 | return messaging.NewEnvelope(msg), err 42 | } 43 | -------------------------------------------------------------------------------- /backend/pipelines/services/messaging/processor.go: -------------------------------------------------------------------------------- 1 | package messaging 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "ylem_pipelines/app/task" 7 | 8 | messaging "github.com/ylem-co/shared-messaging" 9 | ) 10 | 11 | type ProcessorTaskMessageFactory struct { 12 | } 13 | 14 | func (f *ProcessorTaskMessageFactory) CreateMessage(trc TaskRunContext) (*messaging.Envelope, error) { 15 | t := trc.Task 16 | impl, ok := t.Implementation.(*task.Processor) 17 | if !ok { 18 | return nil, fmt.Errorf( 19 | "wrong task type. Expected %s, got %s", 20 | reflect.TypeOf(&task.Processor{}).String(), 21 | reflect.TypeOf(t.Implementation).String(), 22 | ) 23 | } 24 | 25 | task, err := createTaskMessage(trc) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | msg := &messaging.ProcessDataTask{ 31 | Task: task, 32 | Expression: impl.Expression, 33 | Strategy: impl.Strategy, 34 | } 35 | 36 | return messaging.NewEnvelope(msg), nil 37 | } 38 | -------------------------------------------------------------------------------- /backend/pipelines/services/messaging/run_pipeline.go: -------------------------------------------------------------------------------- 1 | package messaging 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "ylem_pipelines/app/task" 7 | 8 | messaging "github.com/ylem-co/shared-messaging" 9 | ) 10 | 11 | type RunPipelineMessageFactory struct { 12 | } 13 | 14 | func (f *RunPipelineMessageFactory) CreateMessage(trc TaskRunContext) (*messaging.Envelope, error) { 15 | t := trc.Task 16 | impl, ok := t.Implementation.(*task.RunPipeline) 17 | if !ok { 18 | return nil, fmt.Errorf( 19 | "wrong task type. Expected %s, got %s", 20 | reflect.TypeOf(&task.RunPipeline{}).String(), 21 | reflect.TypeOf(t.Implementation).String(), 22 | ) 23 | } 24 | 25 | task, err := createTaskMessage(trc) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | msg := &messaging.RunPipelineTask{ 31 | Task: task, 32 | PipelineToRunUuid: impl.PipelineUuid, 33 | } 34 | 35 | return messaging.NewEnvelope(msg), nil 36 | } 37 | -------------------------------------------------------------------------------- /backend/pipelines/services/messaging/transformer.go: -------------------------------------------------------------------------------- 1 | package messaging 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "ylem_pipelines/app/task" 7 | 8 | messaging "github.com/ylem-co/shared-messaging" 9 | ) 10 | 11 | type TransformerTaskMessageFactory struct { 12 | } 13 | 14 | func (f *TransformerTaskMessageFactory) CreateMessage(trc TaskRunContext) (*messaging.Envelope, error) { 15 | t := trc.Task 16 | impl, ok := t.Implementation.(*task.Transformer) 17 | if !ok { 18 | return nil, fmt.Errorf( 19 | "wrong task type. Expected %s, got %s", 20 | reflect.TypeOf(&task.Transformer{}).String(), 21 | reflect.TypeOf(t.Implementation).String(), 22 | ) 23 | } 24 | 25 | task, err := createTaskMessage(trc) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | msg := &messaging.TransformDataTask{ 31 | Task: task, 32 | Type: impl.Type, 33 | JsonQueryExpression: impl.JsonQueryExpression, 34 | Delimiter: impl.Delimiter, 35 | CastToType: impl.CastToType, 36 | DecodeFormat: impl.DecodeFormat, 37 | EncodeFormat: impl.EncodeFormat, 38 | } 39 | 40 | return messaging.NewEnvelope(msg), nil 41 | } 42 | -------------------------------------------------------------------------------- /backend/pipelines/services/pipeline_connection.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | "net/http" 7 | "encoding/json" 8 | "ylem_pipelines/config" 9 | 10 | "github.com/kelseyhightower/envconfig" 11 | ) 12 | 13 | func UpdatePipelineConnection(organizationUuid string, isPipelineCreated bool) bool { 14 | var config config.Config 15 | err := envconfig.Process("", &config) 16 | if err != nil { 17 | return false 18 | } 19 | 20 | url := strings.Replace(config.NetworkConfig.UpdateConnectionsUrl, "{uuid}", organizationUuid, -1); 21 | 22 | rp, _ := json.Marshal(map[string]bool{"is_pipeline_created": isPipelineCreated}) 23 | var jsonStr = []byte(rp) 24 | req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr)) 25 | if err != nil { 26 | return false 27 | } 28 | 29 | req.Header.Set("Content-Type", "application/json") 30 | 31 | client := &http.Client{} 32 | resp, err := client.Do(req) 33 | if err != nil { 34 | return false 35 | } 36 | defer resp.Body.Close() 37 | 38 | if resp.StatusCode == http.StatusOK { 39 | return true 40 | } else { 41 | return false 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /backend/pipelines/services/provider/pipeline_provider.go: -------------------------------------------------------------------------------- 1 | package provider 2 | 3 | import ( 4 | "database/sql" 5 | "ylem_pipelines/app/pipeline" 6 | "ylem_pipelines/helpers" 7 | ) 8 | 9 | type PipelineProvider interface { 10 | GetPipeline(id int64) (*pipeline.Pipeline, error) 11 | } 12 | 13 | type DbPipelineProvider struct { 14 | db *sql.DB 15 | } 16 | 17 | func (wp *DbPipelineProvider) GetPipeline(id int64) (*pipeline.Pipeline, error) { 18 | return pipeline.GetPipelineById(wp.db, id) 19 | } 20 | 21 | func NewPipelineProvider() *DbPipelineProvider { 22 | return &DbPipelineProvider{ 23 | db: helpers.DbConn(), 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /backend/pipelines/tests/services/messaging/.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/backend/pipelines/tests/services/messaging/.env -------------------------------------------------------------------------------- /backend/statistics/.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | insert_final_newline = true 3 | indent_style = tab 4 | -------------------------------------------------------------------------------- /backend/statistics/.env: -------------------------------------------------------------------------------- 1 | STATISTICS_DB_DSN=clickhouse://dtmnuser:dtmnpassword@ylem_statistics_database:9000/statistics?dial_timeout=200ms 2 | STATISTICS_DB=statistics 3 | STATISTICS_DB_USER=dtmnuser 4 | STATISTICS_DB_PASSWORD=dtmnpassword 5 | -------------------------------------------------------------------------------- /backend/statistics/.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | 9 | # Test binary, built with `go test -c` 10 | *.test 11 | 12 | # Output of the go coverage tool, specifically when used with LiteIDE 13 | *.out 14 | cover.html 15 | 16 | # Dependency directories (remove the comment below to include it) 17 | # vendor/ 18 | 19 | # Others 20 | .DS_Store 21 | .idea 22 | ylem_statistics 23 | database/data/ 24 | .env.local 25 | -------------------------------------------------------------------------------- /backend/statistics/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.23.2-alpine AS builder 2 | 3 | RUN apk add --no-cache ca-certificates git wget 4 | 5 | RUN mkdir /user && \ 6 | echo 'nobody:x:65534:65534:nobody:/:' > /user/passwd && \ 7 | echo 'nobody:x:65534:' > /user/group 8 | 9 | WORKDIR /opt/ylem_statistics 10 | 11 | COPY go.mod ./ 12 | COPY go.sum ./ 13 | 14 | RUN go mod download 15 | 16 | COPY . . 17 | 18 | RUN go build . 19 | 20 | FROM golang:1.23.2-alpine AS final 21 | 22 | RUN apk add --no-cache wget 23 | 24 | COPY --from=builder /user/group /user/passwd /etc/ 25 | 26 | COPY --from=builder /opt /opt 27 | 28 | #USER root 29 | 30 | EXPOSE 7332 31 | 32 | WORKDIR /opt/ylem_statistics 33 | 34 | #CMD ["/opt/ylem_statistics/ylem_statistics", "server", "serve"] 35 | -------------------------------------------------------------------------------- /backend/statistics/Dockerfile-db: -------------------------------------------------------------------------------- 1 | FROM clickhouse/clickhouse-server:24.4.3-alpine 2 | 3 | EXPOSE 9000 4 | -------------------------------------------------------------------------------- /backend/statistics/cli/app.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "ylem_statistics/cli/command/db" 5 | "ylem_statistics/cli/command/server" 6 | "ylem_statistics/cli/command/taskrun" 7 | 8 | "github.com/urfave/cli/v2" 9 | ) 10 | 11 | func NewApplication() *cli.App { 12 | return &cli.App{ 13 | Commands: []*cli.Command{ 14 | db.Command, 15 | server.Command, 16 | taskrun.ResultListenerCommands, 17 | }, 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /backend/statistics/cli/command/db/db.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import "github.com/urfave/cli/v2" 4 | 5 | var Command = &cli.Command{ 6 | Name: "db", 7 | Usage: "Database management commands", 8 | Subcommands: []*cli.Command{ 9 | MigrationsCommand, 10 | FixturesCommand, 11 | }, 12 | } 13 | -------------------------------------------------------------------------------- /backend/statistics/cli/command/db/migrate.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "ylem_statistics/services/db" 5 | _ "ylem_statistics/services/db/migration" 6 | 7 | "github.com/golang-migrate/migrate/v4" 8 | log "github.com/sirupsen/logrus" 9 | "github.com/urfave/cli/v2" 10 | ) 11 | 12 | var migrateHandler cli.ActionFunc = func(c *cli.Context) error { 13 | log.Info("Applying migrations to database") 14 | 15 | m, err := db.NewMigrator() 16 | if err != nil { 17 | log.Info("migrate command failed: " + err.Error()) 18 | return err 19 | } 20 | 21 | err = m.Up() 22 | if err == migrate.ErrNoChange { 23 | log.Info("Nothing to migrate") 24 | return nil 25 | } 26 | 27 | return err 28 | } 29 | 30 | var MigrateCommand = &cli.Command{ 31 | Name: "migrate", 32 | Usage: "Migrate the DB to the latest version", 33 | Action: migrateHandler, 34 | } 35 | 36 | var MigrationsCommand = &cli.Command{ 37 | Name: "migrations", 38 | Usage: "Migration-related commands", 39 | Subcommands: []*cli.Command{ 40 | MigrateCommand, 41 | }, 42 | } 43 | -------------------------------------------------------------------------------- /backend/statistics/cli/command/server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "ylem_statistics/config" 5 | "ylem_statistics/services/server" 6 | 7 | log "github.com/sirupsen/logrus" 8 | "github.com/urfave/cli/v2" 9 | ) 10 | 11 | var serveHandler cli.ActionFunc = func(c *cli.Context) error { 12 | log.Debug("serve command called") 13 | return server.NewServer(c.Context, config.Cfg().Listen).Run() 14 | } 15 | 16 | var ServeCommand = &cli.Command{ 17 | Name: "serve", 18 | Description: "Start a HTTP(s) server", 19 | Usage: "Start a HTTP(s) server", 20 | Action: serveHandler, 21 | } 22 | 23 | var Command = &cli.Command{ 24 | Name: "server", 25 | Description: "HTTP(s) server commands", 26 | Usage: "HTTP(s) server commands", 27 | Subcommands: []*cli.Command{ 28 | ServeCommand, 29 | }, 30 | } 31 | -------------------------------------------------------------------------------- /backend/statistics/database/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/backend/statistics/database/.gitignore -------------------------------------------------------------------------------- /backend/statistics/domain/entity/persister/entity_persister.go: -------------------------------------------------------------------------------- 1 | package persister 2 | 3 | import ( 4 | "sync" 5 | "ylem_statistics/domain/entity" 6 | "ylem_statistics/services/db" 7 | 8 | "gorm.io/gorm" 9 | ) 10 | 11 | type EntityPersister interface { 12 | CreateTaskRun(tr *entity.TaskRun) error 13 | } 14 | 15 | type entityPersister struct { 16 | db *gorm.DB 17 | } 18 | 19 | func (p *entityPersister) CreateTaskRun(tr *entity.TaskRun) error { 20 | result := p.db.Create(tr) 21 | 22 | return result.Error 23 | } 24 | 25 | var instance *entityPersister 26 | var mu = &sync.RWMutex{} 27 | 28 | func Instance() EntityPersister { 29 | if instance == nil { 30 | mu.Lock() 31 | defer mu.Unlock() 32 | instance = newEntityPersister() 33 | } 34 | 35 | return instance 36 | } 37 | 38 | func newEntityPersister() *entityPersister { 39 | dbInstance, err := db.Instance() 40 | if err != nil { 41 | panic(err) 42 | } 43 | return &entityPersister{ 44 | db: dbInstance, 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /backend/statistics/domain/readmodel/common.go: -------------------------------------------------------------------------------- 1 | package readmodel 2 | 3 | type Period string 4 | 5 | const ( 6 | DAY Period = "day" 7 | WEEK Period = "week" 8 | MONTH Period = "month" 9 | QUARTER Period = "quarter" 10 | YEAR Period = "year" 11 | ) 12 | 13 | var ValidPeriods = []interface{}{ 14 | string(DAY), 15 | string(WEEK), 16 | string(MONTH), 17 | string(QUARTER), 18 | string(YEAR), 19 | } 20 | -------------------------------------------------------------------------------- /backend/statistics/helpers/time.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | const DateTimeFormat = "2006-01-02 15:04:05" 4 | -------------------------------------------------------------------------------- /backend/statistics/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "syscall" 7 | "time" 8 | "os/signal" 9 | "ylem_statistics/cli" 10 | "ylem_statistics/config" 11 | 12 | log "github.com/sirupsen/logrus" 13 | ) 14 | 15 | func main() { 16 | lvl, err := log.ParseLevel(config.Cfg().LogLevel) 17 | if err != nil { 18 | log.Fatal(err) 19 | } 20 | log.SetLevel(lvl) 21 | 22 | loc, _ := time.LoadLocation("UTC") 23 | time.Local = loc 24 | 25 | ctx, cancel := context.WithCancel(context.Background()) 26 | done := make(chan bool) 27 | go func() { 28 | defer close(done) 29 | err = cli.NewApplication().RunContext(ctx, os.Args) 30 | if err != nil { 31 | log.Fatalf("error running application: %v", err) 32 | } else { 33 | log.Info("Graceful shutdown") 34 | } 35 | }() 36 | 37 | wait := make(chan os.Signal, 1) 38 | signal.Notify(wait, syscall.SIGINT, syscall.SIGTERM) 39 | select { 40 | case <-wait: // wait for SIGINT/SIGTERM 41 | signal.Reset(syscall.SIGINT, syscall.SIGTERM) // resetting signal listener, so that repeated Ctrl+C will exit immediately 42 | cancel() // graceful stop 43 | <-done 44 | 45 | case <-done: 46 | cancel() // graceful stop 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /backend/statistics/services/db/db.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "time" 5 | "ylem_statistics/config" 6 | 7 | log "github.com/sirupsen/logrus" 8 | "gorm.io/driver/clickhouse" 9 | "gorm.io/gorm" 10 | ) 11 | 12 | var db *gorm.DB 13 | 14 | func Instance() (*gorm.DB, error) { 15 | if db == nil { 16 | var err error 17 | db, err = newInstance() 18 | if err != nil { 19 | return nil, err 20 | } 21 | } 22 | return db, nil 23 | } 24 | 25 | func newInstance() (*gorm.DB, error) { 26 | log.Debug("Creating a new DB connection instance with DSN " + config.Cfg().DB.DSN) 27 | 28 | db, err := gorm.Open(clickhouse.Open(config.Cfg().DB.DSN), &gorm.Config{}) 29 | if err != nil { 30 | log.Debug("DB connection creation failed: " + err.Error()) 31 | return nil, err 32 | } 33 | 34 | sqlDB, err := db.DB() 35 | if err != nil { 36 | log.Debug("DB connection creation failed: " + err.Error()) 37 | return nil, err 38 | } 39 | 40 | sqlDB.SetMaxIdleConns(20) 41 | sqlDB.SetMaxOpenConns(300) 42 | sqlDB.SetConnMaxLifetime(time.Hour) 43 | 44 | log.Debug("New DB connection created") 45 | 46 | return db, nil 47 | } 48 | -------------------------------------------------------------------------------- /backend/statistics/services/db/migration/20231113203511.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "ylem_statistics/config" 5 | "ylem_statistics/services/db" 6 | ) 7 | 8 | func init() { 9 | db.AddMigration( 10 | 20231113203511, 11 | 12 | `CREATE TABLE `+config.Cfg().DB.StatsTable+`( 13 | uuid UUID, 14 | executor_uuid UUID, 15 | organization_uuid UUID, 16 | creator_uuid UUID, 17 | pipeline_uuid UUID, 18 | pipeline_run_uuid UUID, 19 | task_uuid UUID, 20 | task_type String, 21 | output BLOB DEFAULT '', 22 | pipeline_type String DEFAULT '', 23 | metric_value Float64 DEFAULT 0, 24 | is_metric_value_set UInt8 DEFAULT 0, 25 | is_initial_task UInt8, 26 | is_final_task UInt8, 27 | is_successful UInt8, 28 | is_fatal_failure UInt8, 29 | executed_at DateTime64(6, 'UTC'), 30 | duration UInt32, 31 | PRIMARY KEY(uuid) 32 | ) ENGINE = MergeTree 33 | ORDER BY (uuid, executed_at) 34 | `, 35 | 36 | `DROP TABLE `+config.Cfg().DB.StatsTable, 37 | ) 38 | } 39 | -------------------------------------------------------------------------------- /backend/statistics/tests/.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/backend/statistics/tests/.env -------------------------------------------------------------------------------- /backend/users/.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | insert_final_newline = true 3 | indent_style = tab 4 | -------------------------------------------------------------------------------- /backend/users/.env: -------------------------------------------------------------------------------- 1 | USERS_DATABASE_NAME=users 2 | 3 | # If you want to enable user authentication in Ylem's UI through Google, 4 | # you need to configure the following parameters 5 | # More information https://docs.ylem.co/open-source-edition/configuring-integrations-with-.env-variables#user-authentication-with-google 6 | USERS_GOOGLE_CLIENT_ID= 7 | USERS_GOOGLE_CLIENT_SECRET= 8 | USERS_GOOGLE_CALLBACK_URL=http://%%YOUR_DOMAIN_IS_HERE%%/auth/google/callback 9 | -------------------------------------------------------------------------------- /backend/users/.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | cover.html 14 | 15 | # Dependency directories (remove the comment below to include it) 16 | # vendor/ 17 | 18 | # Others 19 | .DS_Store 20 | .idea 21 | ylem_users 22 | config/jwt/private.pem 23 | config/jwt/public.pem 24 | .env.local 25 | -------------------------------------------------------------------------------- /backend/users/api/ActivateUser.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/gorilla/mux" 5 | "net/http" 6 | "ylem_users/entities" 7 | "ylem_users/helpers" 8 | "ylem_users/repositories" 9 | "ylem_users/services" 10 | ) 11 | 12 | func ActivateUser(w http.ResponseWriter, r *http.Request) { 13 | user := r.Context().Value(authUserKey).(*CtxAuthorizedUser) 14 | userUUID := user.UserUuid 15 | 16 | w.Header().Set("Content-Type", "application/json") 17 | 18 | vars := mux.Vars(r) 19 | targetUserUuid := vars["uuid"] 20 | 21 | db := helpers.DbConn() 22 | defer db.Close() 23 | 24 | org, ok := repositories.GetOrganizationByUserUuid(db, targetUserUuid) 25 | if !ok { 26 | helpers.HttpReturnErrorForbidden(w) 27 | return 28 | } 29 | 30 | permissionCheck := services.HttpPermissionCheck{UserUuid: userUUID, OrganizationUuid: org.Uuid, ResourceUuid: targetUserUuid, ResourceType: entities.RESOURCE_USER, Action: entities.ACTION_CREATE} 31 | ok = services.IsUserActionAllowed(permissionCheck) 32 | if !ok { 33 | helpers.HttpReturnErrorForbidden(w) 34 | return 35 | } 36 | 37 | ok = repositories.ActivateUser(db, targetUserUuid) 38 | 39 | if ok { 40 | w.WriteHeader(201) 41 | } else { 42 | w.WriteHeader(500) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /backend/users/api/ConfirmEmail.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/gorilla/mux" 5 | "net/http" 6 | "ylem_users/helpers" 7 | "ylem_users/repositories" 8 | ) 9 | 10 | const isEmailConfirmed = 1 11 | 12 | func ConfirmEmail(w http.ResponseWriter, r *http.Request) { 13 | w.Header().Set("Content-Type", "application/json") 14 | vars := mux.Vars(r) 15 | emailToken := vars["key"] 16 | 17 | db := helpers.DbConn() 18 | defer db.Close() 19 | 20 | user, err := repositories.GetUserByEmailToken(db, emailToken) 21 | 22 | if err != nil { 23 | helpers.HttpReturnErrorInternal(w) 24 | return 25 | } 26 | 27 | if user == nil { 28 | helpers.HttpReturnErrorNotFound(w) 29 | return 30 | } 31 | 32 | if user.IsEmailConfirmed == isEmailConfirmed { 33 | helpers.HttpReturnErrorForbidden(w) 34 | return 35 | } 36 | 37 | err = repositories.UpdateUserIsEmailConfirmed(db, user, isEmailConfirmed) 38 | if err != nil { 39 | helpers.HttpReturnErrorInternal(w) 40 | return 41 | } 42 | w.WriteHeader(http.StatusOK) 43 | } 44 | -------------------------------------------------------------------------------- /backend/users/api/ConfirmEmailInternal.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/gorilla/mux" 5 | "net/http" 6 | "ylem_users/helpers" 7 | "ylem_users/repositories" 8 | ) 9 | 10 | func ConfirmEmailInternal(w http.ResponseWriter, r *http.Request) { 11 | w.Header().Set("Content-Type", "application/json") 12 | vars := mux.Vars(r) 13 | userUUID := vars["uuid"] 14 | 15 | db := helpers.DbConn() 16 | defer db.Close() 17 | 18 | user, ok := repositories.GetUserByUuid(db, userUUID) 19 | 20 | if !ok { 21 | helpers.HttpReturnErrorInternal(w) 22 | return 23 | } 24 | 25 | if user.IsEmailConfirmed == isEmailConfirmed { 26 | helpers.HttpReturnErrorForbidden(w) 27 | return 28 | } 29 | 30 | err := repositories.UpdateUserIsEmailConfirmed(db, &user, isEmailConfirmed) 31 | if err != nil { 32 | helpers.HttpReturnErrorInternal(w) 33 | return 34 | } 35 | w.WriteHeader(http.StatusOK) 36 | } 37 | -------------------------------------------------------------------------------- /backend/users/api/DeleteUser.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/gorilla/mux" 5 | "net/http" 6 | "ylem_users/entities" 7 | "ylem_users/helpers" 8 | "ylem_users/repositories" 9 | "ylem_users/services" 10 | ) 11 | 12 | func DeleteUser(w http.ResponseWriter, r *http.Request) { 13 | user := r.Context().Value(authUserKey).(*CtxAuthorizedUser) 14 | userUUID := user.UserUuid 15 | 16 | w.Header().Set("Content-Type", "application/json") 17 | 18 | vars := mux.Vars(r) 19 | targetUserUuid := vars["uuid"] 20 | 21 | db := helpers.DbConn() 22 | defer db.Close() 23 | 24 | org, ok := repositories.GetOrganizationByUserUuid(db, targetUserUuid) 25 | if !ok { 26 | helpers.HttpReturnErrorForbidden(w) 27 | return 28 | } 29 | 30 | permissionCheck := services.HttpPermissionCheck{UserUuid: userUUID, OrganizationUuid: org.Uuid, ResourceUuid: targetUserUuid, ResourceType: entities.RESOURCE_USER, Action: entities.ACTION_DELETE} 31 | ok = services.IsUserActionAllowed(permissionCheck) 32 | if !ok { 33 | helpers.HttpReturnErrorForbidden(w) 34 | return 35 | } 36 | 37 | ok = repositories.DeleteUser(db, targetUserUuid) 38 | 39 | if ok { 40 | w.WriteHeader(201) 41 | } else { 42 | w.WriteHeader(500) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /backend/users/api/ExternalAuth.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/markbates/goth/gothic" 6 | log "github.com/sirupsen/logrus" 7 | "net/http" 8 | ) 9 | 10 | func ExternalAuth(w http.ResponseWriter, r *http.Request) { 11 | w.Header().Set("Content-Type", "application/json") 12 | 13 | url, err := gothic.GetAuthURL(w, r) 14 | if err != nil { 15 | log.Error(err) 16 | 17 | rp, _ := json.Marshal(map[string]string{"error": "Failed to generate a redirect link"}) 18 | w.WriteHeader(http.StatusBadRequest) 19 | _, err = w.Write(rp) 20 | if err != nil { 21 | log.Error(err) 22 | } 23 | 24 | return 25 | } 26 | 27 | rp, _ := json.Marshal(map[string]string{"url": url}) 28 | w.WriteHeader(http.StatusOK) 29 | _, err = w.Write(rp) 30 | if err != nil { 31 | log.Error(err) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /backend/users/api/GetMyOrganization.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "ylem_users/helpers" 7 | "ylem_users/repositories" 8 | 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | func GetMyOrganization(w http.ResponseWriter, r *http.Request) { 13 | user := r.Context().Value(authUserKey).(*CtxAuthorizedUser) 14 | userUUID := user.UserUuid 15 | 16 | w.Header().Set("Content-Type", "application/json") 17 | 18 | db := helpers.DbConn() 19 | defer db.Close() 20 | 21 | org, ok := repositories.GetOrganizationByUserUuid(db, userUUID) 22 | 23 | if ok { 24 | rp, _ := json.Marshal(org) 25 | w.WriteHeader(http.StatusOK) 26 | _, err := w.Write(rp) 27 | if err != nil { 28 | log.Error(err) 29 | } 30 | } else { 31 | w.WriteHeader(500) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /backend/users/api/GetOrganizationDataKey.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/gorilla/mux" 5 | "net/http" 6 | "ylem_users/helpers" 7 | "ylem_users/repositories" 8 | 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | func GetOrganizationDataKey(w http.ResponseWriter, r *http.Request) { 13 | vars := mux.Vars(r) 14 | organizationUuid := vars["uuid"] 15 | 16 | db := helpers.DbConn() 17 | defer db.Close() 18 | 19 | org, ok := repositories.GetOrganizationByUuid(db, organizationUuid) 20 | if !ok { 21 | helpers.HttpReturnErrorInternal(w) 22 | 23 | return 24 | } 25 | 26 | w.Header().Set("Content-Type", "application/octet-stream") 27 | 28 | _, err := w.Write(org.DataKey) 29 | if err != nil { 30 | log.Error(err) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /backend/users/api/GetPendingInvitationsInOrganization.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/gorilla/mux" 6 | "net/http" 7 | "ylem_users/entities" 8 | "ylem_users/helpers" 9 | "ylem_users/repositories" 10 | "ylem_users/services" 11 | 12 | log "github.com/sirupsen/logrus" 13 | ) 14 | 15 | func GetPendingInvitationsInOrganization(w http.ResponseWriter, r *http.Request) { 16 | user := r.Context().Value(authUserKey).(*CtxAuthorizedUser) 17 | userUUID := user.UserUuid 18 | db := helpers.DbConn() 19 | defer db.Close() 20 | 21 | vars := mux.Vars(r) 22 | organizationUuid := vars["uuid"] 23 | 24 | permissionCheck := services.HttpPermissionCheck{UserUuid: userUUID, OrganizationUuid: organizationUuid, ResourceUuid: "", ResourceType: entities.RESOURCE_INVITATION, Action: entities.ACTION_READ_LIST} 25 | ok := services.IsInvitationActionAllowed(permissionCheck) 26 | if !ok { 27 | helpers.HttpReturnErrorForbidden(w) 28 | return 29 | } 30 | 31 | w.Header().Set("Content-Type", "application/json") 32 | 33 | invitations, ok := repositories.GetPendingInvitationsByOrganizationUuid(db, organizationUuid) 34 | if ok { 35 | response, _ := json.Marshal( 36 | map[string][]entities.InvitationToExpose{"items": invitations}, 37 | ) 38 | 39 | _, err := w.Write(response) 40 | if err != nil { 41 | log.Error(err) 42 | } 43 | } else { 44 | w.WriteHeader(500) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /backend/users/api/GetUsersInOrganization.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/gorilla/mux" 6 | "net/http" 7 | "ylem_users/entities" 8 | "ylem_users/helpers" 9 | "ylem_users/repositories" 10 | "ylem_users/services" 11 | 12 | log "github.com/sirupsen/logrus" 13 | ) 14 | 15 | func GetUsersInOrganization(w http.ResponseWriter, r *http.Request) { 16 | user := r.Context().Value(authUserKey).(*CtxAuthorizedUser) 17 | userUUID := user.UserUuid 18 | db := helpers.DbConn() 19 | defer db.Close() 20 | 21 | vars := mux.Vars(r) 22 | organizationUuid := vars["uuid"] 23 | 24 | permissionCheck := services.HttpPermissionCheck{UserUuid: userUUID, OrganizationUuid: organizationUuid, ResourceUuid: "", ResourceType: entities.RESOURCE_USER, Action: entities.ACTION_READ_LIST} 25 | ok := services.IsUserActionAllowed(permissionCheck) 26 | if !ok { 27 | helpers.HttpReturnErrorForbidden(w) 28 | return 29 | } 30 | 31 | w.Header().Set("Content-Type", "application/json") 32 | 33 | users, ok := repositories.GetUsersByOrganizationUuid(db, organizationUuid) 34 | if ok { 35 | response, _ := json.Marshal( 36 | map[string][]entities.UserToExpose{"items": users}, 37 | ) 38 | 39 | _, err := w.Write(response) 40 | if err != nil { 41 | log.Error(err) 42 | } 43 | } else { 44 | w.WriteHeader(500) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /backend/users/api/IsExternalAuthAvailable.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "net/http" 5 | "ylem_users/config" 6 | ) 7 | 8 | func IsExternalAuthAvailable(w http.ResponseWriter, r *http.Request) { 9 | w.Header().Set("Content-Type", "application/json") 10 | 11 | if config.Cfg().Google.ClientId != "" { 12 | w.WriteHeader(http.StatusOK) 13 | } else { 14 | w.WriteHeader(http.StatusNotFound) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /backend/users/api/ValidateInvitation.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/gorilla/mux" 5 | "net/http" 6 | "ylem_users/helpers" 7 | "ylem_users/repositories" 8 | ) 9 | 10 | func ValidateInvitation(w http.ResponseWriter, r *http.Request) { 11 | 12 | db := helpers.DbConn() 13 | defer db.Close() 14 | 15 | vars := mux.Vars(r) 16 | key := vars["key"] 17 | 18 | w.Header().Set("Content-Type", "application/json") 19 | 20 | ok := repositories.ValidateInvitationByKey(db, key) 21 | if ok { 22 | w.WriteHeader(201) 23 | } else { 24 | w.WriteHeader(404) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /backend/users/cli/app.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "ylem_users/cli/command" 5 | "ylem_users/cli/command/encrypt" 6 | "ylem_users/cli/command/server" 7 | 8 | "github.com/urfave/cli/v2" 9 | ) 10 | 11 | func NewApplication() *cli.App { 12 | return &cli.App{ 13 | Commands: []*cli.Command{ 14 | command.DbCommands, 15 | server.Command, 16 | encrypt.Command, 17 | }, 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /backend/users/cli/command/db.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import "github.com/urfave/cli/v2" 4 | 5 | var DbCommands = &cli.Command{ 6 | Name: "db", 7 | Usage: "Database management commands", 8 | Subcommands: []*cli.Command{ 9 | FixturesCommand, 10 | MigrationsCommand, 11 | }, 12 | } 13 | -------------------------------------------------------------------------------- /backend/users/cli/command/fixtures.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "ylem_users/helpers" 5 | 6 | log "github.com/sirupsen/logrus" 7 | "github.com/urfave/cli/v2" 8 | ) 9 | 10 | var fixtureLoadHandler cli.ActionFunc = func(c *cli.Context) error { 11 | log.Info("Loading fixtures...") 12 | db := helpers.DbConn() 13 | tx, err := db.Begin() 14 | if err != nil { 15 | return err 16 | } 17 | 18 | err = tx.Commit() 19 | if err != nil { 20 | return err 21 | } 22 | 23 | log.Info("Done.") 24 | 25 | return nil 26 | } 27 | 28 | var FixtureLoadCommand = &cli.Command{ 29 | Name: "load", 30 | Usage: "Load fixtures into database", 31 | Action: fixtureLoadHandler, 32 | } 33 | 34 | var FixturesCommand = &cli.Command{ 35 | Name: "fixtures", 36 | Usage: "Fixtures", 37 | Subcommands: []*cli.Command{ 38 | FixtureLoadCommand, 39 | }, 40 | } 41 | -------------------------------------------------------------------------------- /backend/users/cli/command/migrate.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "ylem_users/helpers" 5 | 6 | "github.com/golang-migrate/migrate/v4" 7 | "github.com/golang-migrate/migrate/v4/database/mysql" 8 | _ "github.com/golang-migrate/migrate/v4/source/file" 9 | log "github.com/sirupsen/logrus" 10 | "github.com/urfave/cli/v2" 11 | ) 12 | 13 | var migrateHandler cli.ActionFunc = func(c *cli.Context) error { 14 | log.Info("Applying migrations to database") 15 | 16 | driver, err := mysql.WithInstance(helpers.DbConn(), &mysql.Config{}) 17 | if err != nil { 18 | log.Info("migrate command failed: " + err.Error()) 19 | return err 20 | } 21 | 22 | m, err := migrate.NewWithDatabaseInstance( 23 | "file://db/migration", 24 | "mysql", 25 | driver, 26 | ) 27 | if err != nil { 28 | log.Info("migrate command failed: " + err.Error()) 29 | return err 30 | } 31 | 32 | err = m.Up() 33 | if err == migrate.ErrNoChange { 34 | log.Info("Nothing to migrate") 35 | return nil 36 | } 37 | 38 | return err 39 | } 40 | 41 | var MigrateCommand = &cli.Command{ 42 | Name: "migrate", 43 | Usage: "Migrate the DB to the latest version", 44 | Action: migrateHandler, 45 | } 46 | 47 | var MigrationsCommand = &cli.Command{ 48 | Name: "migrations", 49 | Usage: "Migration-related commands", 50 | Subcommands: []*cli.Command{ 51 | MigrateCommand, 52 | }, 53 | } 54 | -------------------------------------------------------------------------------- /backend/users/cli/command/server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "ylem_users/config" 5 | "ylem_users/services/server" 6 | 7 | log "github.com/sirupsen/logrus" 8 | "github.com/urfave/cli/v2" 9 | ) 10 | 11 | var serveHandler cli.ActionFunc = func(c *cli.Context) error { 12 | log.Debug("serve command called") 13 | return server.NewServer(config.Cfg().Listen).Run(c.Context) 14 | } 15 | 16 | var ServeCommand = &cli.Command{ 17 | Name: "serve", 18 | Description: "Start a HTTP(s) server", 19 | Usage: "Start a HTTP(s) server", 20 | Action: serveHandler, 21 | } 22 | 23 | var Command = &cli.Command{ 24 | Name: "server", 25 | Description: "HTTP(s) server commands", 26 | Usage: "HTTP(s) server commands", 27 | Subcommands: []*cli.Command{ 28 | ServeCommand, 29 | }, 30 | } 31 | -------------------------------------------------------------------------------- /backend/users/config/jwt/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/backend/users/config/jwt/.gitkeep -------------------------------------------------------------------------------- /backend/users/db/migration/000001_init_schema.down.sql: -------------------------------------------------------------------------------- 1 | DO NOT ROLL MIGRATIONS BACK; 2 | -------------------------------------------------------------------------------- /backend/users/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | ylem_users_migrations: 3 | env_file: 4 | - .env 5 | - ../../.env.common 6 | build: 7 | context: . 8 | dockerfile: Dockerfile 9 | command: /opt/ylem_users/ylem_users db migrations migrate 10 | container_name: ylem_users_migrations 11 | networks: 12 | - ylem_network 13 | depends_on: 14 | - ylem_session_storage 15 | volumes: 16 | - .:/go/src/ylem_users 17 | working_dir: /go/src/ylem_users 18 | stdin_open: true 19 | tty: true 20 | 21 | ylem_users: 22 | env_file: 23 | - .env 24 | - ../../.env.common 25 | build: 26 | context: . 27 | dockerfile: Dockerfile 28 | command: /opt/ylem_users/ylem_users server serve 29 | container_name: ylem_users 30 | networks: 31 | - ylem_network 32 | depends_on: 33 | - ylem_session_storage 34 | - ylem_users_migrations 35 | ports: 36 | - "7333:7333" 37 | volumes: 38 | - .:/go/src/ylem_users 39 | working_dir: /opt/ylem_users 40 | stdin_open: true 41 | tty: true 42 | 43 | networks: 44 | default: 45 | name: ylem_network 46 | external: true 47 | -------------------------------------------------------------------------------- /backend/users/entities/Action.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | const ACTION_CREATE = "create" 4 | const ACTION_READ = "read" 5 | const ACTION_READ_LIST = "read_list" 6 | const ACTION_UPDATE = "update" 7 | const ACTION_DELETE = "delete" 8 | const ACTION_RUN = "run" 9 | 10 | func IsActionValid(action string) bool { 11 | switch action { 12 | case 13 | ACTION_CREATE, 14 | ACTION_READ, 15 | ACTION_READ_LIST, 16 | ACTION_UPDATE, 17 | ACTION_RUN, 18 | ACTION_DELETE: 19 | return true 20 | } 21 | return false 22 | } 23 | -------------------------------------------------------------------------------- /backend/users/entities/Invitations.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | type InvitationToExpose struct { 4 | Uuid string `json:"uuid"` 5 | Email string `json:"email"` 6 | CreatedAt string `json:"created_at"` 7 | InvitationCode string `json:"invitation_code"` 8 | } 9 | -------------------------------------------------------------------------------- /backend/users/entities/Organization.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | type Organization struct { 4 | Id int `json:"id"` 5 | Uuid string `json:"uuid"` 6 | Name string `json:"name"` 7 | IsDataSourceCreated bool `json:"is_data_source_created"` 8 | IsDestinationCreated bool `json:"is_destination_created"` 9 | IsPipelineCreated bool `json:"is_pipeline_created"` 10 | DataKey []byte `json:"-"` 11 | } 12 | -------------------------------------------------------------------------------- /backend/users/entities/Resources.go: -------------------------------------------------------------------------------- 1 | package entities 2 | 3 | const RESOURCE_USER = "user" 4 | const RESOURCE_ORGANIZATION = "organization" 5 | const RESOURCE_PIPELINE = "pipeline" 6 | const RESOURCE_METRICS = "metrics" 7 | const RESOURCE_TASK = "task" 8 | const RESOURCE_STAT = "stat" 9 | const RESOURCE_INTEGRATION = "integration" 10 | const RESOURCE_INVITATION = "invitation" 11 | const RESOURCE_FOLDER = "folder" 12 | const RESOURCE_ENVVARIABLE = "envvariable" 13 | const RESOURCE_OAUTH_CLIENT = "oauth_client" 14 | const RESOURCE_SUBSCRIPTION_PLAN = "subscription_plan" 15 | const RESOURCE_SUBSCRIPTION = "subscription" 16 | const RESOURCE_CHECKOUT_SESSION = "checkout_session" 17 | const RESOURCE_API_CALL = "api_call" 18 | -------------------------------------------------------------------------------- /backend/users/helpers/db.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "fmt" 7 | "ylem_users/config" 8 | 9 | "github.com/go-redis/redis/v8" 10 | ) 11 | 12 | func DbConn() *sql.DB { 13 | config := config.Cfg() 14 | 15 | db, err := sql.Open( 16 | "mysql", 17 | fmt.Sprintf( 18 | "%s:%s@tcp(%s:%s)/%s?parseTime=true&multiStatements=true", 19 | config.DBConfig.User, 20 | config.DBConfig.Password, 21 | config.DBConfig.Host, 22 | config.DBConfig.Port, 23 | config.DBConfig.Name)) 24 | 25 | if err != nil { 26 | panic(err) 27 | } 28 | 29 | return db 30 | } 31 | 32 | func NumRows(rows *sql.Rows) (count int) { 33 | for rows.Next() { 34 | err := rows.Scan(&count) 35 | CheckDbErr(err) 36 | } 37 | return count 38 | } 39 | 40 | func CheckDbErr(err error) { 41 | if err != nil { 42 | panic(err) 43 | } 44 | } 45 | 46 | func RedisDbConn(ctx context.Context) *redis.Client { 47 | c := config.Cfg() 48 | 49 | rdb := redis.NewClient(&redis.Options{ 50 | Addr: fmt.Sprintf("%s:%s", c.RedisDBConfig.Host, c.RedisDBConfig.Port), 51 | Password: c.RedisDBConfig.Password, 52 | DB: 0, 53 | }).WithContext(ctx) 54 | 55 | _, err := rdb.Ping(ctx).Result() 56 | if err != nil { 57 | panic(err) 58 | } 59 | 60 | return rdb 61 | } 62 | -------------------------------------------------------------------------------- /backend/users/helpers/randSeq.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | ) 7 | 8 | var seededRand *rand.Rand = rand.New( 9 | rand.NewSource(time.Now().UnixNano())) 10 | 11 | var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890") 12 | 13 | func RandSeq(n int) string { 14 | b := make([]rune, n) 15 | for i := range b { 16 | b[i] = letters[seededRand.Intn(len(letters))] 17 | } 18 | return string(b) 19 | } 20 | 21 | func CreateRandomNumericString(length int) string { 22 | b := make([]rune, length) 23 | for i := range b { 24 | b[i] = letters[seededRand.Intn(len(letters))] 25 | } 26 | return string(b) 27 | } 28 | -------------------------------------------------------------------------------- /backend/users/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "os/signal" 7 | "ylem_users/cli" 8 | "ylem_users/config" 9 | "syscall" 10 | "time" 11 | 12 | log "github.com/sirupsen/logrus" 13 | ) 14 | 15 | func main() { 16 | lvl, err := log.ParseLevel(config.Cfg().LogLevel) 17 | if err != nil { 18 | log.Fatal(err) 19 | } 20 | log.SetLevel(lvl) 21 | 22 | loc, _ := time.LoadLocation("UTC") 23 | time.Local = loc 24 | 25 | ctx, cancel := context.WithCancel(context.Background()) 26 | done := make(chan bool) 27 | go func() { 28 | defer close(done) 29 | err = cli.NewApplication().RunContext(ctx, os.Args) 30 | if err != nil { 31 | log.Fatalf("error running application: %v", err) 32 | } else { 33 | log.Info("Graceful shutdown") 34 | } 35 | }() 36 | 37 | wait := make(chan os.Signal, 1) 38 | signal.Notify(wait, syscall.SIGINT, syscall.SIGTERM) 39 | select { 40 | case <-wait: // wait for SIGINT/SIGTERM 41 | signal.Reset(syscall.SIGINT, syscall.SIGTERM) // resetting signal listener, so that repeated Ctrl+C will exit immediately 42 | cancel() // graceful stop 43 | <-done 44 | 45 | case <-done: 46 | cancel() // graceful stop 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /backend/users/services/CreateTrialPipelines.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "log" 8 | "net/http" 9 | "ylem_users/config" 10 | ) 11 | 12 | func CreateTrialPipelines(organizationUuid string, destinationUuid string, sourceUuid string, token string) error { 13 | url := config.Cfg().NetworkConfig.YlemPipelinesBaseUrl + "pipeline/trials" 14 | 15 | rp, _ := json.Marshal(map[string]string{"organization_uuid": organizationUuid, "destination_uuid": destinationUuid, "source_uuid": sourceUuid}) 16 | req, err := http.NewRequest("POST", url, bytes.NewBuffer(rp)) 17 | if err != nil { 18 | log.Println(err.Error()) 19 | return err 20 | } 21 | 22 | req.Header.Set("Content-Type", "application/json") 23 | req.Header.Set("Authorization", "Bearer " + token) 24 | 25 | client := &http.Client{} 26 | resp, err := client.Do(req) 27 | if err != nil { 28 | log.Println(err.Error()) 29 | 30 | return err 31 | } 32 | defer resp.Body.Close() 33 | 34 | if resp.StatusCode == http.StatusOK { 35 | return nil 36 | } 37 | 38 | return errors.New("Failed to create trial pipelines. Error: " + resp.Status) 39 | } 40 | -------------------------------------------------------------------------------- /backend/users/services/TestTrialDBDataSource.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "log" 7 | "net/http" 8 | "ylem_users/config" 9 | ) 10 | 11 | func TestTrialDBDataSource(sourceUuid string, token string) bool { 12 | url := config.Cfg().NetworkConfig.YlemIntegrationsBaseUrl + "integration/sql/" + sourceUuid + "/test"; 13 | 14 | rp, _ := json.Marshal("{}") 15 | req, err := http.NewRequest("POST", url, bytes.NewBuffer(rp)) 16 | if err != nil { 17 | log.Println(err.Error()) 18 | return false 19 | } 20 | req.Header.Set("Content-Type", "application/json") 21 | req.Header.Set("Authorization", "Bearer " + token) 22 | 23 | client := &http.Client{} 24 | resp, err := client.Do(req) 25 | if err != nil { 26 | log.Println(err.Error()) 27 | return false 28 | } 29 | defer resp.Body.Close() 30 | 31 | return resp.StatusCode == http.StatusOK 32 | } 33 | -------------------------------------------------------------------------------- /backend/users/services/Validators.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "regexp" 5 | "unicode" 6 | ) 7 | 8 | func IsPhoneValid(phone string) bool { 9 | phoneRegex := regexp.MustCompile(`^(?:(?:\(?(?:00|\+)([1-4]\d\d|[1-9]\d?)\)?)?[\-\.\ \\\/]?)?((?:\(?\d{1,}\)?[\-\.\ \\\/]?){0,})(?:[\-\.\ \\\/]?(?:#|ext\.?|extension|x)[\-\.\ \\\/]?(\d+))?$`) 10 | return phoneRegex.MatchString(phone) 11 | } 12 | 13 | func IsPasswordValid(pwd string) bool { 14 | symbols := 0 15 | number := false 16 | upper := false 17 | special := false 18 | for _, c := range pwd { 19 | switch { 20 | case unicode.IsNumber(c): 21 | number = true 22 | case unicode.IsUpper(c): 23 | upper = true 24 | case unicode.IsPunct(c) || unicode.IsSymbol(c): 25 | special = true 26 | case unicode.IsLetter(c) || c == ' ': 27 | default: 28 | //return 29 | } 30 | symbols++ 31 | } 32 | 33 | if symbols >= 8 && number && upper && special { 34 | return true 35 | } else { 36 | return false 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /backend/users/services/kms/kms.go: -------------------------------------------------------------------------------- 1 | package kms 2 | 3 | import ( 4 | "context" 5 | "ylem_users/config" 6 | 7 | awsconfig "github.com/aws/aws-sdk-go-v2/config" 8 | awskms "github.com/aws/aws-sdk-go-v2/service/kms" 9 | awstypes "github.com/aws/aws-sdk-go-v2/service/kms/types" 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | var svc *awskms.Client 14 | 15 | func IssueDataKeyWithContext(ctx context.Context) ([]byte, error) { 16 | masterKeyId := config.Cfg().Aws.KmsKeyId 17 | 18 | if len(masterKeyId) > 0 { 19 | keyOutput, err := svc.GenerateDataKey(ctx, &awskms.GenerateDataKeyInput{ 20 | KeyId: &masterKeyId, 21 | KeySpec: awstypes.DataKeySpecAes256, 22 | }) 23 | 24 | if err != nil { 25 | log.Error("data key generation error: " + err.Error()) 26 | 27 | return nil, err 28 | } 29 | 30 | return keyOutput.CiphertextBlob, nil 31 | } else { 32 | return nil, nil 33 | } 34 | } 35 | 36 | func init() { 37 | cfg, err := awsconfig.LoadDefaultConfig(context.Background()) 38 | if err != nil { 39 | panic("aws configuration error: " + err.Error()) 40 | } 41 | 42 | svc = awskms.NewFromConfig(cfg) 43 | } 44 | -------------------------------------------------------------------------------- /backend/users/tests/entities/.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/backend/users/tests/entities/.env -------------------------------------------------------------------------------- /backend/users/tests/entities/Action_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "ylem_users/entities" 5 | "testing" 6 | ) 7 | 8 | func TestIsActionValid(t *testing.T) { 9 | type args struct { 10 | action string 11 | } 12 | tests := []struct { 13 | name string 14 | args args 15 | want bool 16 | }{ 17 | {"Valid 1", args{action: entities.ACTION_CREATE}, true}, 18 | {"Valid 2", args{action: entities.ACTION_READ}, true}, 19 | {"Valid 3", args{action: entities.ACTION_READ_LIST}, true}, 20 | {"Valid 4", args{action: entities.ACTION_UPDATE}, true}, 21 | {"Valid 5", args{action: entities.ACTION_RUN}, true}, 22 | {"Valid 6", args{action: entities.ACTION_DELETE}, true}, 23 | {"Invalid 1", args{action: "Some random action type"}, false}, 24 | } 25 | for _, tt := range tests { 26 | t.Run(tt.name, func(t *testing.T) { 27 | if got := entities.IsActionValid(tt.args.action); got != tt.want { 28 | t.Errorf("IsActionValid() = %v, want %v", got, tt.want) 29 | } 30 | }) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /backend/users/tests/services/.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/backend/users/tests/services/.env -------------------------------------------------------------------------------- /database/.gitignore: -------------------------------------------------------------------------------- 1 | # If you prefer the allow list template instead of the deny list, see community template: 2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 3 | # 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Dependency directories (remove the comment below to include it) 18 | # vendor/ 19 | 20 | # Go workspace file 21 | go.work 22 | .DS_Store 23 | .idea/ -------------------------------------------------------------------------------- /processor/python_processor/.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | insert_final_newline = true 3 | indent_style = tab 4 | -------------------------------------------------------------------------------- /processor/python_processor/.gitignore: -------------------------------------------------------------------------------- 1 | # If you prefer the allow list template instead of the deny list, see community template: 2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 3 | # 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Dependency directories (remove the comment below to include it) 18 | # vendor/ 19 | 20 | # Go workspace file 21 | go.work 22 | .DS_Store 23 | __pycache__/ 24 | -------------------------------------------------------------------------------- /processor/python_processor/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.12-alpine 2 | 3 | RUN apk add build-base libressl-dev libffi-dev python3-dev 4 | 5 | WORKDIR /opt/ylem_python_processor 6 | COPY . . 7 | RUN pip install -r requirements.txt 8 | 9 | EXPOSE 7338 10 | 11 | CMD ["uvicorn", "--host", "0.0.0.0", "--port", "7338", "main:app"] 12 | -------------------------------------------------------------------------------- /processor/python_processor/README.md: -------------------------------------------------------------------------------- 1 | # YLEM PYTHON CODE PROCESSOR 2 | 3 | ![Static Badge](https://img.shields.io/badge/license-Apache%202.0-black) 4 | ![Static Badge](https://img.shields.io/badge/website-ylem.co-black) 5 | ![Static Badge](https://img.shields.io/badge/documentation-docs.ylem.co-black) 6 | ![Static Badge](https://img.shields.io/badge/community-join%20Slack-black) 7 | 8 | Python code processor is an API for evaluating Python expressions from the "Code" pipeline task. It executes arbitrary Python code and returns the results. 9 | 10 | It is available inside the Ylem network on http://ylem_python_processor:7338 or from the host machine on http://127.0.0.1:7338. 11 | 12 | # Endpoints 13 | 14 | ## POST /eval 15 | 16 | ### Request body: 17 | 18 | ```js 19 | { 20 | "code": "input['value'] = 2", // the code to execute 21 | "input": "{\"value\": 1}" // input value, available in the code as "input" variable 22 | } 23 | ``` 24 | 25 | ### Response body: 26 | ```js 27 | { 28 | "statusCode": 200, 29 | "body": "{\"value\": 2}" // execution result 30 | } 31 | ``` 32 | -------------------------------------------------------------------------------- /processor/python_processor/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | ylem_python_processor: 3 | build: 4 | context: . 5 | dockerfile: Dockerfile 6 | container_name: ylem_python_processor 7 | networks: 8 | - ylem_network 9 | ports: 10 | - "7338:7338" 11 | volumes: 12 | - .:/opt/ylem_python_processor 13 | working_dir: /opt/ylem_python_processor 14 | stdin_open: true 15 | tty: true 16 | 17 | networks: 18 | default: 19 | name: ylem_network 20 | external: true 21 | -------------------------------------------------------------------------------- /processor/python_processor/requirements.txt: -------------------------------------------------------------------------------- 1 | AccessControl==6.3 2 | fastapi==0.108.0 3 | uvicorn[standard]==0.25.0 4 | RestrictedPython==7.0 5 | cython==3.0.2 6 | pandas==2.1.4 7 | -------------------------------------------------------------------------------- /processor/taskrunner/.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | insert_final_newline = true 3 | indent_style = tab 4 | -------------------------------------------------------------------------------- /processor/taskrunner/.env: -------------------------------------------------------------------------------- 1 | TASK_RUNNER_LISTEN=0.0.0.0:7335 2 | TASK_RUNNER_GOPYK_BASE_URL=http://ylem_python_processor:7338/eval 3 | 4 | # To enable Tableau integration, install https://github.com/ylem-co/tableau-http-wrapper 5 | # And place its URL here 6 | # By default it assumes that it is running on the port 7890 on your host machine 7 | TASK_RUNNER_TABLEAU_HTTP_WRAPPER_BASE_URL=http://host.docker.internal:7890 8 | 9 | # To enable ChatGPT integration, create its API secret key 10 | # And place it here 11 | # More information: https://docs.ylem.co/pipelines/tasks-ip/gpt 12 | TASK_RUNNER_OPENAI_GPT_KEY= 13 | TASK_RUNNER_OPENAI_MODEL=gpt-4o-mini 14 | 15 | # To enable Gemini integration, create its API secret key 16 | # And place it here 17 | TASK_RUNNER_GEMINI_KEY= 18 | TASK_RUNNER_GEMINI_MODEL=gemini-2.0-flash 19 | 20 | # To tell Ylem which AI provider it should use, write it here. Possible values: openai | gemini 21 | TASK_RUNNER_AI_PROVIDER=openai 22 | -------------------------------------------------------------------------------- /processor/taskrunner/.gitignore: -------------------------------------------------------------------------------- 1 | # If you prefer the allow list template instead of the deny list, see community template: 2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 3 | # 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | cover.html 17 | 18 | # Dependency directories (remove the comment below to include it) 19 | # vendor/ 20 | 21 | # Go workspace file 22 | go.work 23 | .DS_Store 24 | #.env 25 | ylem_taskrunner 26 | config/keys/id_rsa 27 | config/keys/id_rsa.pub 28 | -------------------------------------------------------------------------------- /processor/taskrunner/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/b5q6i6w4/ylem-public-images@sha256:c73b7d09874740f2c0df7003954fbbc46310ae30363e4a201d809aac5dff6afc AS builder 2 | 3 | RUN apt-get update && apt-get install -y ca-certificates git curl 4 | 5 | RUN mkdir /user && \ 6 | echo 'nobody:x:65534:65534:nobody:/:' > /user/passwd && \ 7 | echo 'nobody:x:65534:' > /user/group 8 | 9 | WORKDIR /opt/ylem_taskrunner 10 | 11 | COPY go.mod ./ 12 | COPY go.sum ./ 13 | 14 | RUN go mod download 15 | 16 | COPY . . 17 | 18 | RUN go build . 19 | 20 | FROM public.ecr.aws/b5q6i6w4/ylem-public-images@sha256:c73b7d09874740f2c0df7003954fbbc46310ae30363e4a201d809aac5dff6afc AS final 21 | 22 | COPY --from=builder /user/group /user/passwd /etc/ 23 | 24 | COPY --from=builder /opt /opt 25 | 26 | USER nobody:nobody 27 | 28 | EXPOSE 7335 29 | 30 | WORKDIR /opt/ylem_taskrunner 31 | 32 | #CMD ["/opt/ylem_taskrunner/ylem_taskrunner", "loadbalancer", "start"] 33 | -------------------------------------------------------------------------------- /processor/taskrunner/api/example_handler.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "fmt" 5 | "encoding/json" 6 | "net/http" 7 | "ylem_taskrunner/helpers" 8 | 9 | "github.com/gorilla/mux" 10 | ) 11 | 12 | var ExampleHandler = func(w http.ResponseWriter, r *http.Request) { 13 | vars := mux.Vars(r) 14 | 15 | response, err := json.Marshal(&struct { 16 | Message string 17 | }{ 18 | Message: fmt.Sprintf("Hello, %s!", vars["name"]), 19 | }) 20 | 21 | if err != nil { 22 | helpers.HttpReturnErrorInternal(w) 23 | return 24 | } 25 | 26 | w.Header().Set("Content-Type", "application/json") 27 | _, err = w.Write(response) 28 | if err != nil { 29 | helpers.HttpReturnErrorInternal(w) 30 | return 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /processor/taskrunner/cli/app.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "ylem_taskrunner/cli/command" 5 | "ylem_taskrunner/cli/command/server" 6 | 7 | "github.com/urfave/cli/v2" 8 | ) 9 | 10 | func NewApplication() *cli.App { 11 | return &cli.App{ 12 | Commands: []*cli.Command{ 13 | command.KafkaCommands, 14 | server.Command, 15 | command.LoadBalancerCommands, 16 | command.TaskRunnerCommands, 17 | }, 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /processor/taskrunner/cli/command/server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "ylem_taskrunner/config" 5 | "ylem_taskrunner/services/server" 6 | 7 | log "github.com/sirupsen/logrus" 8 | "github.com/urfave/cli/v2" 9 | ) 10 | 11 | var serveHandler cli.ActionFunc = func(c *cli.Context) error { 12 | log.Debug("serve command called") 13 | return server.NewServer(config.Cfg().Listen).Run() 14 | } 15 | 16 | var ServeCommand = &cli.Command{ 17 | Name: "serve", 18 | Description: "Start a HTTP(s) server", 19 | Usage: "Start a HTTP(s) server", 20 | Action: serveHandler, 21 | } 22 | 23 | var Command = &cli.Command{ 24 | Name: "server", 25 | Description: "HTTP(s) server commands", 26 | Usage: "HTTP(s) server commands", 27 | Subcommands: []*cli.Command{ 28 | ServeCommand, 29 | }, 30 | } 31 | -------------------------------------------------------------------------------- /processor/taskrunner/config/keys/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/processor/taskrunner/config/keys/.gitkeep -------------------------------------------------------------------------------- /processor/taskrunner/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | ylem_taskrunner: 3 | env_file: 4 | - .env 5 | - ../../.env.common 6 | build: 7 | context: . 8 | dockerfile: Dockerfile 9 | command: /opt/ylem_taskrunner/ylem_taskrunner taskrunner start 10 | container_name: ylem_taskrunner 11 | networks: 12 | - ylem_network 13 | links: 14 | - ylem_session_storage 15 | depends_on: 16 | - ylem_session_storage 17 | ports: 18 | - "7335:7335" 19 | volumes: 20 | - .:/go/src/ylem_taskrunner 21 | working_dir: /go/src/ylem_taskrunner 22 | stdin_open: true 23 | tty: true 24 | 25 | ylem_loadbalancer: 26 | env_file: 27 | - .env 28 | - ../../.env.common 29 | build: 30 | context: . 31 | dockerfile: Dockerfile 32 | command: /opt/ylem_taskrunner/ylem_taskrunner loadbalancer start 33 | container_name: ylem_loadbalancer 34 | networks: 35 | - ylem_network 36 | links: 37 | - ylem_session_storage 38 | depends_on: 39 | - ylem_session_storage 40 | ports: 41 | - "7334:7335" 42 | volumes: 43 | - .:/go/src/ylem_taskrunner 44 | working_dir: /go/src/ylem_taskrunner 45 | stdin_open: true 46 | tty: true 47 | 48 | networks: 49 | default: 50 | name: ylem_network 51 | external: true 52 | -------------------------------------------------------------------------------- /processor/taskrunner/domain/runner/code.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "ylem_taskrunner/services/gopyk" 5 | 6 | messaging "github.com/ylem-co/shared-messaging" 7 | ) 8 | 9 | func CodeTaskRunner(t *messaging.ExecuteCodeTask) *messaging.TaskRunResult { 10 | return runMeasured(func() *messaging.TaskRunResult { 11 | tr := messaging.NewTaskRunResult(t.TaskUuid) 12 | 13 | tr.PipelineUuid = t.PipelineUuid 14 | tr.CreatorUuid = t.CreatorUuid 15 | tr.OrganizationUuid = t.OrganizationUuid 16 | tr.IsSuccessful = true 17 | tr.PipelineRunUuid = t.PipelineRunUuid 18 | tr.TaskRunUuid = t.TaskRunUuid 19 | tr.TaskType = messaging.TaskTypeCode 20 | tr.IsInitialTask = t.IsInitialTask 21 | tr.IsFinalTask = t.IsFinalTask 22 | tr.Meta = t.Meta 23 | 24 | inst := gopyk.Instance() 25 | resp, err := inst.Evaluate(gopyk.Request{ 26 | Code: t.Code, 27 | Type: t.Type, 28 | Input: string(t.Input), 29 | }) 30 | 31 | if err != nil { 32 | tr.IsSuccessful = false 33 | tr.Errors = []messaging.TaskRunError{ 34 | { 35 | Code: messaging.ErrorExecuteCodeFailure, 36 | Severity: messaging.ErrorSeverityError, 37 | Message: err.Error(), 38 | }, 39 | } 40 | 41 | return tr 42 | } 43 | 44 | tr.Output = resp 45 | 46 | return tr 47 | }) 48 | } 49 | -------------------------------------------------------------------------------- /processor/taskrunner/domain/runner/external_trigger.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | messaging "github.com/ylem-co/shared-messaging" 5 | ) 6 | 7 | func ExternalTriggerTaskRunner(t *messaging.ExternalTriggerTask) *messaging.TaskRunResult { 8 | return runMeasured(func() *messaging.TaskRunResult { 9 | tr := messaging.NewTaskRunResult(t.TaskUuid) 10 | 11 | tr.PipelineType = t.PipelineType 12 | tr.PipelineUuid = t.PipelineUuid 13 | tr.CreatorUuid = t.CreatorUuid 14 | tr.OrganizationUuid = t.OrganizationUuid 15 | tr.TaskType = messaging.TaskTypeExternalTrigger 16 | tr.PipelineRunUuid = t.PipelineRunUuid 17 | tr.TaskRunUuid = t.TaskRunUuid 18 | tr.IsInitialTask = t.IsInitialTask 19 | tr.IsFinalTask = t.IsFinalTask 20 | tr.Meta = t.Meta 21 | tr.IsSuccessful = true 22 | 23 | tr.Output = t.Input 24 | 25 | return tr 26 | }) 27 | } 28 | -------------------------------------------------------------------------------- /processor/taskrunner/domain/runner/filter.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "encoding/json" 5 | "ylem_taskrunner/helpers/kafka" 6 | "ylem_taskrunner/services/transformers" 7 | 8 | messaging "github.com/ylem-co/shared-messaging" 9 | ) 10 | 11 | func FilterTaskRunner(t *messaging.FilterTask) *messaging.TaskRunResult { 12 | return runMeasured(func() *messaging.TaskRunResult { 13 | tr := messaging.NewTaskRunResult(t.TaskUuid) 14 | 15 | tr.PipelineType = t.PipelineType 16 | tr.PipelineUuid = t.PipelineUuid 17 | tr.CreatorUuid = t.CreatorUuid 18 | tr.OrganizationUuid = t.OrganizationUuid 19 | tr.IsSuccessful = true 20 | tr.PipelineRunUuid = t.PipelineRunUuid 21 | tr.TaskRunUuid = t.TaskRunUuid 22 | tr.TaskType = messaging.TaskTypeFilter 23 | tr.IsInitialTask = t.IsInitialTask 24 | tr.IsFinalTask = t.IsFinalTask 25 | tr.Meta = t.Meta 26 | 27 | result := transformers.ExtractFromJsonWithJsonQuery(t.Input, t.Expression) 28 | newValue, err := json.Marshal(result.Value()) 29 | 30 | if err != nil { 31 | kafka.HandleBadRequestError(t.TaskUuid, messaging.TaskFilterMessageName, err, tr) 32 | 33 | return tr 34 | } 35 | 36 | tr.Output = newValue 37 | 38 | return tr 39 | }) 40 | } 41 | -------------------------------------------------------------------------------- /processor/taskrunner/domain/runner/run_pipeline.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | messaging "github.com/ylem-co/shared-messaging" 5 | ) 6 | 7 | func RunPipelineTaskRunner(t *messaging.RunPipelineTask) *messaging.TaskRunResult { 8 | return runMeasured(func() *messaging.TaskRunResult { 9 | tr := messaging.NewTaskRunResult(t.TaskUuid) 10 | 11 | tr.PipelineType = t.PipelineType 12 | tr.PipelineUuid = t.PipelineUuid 13 | tr.CreatorUuid = t.CreatorUuid 14 | tr.OrganizationUuid = t.OrganizationUuid 15 | tr.IsSuccessful = true 16 | tr.PipelineRunUuid = t.PipelineRunUuid 17 | tr.TaskRunUuid = t.TaskRunUuid 18 | tr.TaskType = messaging.TaskTypeRunPipeline 19 | tr.IsInitialTask = t.IsInitialTask 20 | tr.IsFinalTask = t.IsFinalTask 21 | tr.Meta = t.Meta 22 | tr.Output = t.Input 23 | 24 | return tr 25 | }) 26 | } 27 | -------------------------------------------------------------------------------- /processor/taskrunner/helpers/aws.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "context" 5 | "ylem_taskrunner/config" 6 | "ylem_taskrunner/services/aws/kms" 7 | ) 8 | 9 | func DecryptData(ctx context.Context, dataKey []byte, encryptedData []byte) (string, error) { 10 | decryptedDataKey, err := kms.DecryptDataKey( 11 | ctx, 12 | config.Cfg().Aws.KmsKeyId, 13 | dataKey, 14 | ) 15 | 16 | if err != nil { 17 | return "", err 18 | } 19 | 20 | decryptedData, err := kms.Decrypt(encryptedData, decryptedDataKey) 21 | 22 | if err != nil { 23 | return "", err 24 | } 25 | 26 | return string(decryptedData), err 27 | } 28 | -------------------------------------------------------------------------------- /processor/taskrunner/helpers/evaluate.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "context" 5 | "regexp" 6 | "ylem_taskrunner/helpers/evaluate" 7 | 8 | "github.com/PaesslerAG/gval" 9 | ) 10 | 11 | func EvaluateGValExpressionWithContext(ctx context.Context, expression string, data interface{}) (interface{}, error) { 12 | return evaluateWithContext(ctx, expression, data) 13 | } 14 | 15 | func evaluateWithContext(ctx context.Context, expression string, data interface{}) (interface{}, error) { 16 | rx, err := regexp.Compile("COUNT *\\( *\\* *\\)") //nolint:all 17 | if err != nil { 18 | return nil, err 19 | } 20 | replacedExpression := rx.ReplaceAllString(expression, "COUNT()") 21 | 22 | return gval.EvaluateWithContext( 23 | ctx, 24 | replacedExpression, 25 | data, 26 | evaluate.Language(), 27 | ) 28 | } 29 | -------------------------------------------------------------------------------- /processor/taskrunner/helpers/evaluate/language.go: -------------------------------------------------------------------------------- 1 | package evaluate 2 | 3 | import ( 4 | "github.com/PaesslerAG/gval" 5 | "github.com/google/uuid" 6 | log "github.com/sirupsen/logrus" 7 | ) 8 | 9 | type Context struct { 10 | TaskInput interface{} 11 | EnvVars map[string]interface{} 12 | PipelineUuid uuid.UUID 13 | } 14 | 15 | type noIdentifierPresented struct{} 16 | 17 | var lng = gval.NewLanguage( 18 | gval.Bitmask(), 19 | gval.Text(), 20 | gval.PropositionalLogic(), 21 | gval.JSON(), 22 | arithmetic, 23 | text, 24 | funcs, 25 | dates, 26 | varselector, 27 | ) 28 | 29 | func recoverGvalFunc(fName string) { 30 | if r := recover(); r != nil { 31 | log.Errorf("Panic while executing a gval function\" %s\", recovered and skipped\n", fName) 32 | log.Error(r) 33 | } 34 | } 35 | 36 | func Language() gval.Language { 37 | return lng 38 | } 39 | -------------------------------------------------------------------------------- /processor/taskrunner/helpers/evaluate/text.go: -------------------------------------------------------------------------------- 1 | package evaluate 2 | 3 | import ( 4 | "github.com/PaesslerAG/gval" 5 | ) 6 | 7 | var text = gval.NewLanguage( 8 | gval.InfixTextOperator("==", func(a, b string) (interface{}, error) { 9 | defer recoverGvalFunc("==") 10 | 11 | return a == b, nil 12 | }), 13 | gval.InfixTextOperator("===", func(a, b string) (interface{}, error) { 14 | defer recoverGvalFunc("===") 15 | 16 | return a == b, nil 17 | }), 18 | gval.InfixTextOperator("!=", func(a, b string) (interface{}, error) { 19 | defer recoverGvalFunc("!=") 20 | 21 | return a != b, nil 22 | }), 23 | gval.InfixTextOperator("!==", func(a, b string) (interface{}, error) { 24 | defer recoverGvalFunc("!==") 25 | 26 | return a != b, nil 27 | }), 28 | ) 29 | -------------------------------------------------------------------------------- /processor/taskrunner/helpers/kafka/decode_input.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/google/uuid" 7 | log "github.com/sirupsen/logrus" 8 | messaging "github.com/ylem-co/shared-messaging" 9 | ) 10 | 11 | func DecodeKafkaTaskValue(t messaging.Task, messageName string, tr *messaging.TaskRunResult) (interface{}, error) { 12 | var ( 13 | err error 14 | taskValue interface{} 15 | ) 16 | 17 | if len(t.Input) == 0 { 18 | return make(map[string]interface{}), nil 19 | } 20 | 21 | err = json.Unmarshal(t.Input, &taskValue) 22 | 23 | if err != nil { 24 | HandleBadRequestError(t.TaskUuid, messageName, err, tr) 25 | 26 | return taskValue, err 27 | } 28 | 29 | return taskValue, nil 30 | } 31 | 32 | func HandleBadRequestError(taskUuid uuid.UUID, messageName string, err error, tr *messaging.TaskRunResult) { 33 | log.Errorf( 34 | `could not execute task "%s"" with uuid "%s": %v`, 35 | messageName, 36 | taskUuid, 37 | err, 38 | ) 39 | 40 | tr.IsSuccessful = false 41 | tr.Errors = []messaging.TaskRunError{ 42 | { 43 | Code: messaging.ErrorBadRequest, 44 | Severity: messaging.ErrorSeverityError, 45 | Message: err.Error(), 46 | }, 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /processor/taskrunner/helpers/time.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | const DateTimeFormat = "2006-01-02 15:04:05" 4 | -------------------------------------------------------------------------------- /processor/taskrunner/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "time" 7 | "ylem_taskrunner/cli" 8 | 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | func main() { 13 | loc, _ := time.LoadLocation("UTC") 14 | time.Local = loc 15 | 16 | ctx, cancel := context.WithCancel(context.Background()) 17 | defer cancel() 18 | err := cli.NewApplication().RunContext(ctx, os.Args) 19 | 20 | if err != nil { 21 | log.Fatal(err) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /processor/taskrunner/services/aws/kms/source.go: -------------------------------------------------------------------------------- 1 | package kms 2 | 3 | import ( 4 | "context" 5 | "ylem_taskrunner/config" 6 | 7 | messaging "github.com/ylem-co/shared-messaging" 8 | ) 9 | 10 | func DecryptSource(s *messaging.SQLIntegration, ctx context.Context) error { 11 | var ( 12 | value []byte 13 | err error 14 | ) 15 | 16 | keyId := config.Cfg().Aws.KmsKeyId 17 | decryptedDataKey, err := DecryptDataKey(ctx, keyId, s.DataKey) 18 | if err != nil { 19 | return err 20 | } 21 | 22 | if len(s.Host) > 0 { 23 | value, err = Decrypt(s.Host, decryptedDataKey) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | s.Host = value 29 | } 30 | 31 | // Password 32 | if len(s.Password) > 0 { 33 | value, err = Decrypt(s.Password, decryptedDataKey) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | s.Password = value 39 | } 40 | 41 | // SSH Host 42 | if len(s.SshHost) > 0 { 43 | value, err = Decrypt(s.SshHost, decryptedDataKey) 44 | if err != nil { 45 | return err 46 | } 47 | 48 | s.SshHost = value 49 | } 50 | 51 | // Credentials 52 | if len(s.Credentials) > 0 { 53 | value, err = Decrypt(s.Credentials, decryptedDataKey) 54 | if err != nil { 55 | return err 56 | } 57 | 58 | s.Credentials = value 59 | } 60 | 61 | return nil 62 | } 63 | -------------------------------------------------------------------------------- /processor/taskrunner/services/gemini/gemini.go: -------------------------------------------------------------------------------- 1 | package gemini 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | "encoding/json" 8 | "ylem_taskrunner/config" 9 | 10 | "google.golang.org/genai" 11 | ) 12 | 13 | func Process(JSON string, UserPrompt string) (string, error) { 14 | ctx := context.Background() 15 | client, err := genai.NewClient(ctx, &genai.ClientConfig{ 16 | APIKey: config.Cfg().Gemini.Key, 17 | Backend: genai.BackendGeminiAPI, 18 | }) 19 | if err != nil { 20 | return "", err 21 | } 22 | 23 | text, err := BuildPrompt(JSON, UserPrompt) 24 | if err != nil { 25 | return "", err 26 | } 27 | 28 | result, err := client.Models.GenerateContent( 29 | ctx, 30 | config.Cfg().Gemini.Model, 31 | genai.Text(text), 32 | nil, 33 | ) 34 | if err != nil { 35 | return "", err 36 | } 37 | 38 | return result.Text(), nil 39 | } 40 | 41 | func BuildPrompt(JSON string, UserPrompt string) (string, error) { 42 | j, err := json.Marshal(JSON) 43 | if err != nil { 44 | return "", err 45 | } 46 | 47 | tJ := strings.Trim(string(j), "\"") 48 | 49 | message := fmt.Sprintf("In that JSON %s %s", tJ, UserPrompt) 50 | 51 | return message, nil 52 | } 53 | -------------------------------------------------------------------------------- /processor/taskrunner/services/gopyk/client.go: -------------------------------------------------------------------------------- 1 | package gopyk 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "ylem_taskrunner/config" 7 | 8 | "github.com/go-resty/resty/v2" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | var srv *Gopyk 13 | 14 | type Gopyk struct { 15 | client *resty.Client 16 | } 17 | 18 | type Request struct { 19 | Code string `json:"code"` 20 | Type string `json:"type"` 21 | Input string `json:"input"` 22 | } 23 | 24 | func (g *Gopyk) Evaluate(r Request) ([]byte, error) { 25 | log.Tracef("gopyk: evaluation") 26 | response, err := g.client. 27 | R(). 28 | SetBody(r). 29 | Post("") 30 | 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | if response.StatusCode() != http.StatusOK { 36 | log.Debug(string(response.Body())) 37 | 38 | return nil, fmt.Errorf("python execution failed: %s", string(response.Body())) 39 | } 40 | 41 | return response.Body(), nil 42 | } 43 | 44 | func Instance() *Gopyk { 45 | if srv == nil { 46 | srv = &Gopyk{ 47 | client: resty.New().SetBaseURL(config.Cfg().Gopyk.BaseUrl), 48 | } 49 | } 50 | 51 | return srv 52 | } 53 | -------------------------------------------------------------------------------- /processor/taskrunner/services/hubspot/hubspot.go: -------------------------------------------------------------------------------- 1 | package hubspot 2 | 3 | import ( 4 | "context" 5 | "time" 6 | "ylem_taskrunner/helpers" 7 | 8 | hubspotclient "github.com/ylem-co/hubspot-client" 9 | "golang.org/x/oauth2" 10 | ) 11 | 12 | func init() { 13 | // we don't really need any of that, yet we need the config to be initiated 14 | hubspotclient.Initiate(hubspotclient.Config{ 15 | ClientID: "", 16 | ClientSecret: "", 17 | RedirectUrl: "", 18 | Scopes: []string{}, 19 | }) 20 | } 21 | 22 | type Authentication struct { 23 | EncryptedDataKey []byte 24 | EncryptedAccessToken []byte 25 | } 26 | 27 | func CreateTicket(ctx context.Context, request hubspotclient.CreateTicketRequest, auth Authentication) error { 28 | accessToken, err := helpers.DecryptData(ctx, auth.EncryptedDataKey, auth.EncryptedAccessToken) 29 | if err != nil { 30 | return err 31 | } 32 | 33 | token := oauth2.Token{ 34 | AccessToken: accessToken, 35 | Expiry: time.Now().Add(1 * time.Hour), // it's always fresh here 36 | } 37 | 38 | client, err := hubspotclient.CreateInstance(ctx, &token) 39 | if err != nil { 40 | return err 41 | } 42 | 43 | return client.CreateTicket(request) 44 | } 45 | -------------------------------------------------------------------------------- /processor/taskrunner/services/jenkins/client.go: -------------------------------------------------------------------------------- 1 | package jenkins 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | 8 | "github.com/go-resty/resty/v2" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | var instance *Jenkins 13 | 14 | type Jenkins struct { 15 | client *resty.Client 16 | ctx context.Context 17 | } 18 | 19 | func (j Jenkins) RunBuild(baseUrl string, project string, token string) error { 20 | log.Tracef("jenkins: run build") 21 | url := fmt.Sprintf("%s/job/%s/build?token=%s", baseUrl, project, token) 22 | 23 | response, err := j.client. 24 | R(). 25 | Post(url) 26 | 27 | if err != nil { 28 | return err 29 | } 30 | 31 | if response.StatusCode() >= http.StatusBadRequest { 32 | log.Debug(string(response.Body())) 33 | 34 | return fmt.Errorf("jenkins: run build: expected http 2xx, got %s", response.Status()) 35 | } 36 | 37 | return nil 38 | } 39 | 40 | func Instance(ctx context.Context) *Jenkins { 41 | if instance == nil { 42 | instance = &Jenkins{ 43 | client: resty.New(), 44 | ctx: ctx, 45 | } 46 | 47 | } 48 | 49 | return instance 50 | } 51 | -------------------------------------------------------------------------------- /processor/taskrunner/services/openai/openai.go: -------------------------------------------------------------------------------- 1 | package openai 2 | 3 | import ( 4 | "ylem_taskrunner/config" 5 | 6 | "github.com/go-resty/resty/v2" 7 | ) 8 | 9 | var srv *OpenAi 10 | 11 | type OpenAi struct { 12 | client *resty.Client 13 | } 14 | 15 | func Instance() *OpenAi { 16 | if srv == nil { 17 | srv = &OpenAi{ 18 | client: resty.New(). 19 | SetBaseURL("https://api.openai.com/v1"). 20 | SetAuthToken(config.Cfg().Openai.GptKey), 21 | } 22 | } 23 | 24 | return srv 25 | } 26 | -------------------------------------------------------------------------------- /processor/taskrunner/services/opsgenie/client.go: -------------------------------------------------------------------------------- 1 | package opsgenie 2 | 3 | import ( 4 | "context" 5 | "ylem_taskrunner/config" 6 | "ylem_taskrunner/services/aws/kms" 7 | 8 | "github.com/ylem-co/opsgenie-client" 9 | ) 10 | 11 | type Alert struct { 12 | Message string 13 | Description string 14 | Priority string 15 | } 16 | 17 | func DecryptKeyAndCreateAlert(ctx context.Context, dataKey []byte, apiKey []byte, alert Alert) error { 18 | decryptedDataKey, err := kms.DecryptDataKey( 19 | ctx, 20 | config.Cfg().Aws.KmsKeyId, 21 | dataKey, 22 | ) 23 | 24 | if err != nil { 25 | return err 26 | } 27 | 28 | decryptedApiKey, err := kms.Decrypt(apiKey, decryptedDataKey) 29 | 30 | if err != nil { 31 | return err 32 | } 33 | 34 | opsgenie, _ := opsgenieclient.CreateInstance(ctx, string(decryptedApiKey)) 35 | 36 | err = opsgenie.CreateAlert(opsgenieclient.CreateAlertRequest{ 37 | Message: alert.Message, 38 | Description: alert.Description, 39 | Priority: alert.Priority, 40 | }) 41 | 42 | if err != nil { 43 | return err 44 | } 45 | 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /processor/taskrunner/services/redis/client.go: -------------------------------------------------------------------------------- 1 | package redis 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "ylem_taskrunner/config" 7 | 8 | "github.com/go-redis/redis/v8" 9 | ) 10 | 11 | var instance *redis.Client 12 | 13 | func Init(ctx context.Context) { 14 | cfg := config.Cfg().Redis 15 | 16 | instance = redis.NewClient(&redis.Options{ 17 | Addr: fmt.Sprintf("%s:%s", cfg.Host, cfg.Port), 18 | Password: cfg.Password, 19 | DB: 0, 20 | }).WithContext(ctx) 21 | 22 | _, err := instance.Ping(ctx).Result() 23 | if err != nil { 24 | panic(err) 25 | } 26 | } 27 | 28 | func Instance() *redis.Client { 29 | if instance == nil { 30 | panic("Redis client not initialized") 31 | } 32 | return instance 33 | } 34 | -------------------------------------------------------------------------------- /processor/taskrunner/services/salesforce/salesforce.go: -------------------------------------------------------------------------------- 1 | package salesforce 2 | 3 | import ( 4 | "context" 5 | "time" 6 | "ylem_taskrunner/helpers" 7 | 8 | "github.com/ylem-co/salesforce-client" 9 | "golang.org/x/oauth2" 10 | ) 11 | 12 | func init() { 13 | // we don't really need any of that, yet we need the config to be initiated 14 | salesforceclient.Initiate(salesforceclient.Config{ 15 | ClientID: "", 16 | ClientSecret: "", 17 | RedirectUrl: "", 18 | Scopes: []string{}, 19 | }) 20 | } 21 | 22 | type Authentication struct { 23 | EncryptedDataKey []byte 24 | EncryptedAccessToken []byte 25 | } 26 | 27 | func CreateCase(ctx context.Context, request salesforceclient.CreateCaseRequest, auth Authentication, domain string) error { 28 | accessToken, err := helpers.DecryptData(ctx, auth.EncryptedDataKey, auth.EncryptedAccessToken) 29 | if err != nil { 30 | return err 31 | } 32 | 33 | token := oauth2.Token{ 34 | AccessToken: accessToken, 35 | Expiry: time.Now().Add(1 * time.Hour), // it's always fresh here 36 | } 37 | 38 | client, err := salesforceclient.CreateInstance(ctx, domain, &token) 39 | if err != nil { 40 | return err 41 | } 42 | 43 | return client.CreateCase(request) 44 | } 45 | -------------------------------------------------------------------------------- /processor/taskrunner/services/server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "net/http" 5 | "ylem_taskrunner/api" 6 | 7 | "github.com/gorilla/mux" 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | type Server struct { 12 | Listen string 13 | } 14 | 15 | func (s *Server) Run() error { 16 | log.Info("Starting server listening on " + s.Listen) 17 | 18 | rtr := mux.NewRouter() 19 | 20 | rtr.HandleFunc("/hello/{name}/", api.ExampleHandler).Methods(http.MethodGet) 21 | http.Handle("/", rtr) 22 | 23 | return http.ListenAndServe(s.Listen, rtr) 24 | } 25 | 26 | func NewServer(listen string) *Server { 27 | s := &Server{ 28 | Listen: listen, 29 | } 30 | 31 | return s 32 | } 33 | -------------------------------------------------------------------------------- /processor/taskrunner/services/slack/slack.go: -------------------------------------------------------------------------------- 1 | package slack 2 | 3 | import ( 4 | "fmt" 5 | 6 | messaging "github.com/ylem-co/shared-messaging" 7 | "github.com/slack-go/slack" 8 | ) 9 | 10 | func SendSlackMessage(ChannelId string, Title string, Text string, Severity string, AccessToken string) error { 11 | api := slack.New(AccessToken) 12 | 13 | severityToColor := map[string]string { 14 | messaging.TaskSeverityCritical: "#8b0000", 15 | messaging.TaskSeverityHigh: "#8b0000", 16 | messaging.TaskSeverityMedium: "#DEC20B", 17 | messaging.TaskSeverityLowest: "#006400", 18 | messaging.TaskSeverityLow: "#006400", 19 | } 20 | 21 | color, ok := severityToColor[Severity] 22 | if !ok { 23 | return fmt.Errorf(`unknown task severity "%s"`, Severity) 24 | } 25 | 26 | headerText := slack.NewTextBlockObject("mrkdwn", fmt.Sprintf("*%s*", Title), false, false) 27 | headerSection := slack.NewSectionBlock(headerText, nil, nil) 28 | 29 | attachment := slack.Attachment{ 30 | Text: Text, 31 | Color: color, 32 | } 33 | 34 | _, _, err := api.PostMessage( 35 | ChannelId, 36 | slack.MsgOptionBlocks(headerSection), 37 | slack.MsgOptionAttachments(attachment), 38 | slack.MsgOptionAsUser(true), 39 | ) 40 | 41 | return err 42 | } 43 | -------------------------------------------------------------------------------- /processor/taskrunner/services/sqlIntegrations/SnowflakeIntegrationConnection.go: -------------------------------------------------------------------------------- 1 | package sqlIntegrations 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "database/sql" 7 | ) 8 | 9 | type SnowflakeSQLIntegrationConnection struct { 10 | AccountId string 11 | User string 12 | Password string 13 | Database string 14 | DB *sql.DB 15 | } 16 | 17 | func (s *SnowflakeSQLIntegrationConnection) Open() error { 18 | var err error 19 | s.DB, err = sql.Open("snowflake", fmt.Sprintf("%s:%s@%s/%s", s.User, s.Password, s.AccountId, s.Database)) 20 | if err != nil { 21 | return err 22 | } 23 | 24 | return nil 25 | } 26 | 27 | func (s *SnowflakeSQLIntegrationConnection) Close() error { 28 | return s.DB.Close() 29 | } 30 | 31 | func (s *SnowflakeSQLIntegrationConnection) Test() error { 32 | return s.DB.PingContext(context.Background()) 33 | } 34 | 35 | func (s *SnowflakeSQLIntegrationConnection) Prepare(query string) (*sql.Stmt, error) { 36 | return s.DB.PrepareContext(context.Background(), query) 37 | } 38 | 39 | func (s *SnowflakeSQLIntegrationConnection) Exec(query string, args ...interface{}) (sql.Result, error) { 40 | return s.DB.ExecContext(context.Background(), query, args...) 41 | } 42 | 43 | func (s *SnowflakeSQLIntegrationConnection) Query(query string, args ...interface{}) (*sql.Rows, error) { 44 | return s.DB.QueryContext(context.Background(), query, args...) 45 | } 46 | -------------------------------------------------------------------------------- /processor/taskrunner/tests/services/evaluate/.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/processor/taskrunner/tests/services/evaluate/.env -------------------------------------------------------------------------------- /processor/taskrunner/tests/services/evaluate/config/keys/id_rsa: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/processor/taskrunner/tests/services/evaluate/config/keys/id_rsa -------------------------------------------------------------------------------- /processor/taskrunner/tests/services/transformers/.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/processor/taskrunner/tests/services/transformers/.env -------------------------------------------------------------------------------- /processor/taskrunner/tests/services/transformers/config/keys/id_rsa: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/processor/taskrunner/tests/services/transformers/config/keys/id_rsa -------------------------------------------------------------------------------- /server/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea/ 3 | -------------------------------------------------------------------------------- /server/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | ylem_server: 3 | build: 4 | context: ./nginx 5 | container_name: ylem_server 6 | networks: 7 | - ylem_network 8 | depends_on: 9 | - ylem_api 10 | - ylem_users 11 | - ylem_integrations 12 | - ylem_pipelines 13 | - ylem_statistics 14 | - ylem_taskrunner 15 | volumes: 16 | - ./nginx/nginx.conf:/etc/nginx/nginx.conf 17 | - ./nginx/sites/:/etc/nginx/sites-available 18 | - ./nginx/conf.d/:/etc/nginx/conf.d 19 | - ./logs:/var/log 20 | ports: 21 | - "7331:7331" 22 | - "443:443" 23 | 24 | networks: 25 | default: 26 | name: ylem_network 27 | external: true 28 | -------------------------------------------------------------------------------- /server/logs/nginx/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /server/nginx/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nginx:alpine 2 | 3 | WORKDIR /var/www 4 | 5 | CMD ["nginx"] 6 | 7 | EXPOSE 80 443 8 | -------------------------------------------------------------------------------- /server/nginx/conf.d/default.conf: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /server/nginx/nginx.conf: -------------------------------------------------------------------------------- 1 | user nginx; 2 | worker_processes 4; 3 | daemon off; 4 | 5 | error_log /var/log/nginx/error.log warn; 6 | pid /var/run/nginx.pid; 7 | 8 | events { 9 | worker_connections 1024; 10 | } 11 | 12 | http { 13 | include /etc/nginx/mime.types; 14 | default_type application/octet-stream; 15 | 16 | access_log /var/log/nginx/access.log; 17 | 18 | sendfile on; 19 | 20 | keepalive_timeout 65; 21 | 22 | include /etc/nginx/conf.d/*.conf; 23 | include /etc/nginx/sites-available/*.conf; 24 | } 25 | -------------------------------------------------------------------------------- /ui/.env: -------------------------------------------------------------------------------- 1 | SKIP_PREFLIGHT_CHECK=true 2 | GENERATE_SOURCEMAP=false 3 | PORT=7330 4 | -------------------------------------------------------------------------------- /ui/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | *_old -------------------------------------------------------------------------------- /ui/Dockerfile: -------------------------------------------------------------------------------- 1 | # get the base node image 2 | FROM node:alpine as builder 3 | 4 | # set the working dir for container 5 | WORKDIR /frontend 6 | 7 | # copy the json file first 8 | COPY ./package.json /frontend 9 | COPY ./package-lock.json /frontend 10 | 11 | # extend npm file sizes 12 | RUN export NODE_OPTIONS=--max_old_space_size=4096 13 | 14 | # install npm dependencies 15 | RUN npm cache clean --force 16 | RUN npm ci --legacy-peer-deps 17 | 18 | # copy other project files 19 | COPY . . 20 | 21 | # build the folder 22 | #ARG REACT_APP_ENVIRONMENT 23 | #ARG REACT_APP_BACKEND_URL 24 | #RUN REACT_APP_ENVIRONMENT="$REACT_APP_ENVIRONMENT" REACT_APP_BACKEND_URL="$REACT_APP_BACKEND_URL" npm run build 25 | RUN npm run build 26 | 27 | # Handle Nginx 28 | FROM nginx 29 | COPY --from=builder /frontend/build /usr/share/nginx/html 30 | COPY ./docker/nginx/default.conf /etc/nginx/conf.d/default.conf 31 | -------------------------------------------------------------------------------- /ui/Dockerfile.dev: -------------------------------------------------------------------------------- 1 | # get the base node image 2 | FROM node:alpine as builder 3 | 4 | # set the working dir for container 5 | WORKDIR /frontend 6 | 7 | # copy the json file first 8 | COPY ./package.json /frontend 9 | COPY ./package-lock.json /frontend 10 | 11 | # extend npm file sizes 12 | RUN export NODE_OPTIONS=--max_old_space_size=4096 13 | 14 | # install npm dependencies 15 | RUN npm cache clean --force 16 | RUN npm ci --legacy-peer-deps 17 | 18 | # copy other project files 19 | COPY . . 20 | 21 | # build the folder 22 | # RUN npm run start -- --no-inline --no-hot 23 | -------------------------------------------------------------------------------- /ui/README.md: -------------------------------------------------------------------------------- 1 | # YLEM FRONTEND UI 2 | 3 | ![Static Badge](https://img.shields.io/badge/React-18.3.1-black) 4 | ![Static Badge](https://img.shields.io/badge/license-Apache%202.0-black) 5 | ![Static Badge](https://img.shields.io/badge/website-ylem.co-black) 6 | ![Static Badge](https://img.shields.io/badge/documentation-docs.ylem.co-black) 7 | ![Static Badge](https://img.shields.io/badge/community-join%20Slack-black) 8 | 9 | Ylem UI platform, connected to APIs provided by other microservices. 10 | 11 | # Usage 12 | 13 | ## Dev environment 14 | 15 | ``` bash 16 | $ docker compose -f docker-compose-dev.yml up 17 | ``` 18 | 19 | Ylem UI is available on http://127.0.0.1:7330/ 20 | 21 | ## Production environment 22 | 23 | ``` bash 24 | $ docker compose -f docker-compose.yml up 25 | ``` 26 | 27 | Ylem UI is available on http://127.0.0.1:7440/ 28 | 29 | # ESLint 30 | 31 | ``` bash 32 | $ eslint '**/*.js' 33 | ``` 34 | -------------------------------------------------------------------------------- /ui/docker-compose-dev.yml: -------------------------------------------------------------------------------- 1 | services: 2 | ylem_ui: 3 | build: 4 | context: . 5 | args: 6 | REACT_APP_ENVIRONMENT: dev 7 | dockerfile: Dockerfile.dev 8 | command: npm run start -- --no-inline --no-hot 9 | container_name: ylem_ui 10 | ports: 11 | - "7330:7330" 12 | networks: 13 | - ylem_network 14 | volumes: 15 | - ./:/frontend 16 | - /frontend/node_modules 17 | stdin_open: true 18 | 19 | networks: 20 | default: 21 | name: ylem_network 22 | external: true 23 | -------------------------------------------------------------------------------- /ui/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | ylem_ui_production: 3 | build: 4 | context: . 5 | args: 6 | REACT_APP_ENVIRONMENT: prod 7 | #REACT_APP_BACKEND_URL: //api.ylem.co 8 | dockerfile: Dockerfile 9 | container_name: ylem_ui_production 10 | networks: 11 | - ylem_network 12 | ports: 13 | - "7440:7440" 14 | volumes: 15 | - ./:/frontend 16 | - /frontend/node_modules 17 | stdin_open: true 18 | 19 | networks: 20 | default: 21 | name: ylem_network 22 | external: true 23 | -------------------------------------------------------------------------------- /ui/docker/nginx/default.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 7440; 3 | server_name _; 4 | 5 | index index.html; 6 | root /usr/share/nginx/html; 7 | 8 | error_log /var/log/nginx/error.log; 9 | access_log /var/log/nginx/access.log; 10 | 11 | gzip on; 12 | gzip_disable "msie6"; 13 | 14 | gzip_comp_level 6; 15 | gzip_min_length 1100; 16 | gzip_buffers 16 8k; 17 | gzip_proxied any; 18 | gzip_types 19 | text/plain 20 | text/css 21 | text/js 22 | text/xml 23 | text/javascript 24 | application/javascript 25 | application/json 26 | application/xml 27 | application/rss+xml 28 | image/svg+xml; 29 | 30 | location / { 31 | try_files $uri /index.html =404; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /ui/eslint.config.js: -------------------------------------------------------------------------------- 1 | const reactRecommended = require('eslint-plugin-react/configs/recommended'); 2 | const reactHooks = require('eslint-plugin-react-hooks'); 3 | const globals = require('globals'); 4 | 5 | const GLOBALS_BROWSER_FIX = Object.assign({}, globals.browser, { 6 | AudioWorkletGlobalScope: globals.browser['AudioWorkletGlobalScope '] 7 | }); 8 | 9 | delete GLOBALS_BROWSER_FIX['AudioWorkletGlobalScope ']; 10 | 11 | module.exports = [ 12 | { 13 | files: ['**/*.{js,mjs,cjs,jsx,mjsx,ts,tsx,mtsx}'], 14 | ...reactRecommended, 15 | }, 16 | { 17 | files: ['**/*.{js,mjs,cjs,jsx,mjsx,ts,tsx,mtsx}'], 18 | plugins: { 19 | reactHooks, 20 | }, 21 | languageOptions: { 22 | parserOptions: { 23 | ecmaFeatures: { 24 | jsx: true, 25 | }, 26 | }, 27 | globals: { 28 | ...globals.serviceworker, 29 | ...GLOBALS_BROWSER_FIX, 30 | }, 31 | }, 32 | rules: { 33 | "react/prop-types": "off", 34 | "react/react-in-jsx-scope": "off", 35 | "react/jsx-uses-react": "off", 36 | "reactHooks/rules-of-hooks": "error", 37 | "reactHooks/exhaustive-deps": "warn" 38 | } 39 | }, 40 | ]; 41 | -------------------------------------------------------------------------------- /ui/jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "baseUrl": "." 4 | }, 5 | "include": ["src"] 6 | } 7 | -------------------------------------------------------------------------------- /ui/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/favicon.ico -------------------------------------------------------------------------------- /ui/public/images/info.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/info.png -------------------------------------------------------------------------------- /ui/public/images/logo-s-dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/logo-s-dark.png -------------------------------------------------------------------------------- /ui/public/images/logo-s.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/logo-s.png -------------------------------------------------------------------------------- /ui/public/images/logo2-dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/logo2-dark.png -------------------------------------------------------------------------------- /ui/public/images/logo2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/logo2.png -------------------------------------------------------------------------------- /ui/public/images/release-notes/r17.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/release-notes/r17.png -------------------------------------------------------------------------------- /ui/public/images/template-previews/0.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/template-previews/0.jpeg -------------------------------------------------------------------------------- /ui/public/images/template-previews/1.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/template-previews/1.jpeg -------------------------------------------------------------------------------- /ui/public/images/template-previews/2.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/template-previews/2.jpeg -------------------------------------------------------------------------------- /ui/public/images/template-previews/3.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/template-previews/3.jpeg -------------------------------------------------------------------------------- /ui/public/images/template-previews/4.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/template-previews/4.jpeg -------------------------------------------------------------------------------- /ui/public/images/template-previews/5.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/template-previews/5.jpeg -------------------------------------------------------------------------------- /ui/public/images/template-previews/6.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/template-previews/6.jpeg -------------------------------------------------------------------------------- /ui/public/images/template-previews/7.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/template-previews/7.jpeg -------------------------------------------------------------------------------- /ui/public/images/template-previews/8.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/template-previews/8.jpeg -------------------------------------------------------------------------------- /ui/public/images/template-previews/9.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/template-previews/9.jpeg -------------------------------------------------------------------------------- /ui/public/images/tour/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/tour/dashboard.png -------------------------------------------------------------------------------- /ui/public/images/tour/env-variables.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/tour/env-variables.png -------------------------------------------------------------------------------- /ui/public/images/tour/metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/tour/metrics.png -------------------------------------------------------------------------------- /ui/public/images/tour/oauth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/tour/oauth.png -------------------------------------------------------------------------------- /ui/public/images/tour/pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/tour/pipeline.png -------------------------------------------------------------------------------- /ui/public/images/tour/profiling.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/tour/profiling.png -------------------------------------------------------------------------------- /ui/public/images/tour/welcome.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/images/tour/welcome.png -------------------------------------------------------------------------------- /ui/public/logo192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/public/logo192.png -------------------------------------------------------------------------------- /ui/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "Ylem", 3 | "name": "Ylem", 4 | "icons": [ 5 | { 6 | "src": "favicon.png", 7 | "sizes": "100x100", 8 | "type": "image/png" 9 | } 10 | ], 11 | "start_url": ".", 12 | "display": "standalone", 13 | "theme_color": "#000000", 14 | "background_color": "#ffffff" 15 | } -------------------------------------------------------------------------------- /ui/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /ui/src/App.test.js: -------------------------------------------------------------------------------- 1 | import { render, screen } from '@testing-library/react'; 2 | import App from './App'; 3 | 4 | test('renders learn react link', () => { 5 | render(); 6 | const linkElement = screen.getByText(/learn react/i); 7 | expect(linkElement).toBeInTheDocument(); 8 | }); 9 | -------------------------------------------------------------------------------- /ui/src/actions/OAuthClients.js: -------------------------------------------------------------------------------- 1 | import { 2 | OAUTH_CLIENT_CREATE_FAIL, 3 | OAUTH_CLIENT_CREATE_SUCCESS, 4 | SET_MESSAGE, 5 | } from "./types"; 6 | 7 | import { 8 | prepareErrorMessage, 9 | } from "../actions/errors"; 10 | 11 | import OAuthService from "../services/oauth.service"; 12 | 13 | export const addOAuthClient = (name) => (dispatch) => { 14 | return OAuthService.createClient(name).then( 15 | (data) => { 16 | dispatch({ 17 | type: OAUTH_CLIENT_CREATE_SUCCESS, 18 | }); 19 | 20 | dispatch({ 21 | type: SET_MESSAGE, 22 | payload: { client: data }, 23 | }); 24 | 25 | return Promise.resolve(); 26 | }, 27 | (error) => { 28 | let message = prepareErrorMessage(error); 29 | 30 | dispatch({ 31 | type: OAUTH_CLIENT_CREATE_FAIL, 32 | }); 33 | 34 | dispatch({ 35 | type: SET_MESSAGE, 36 | payload: message, 37 | }); 38 | 39 | return Promise.reject(); 40 | } 41 | ); 42 | }; 43 | -------------------------------------------------------------------------------- /ui/src/actions/message.js: -------------------------------------------------------------------------------- 1 | import { SET_MESSAGE, CLEAR_MESSAGE } from "./types"; 2 | 3 | export const setMessage = (message) => ({ 4 | type: SET_MESSAGE, 5 | payload: message, 6 | }); 7 | 8 | export const clearMessage = () => ({ 9 | type: CLEAR_MESSAGE, 10 | }); 11 | -------------------------------------------------------------------------------- /ui/src/actions/pipeline.js: -------------------------------------------------------------------------------- 1 | export const PERMISSION_LOGGED_IN = 'PERMISSION_LOGGED_IN'; 2 | export const PERMISSION_LOGGED_OUT = 'PERMISSION_LOGGED_OUT'; 3 | 4 | export const validatePermissions = (isLoggedIn, user, requiredPermission) => { 5 | if ( 6 | requiredPermission === PERMISSION_LOGGED_IN 7 | && !isLoggedIn 8 | ) { 9 | return false; 10 | } else if ( 11 | requiredPermission === PERMISSION_LOGGED_OUT 12 | && isLoggedIn 13 | ) { 14 | return false; 15 | } 16 | 17 | return true; 18 | }; 19 | -------------------------------------------------------------------------------- /ui/src/actions/releaseNotes.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | export const releaseNotes = [ 4 | { 5 | //selector: '.tour-step-profiling', 6 | content: () => ( 7 |
8 |

Pipelines instead of pipelines!

9 |
10 | Evolution is a continuous process. If we look at the functionality of our pipelines when Ylem just started and compare it with the status quo the difference is like the distance from Earth to the Moon. 11 |

12 | Does not it just represent simple business processes anymore, but gives you a Swiss Army knife for data streaming, transformation, enriching, ingestion, cleaning, orchestration, and so on and so on. 13 |

14 | Therefore, we took one more step further and decided to rename pipelines to pipelines, which will allow us to introduce even more exciting features shortly. 15 |

16 |
17 |
18 | Pipelines instead of pipelines 19 |
20 |
21 | ), 22 | //position: [50, 30], 23 | }, 24 | ] 25 | -------------------------------------------------------------------------------- /ui/src/actions/roles.js: -------------------------------------------------------------------------------- 1 | export const ROLE_ORGANIZATION_ADMIN = 'ROLE_ORGANIZATION_ADMIN'; 2 | export const ROLE_TEAM_MEMBER = 'ROLE_TEAM_MEMBER'; 3 | export const ROLE_ALLOWED_TO_SWITCH = 'ROLE_ALLOWED_TO_SWITCH'; 4 | 5 | export const USER_FRIENDLY_ROLES = [ 6 | {system: ROLE_ORGANIZATION_ADMIN, user_friendly: "Administrator"}, 7 | {system: ROLE_TEAM_MEMBER, user_friendly: "Team member"}, 8 | ]; 9 | 10 | export const showUserFriendlyRoles = (roles) => { 11 | var rolesToReturn = []; 12 | 13 | for (const e of USER_FRIENDLY_ROLES) { 14 | if (roles.includes(e.system)) { 15 | rolesToReturn.push(e.user_friendly); 16 | } 17 | } 18 | 19 | return rolesToReturn.join(', '); 20 | } 21 | -------------------------------------------------------------------------------- /ui/src/components/breadcrumbs.component.js: -------------------------------------------------------------------------------- 1 | import useBreadcrumbs from "use-react-router-breadcrumbs"; 2 | import { Link } from "react-router-dom"; 3 | import routes from "../routes"; 4 | 5 | const Breadcrumbs = () => { 6 | 7 | const breadcrumbs = useBreadcrumbs(routes, { disableDefaults: true }); 8 | 9 | return ( 10 | <> 11 | {breadcrumbs.map(({ match, breadcrumb }, index) => ( 12 | 13 | {breadcrumb} 14 | {index < breadcrumbs.length - 1 && " > "} 15 | 16 | ))} 17 | 18 | ); 19 | }; 20 | 21 | export default Breadcrumbs; 22 | -------------------------------------------------------------------------------- /ui/src/components/formControls/input.component.js: -------------------------------------------------------------------------------- 1 | import React, { Fragment } from "react"; 2 | import PropTypes from "prop-types"; 3 | import { control } from "react-validation"; 4 | 5 | const Input = ({ error, isChanged, isUsed, ...props }) => ( 6 | 7 | 8 | {isChanged && isUsed && error} 9 | 10 | ); 11 | 12 | Input.propTypes = { 13 | error: PropTypes.oneOfType([PropTypes.node, PropTypes.string]) 14 | }; 15 | 16 | export default control(Input); 17 | -------------------------------------------------------------------------------- /ui/src/components/formControls/textarea.component.js: -------------------------------------------------------------------------------- 1 | import React, { Fragment } from "react"; 2 | import PropTypes from "prop-types"; 3 | import { control } from "react-validation"; 4 | import TextareaAutosize from 'react-textarea-autosize'; 5 | 6 | const Textarea = ({ error, isChanged, isUsed, ...props }) => ( 7 | 8 | 9 | {isChanged && isUsed && error} 10 | 11 | ); 12 | 13 | Textarea.propTypes = { 14 | error: PropTypes.oneOfType([PropTypes.node, PropTypes.string]) 15 | }; 16 | 17 | export default control(Textarea); 18 | -------------------------------------------------------------------------------- /ui/src/components/pipelines/initial-elements.js: -------------------------------------------------------------------------------- 1 | export default [ 2 | { 3 | id: '1', 4 | type: 'query', 5 | data: { 6 | name: 'DB query', 7 | }, 8 | position: { x: -200, y: 100 }, 9 | }, 10 | { 11 | id: '2', 12 | type: 'condition', 13 | data: { 14 | name: 'IF A > B', 15 | }, 16 | position: { x: -90, y: 100 }, 17 | }, 18 | { 19 | id: '3', 20 | type: 'aggregator', 21 | data: { 22 | name: 'AVG(amount)', 23 | }, 24 | position: { x: 50, y: 100 }, 25 | }, 26 | { 27 | id: '4', 28 | type: 'api_call', 29 | data: { 30 | name: 'Call API', 31 | }, 32 | position: { x: 250, y: 100 }, 33 | }, 34 | { 35 | id: '5', 36 | type: 'transformer', 37 | data: { 38 | name: 'Data transformation', 39 | }, 40 | position: { x: 50, y: 200 }, 41 | }, 42 | { 43 | id: '6', 44 | type: 'notification', 45 | data: { 46 | name: 'Send SMS', 47 | }, 48 | position: { x: 250, y: 200 }, 49 | }, 50 | { id: 'e1-2', source: '2', target: '1' }, 51 | { id: 'e2-3', source: '3', target: '2', label: 'true' }, 52 | { id: 'e3-4', source: '4', target: '3' }, 53 | { id: 'e2-5', source: '5', target: '2', label: 'false' }, 54 | { id: 'e5-6', source: '6', target: '5' }, 55 | ]; 56 | -------------------------------------------------------------------------------- /ui/src/components/pipelines/nodes/APICallNode.component.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import { Handle } from 'react-flow-renderer'; 4 | 5 | export const APICallNodeComponent = ({ data }) => { 6 | return ( 7 |
8 | 9 |
{data.name}
10 | 11 |
12 | ); 13 | }; 14 | -------------------------------------------------------------------------------- /ui/src/components/pipelines/nodes/aggregatorNode.component.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import { Handle } from 'react-flow-renderer'; 4 | 5 | export const AggregatorNodeComponent = ({ data }) => { 6 | return ( 7 |
8 | 9 |
{data.name}
10 | 11 |
12 | ); 13 | }; 14 | -------------------------------------------------------------------------------- /ui/src/components/pipelines/nodes/codeNode.component.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import { Handle } from 'react-flow-renderer'; 4 | 5 | export const CodeNodeComponent = ({ data }) => { 6 | return ( 7 |
8 | 9 |
{data.name}
10 | 11 |
12 | ); 13 | }; 14 | -------------------------------------------------------------------------------- /ui/src/components/pipelines/nodes/conditionNode.component.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import { Handle } from 'react-flow-renderer'; 4 | 5 | import Tooltip from '@mui/material/Tooltip'; 6 | 7 | export const CONDITION_CONNECTOR_TRUE = "condition-connector-true"; 8 | export const CONDITION_CONNECTOR_FALSE = "condition-connector-false"; 9 | 10 | export const ConditionNodeComponent = ({ data }) => { 11 | return ( 12 | <> 13 |
14 | 15 |
{data.name}
16 | 17 | 23 | 24 | 25 | 31 | 32 |
33 | 34 | ); 35 | }; 36 | -------------------------------------------------------------------------------- /ui/src/components/pipelines/nodes/externalTriggerNode.component.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import { Handle } from 'react-flow-renderer'; 4 | 5 | export const ExternalTriggerNodeComponent = ({ data }) => { 6 | return ( 7 |
8 | 9 |
{data.name}
10 | 11 |
12 | ); 13 | }; 14 | -------------------------------------------------------------------------------- /ui/src/components/pipelines/nodes/filterNode.component.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import { Handle } from 'react-flow-renderer'; 4 | 5 | export const FilterNodeComponent = ({ data }) => { 6 | return ( 7 |
8 | 9 |
{data.name}
10 | 11 |
12 | ); 13 | }; 14 | -------------------------------------------------------------------------------- /ui/src/components/pipelines/nodes/forEachNode.component.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import { Handle } from 'react-flow-renderer'; 4 | 5 | export const ForEachNodeComponent = ({ data }) => { 6 | return ( 7 |
8 | 9 |
{data.name}
10 | 11 |
12 | ); 13 | }; 14 | -------------------------------------------------------------------------------- /ui/src/components/pipelines/nodes/gptNode.component.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import { Handle } from 'react-flow-renderer'; 4 | 5 | export const GptNodeComponent = ({ data }) => { 6 | return ( 7 |
8 | 9 |
{data.name}
10 | 11 |
12 | ); 13 | }; 14 | -------------------------------------------------------------------------------- /ui/src/components/pipelines/nodes/mergeNode.component.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import { Handle } from 'react-flow-renderer'; 4 | 5 | export const MergeNodeComponent = ({ data }) => { 6 | return ( 7 |
8 | 9 |
{data.name}
10 | 11 |
12 | ); 13 | }; 14 | -------------------------------------------------------------------------------- /ui/src/components/pipelines/nodes/notificationNode.component.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import { Handle } from 'react-flow-renderer'; 4 | 5 | export const NotificationNodeComponent = ({ data }) => { 6 | return ( 7 |
8 | 9 |
{data.name}
10 |
11 | ); 12 | }; 13 | -------------------------------------------------------------------------------- /ui/src/components/pipelines/nodes/pipelineRunNode.component.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import { Handle } from 'react-flow-renderer'; 4 | 5 | export const PipelineRunNodeComponent = ({ data }) => { 6 | return ( 7 |
8 | 9 |
{data.name}
10 |
11 | ); 12 | }; 13 | -------------------------------------------------------------------------------- /ui/src/components/pipelines/nodes/processorNode.component.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import { Handle } from 'react-flow-renderer'; 4 | 5 | export const ProcessorNodeComponent = ({ data }) => { 6 | return ( 7 |
8 | 9 |
{data.name}
10 | 11 |
12 | ); 13 | }; 14 | -------------------------------------------------------------------------------- /ui/src/components/pipelines/nodes/queryNode.component.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import { Handle } from 'react-flow-renderer'; 4 | 5 | export const QueryNodeComponent = ({ data }) => { 6 | return ( 7 |
8 | 9 |
{data.name}
10 | 11 |
12 | ); 13 | }; 14 | -------------------------------------------------------------------------------- /ui/src/components/pipelines/nodes/transformerNode.component.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import { Handle } from 'react-flow-renderer'; 4 | 5 | export const TransformerNodeComponent = ({ data }) => { 6 | return ( 7 |
8 | 9 |
{data.name}
10 | 11 |
12 | ); 13 | }; 14 | -------------------------------------------------------------------------------- /ui/src/components/pipelines/pipelineRunOutput.component.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import Spinner from "react-bootstrap/Spinner"; 4 | 5 | class PipelineRunOutput extends React.Component { 6 | constructor(props) { 7 | super(props); 8 | 9 | this.state = {}; 10 | } 11 | 12 | render() { 13 | const { finished, output } = this.props; 14 | 15 | return ( 16 |
17 | {output} 18 | { 19 | finished !== true 20 | &&
21 | } 22 |
23 | ); 24 | } 25 | } 26 | 27 | export default PipelineRunOutput; 28 | -------------------------------------------------------------------------------- /ui/src/components/timeAgo.component.js: -------------------------------------------------------------------------------- 1 | const units = [ 2 | 'year', 3 | 'month', 4 | 'week', 5 | 'day', 6 | 'hour', 7 | 'minute', 8 | 'second', 9 | ]; 10 | 11 | export const TimeAgo = (dateTime) => { 12 | //let dateTime = DateTime.fromSQL(date, { zone: 'utc'}) 13 | const diff = dateTime.diffNow().shiftTo(...units); 14 | const unit = units.find((unit) => diff.get(unit) !== 0) || 'second'; 15 | 16 | const relativeFormatter = new Intl.RelativeTimeFormat('en', { 17 | numeric: 'auto', 18 | }); 19 | return relativeFormatter.format(Math.trunc(diff.as(unit)), unit); 20 | }; 21 | -------------------------------------------------------------------------------- /ui/src/containers/content.js: -------------------------------------------------------------------------------- 1 | import React, { Suspense } from 'react' 2 | import { 3 | Route, 4 | Routes 5 | } from 'react-router-dom' 6 | 7 | // routes config 8 | import routes from '../routes' 9 | 10 | const loading = ( 11 |
12 |
13 |
14 | ) 15 | 16 | const Content = () => { 17 | return ( 18 |
19 |
20 | 21 | 22 | {routes.map((route, idx) => { 23 | return route.element && ( 24 | 31 | ) 32 | })} 33 | 34 | 35 |
36 |
37 | ) 38 | } 39 | 40 | export default React.memo(Content) 41 | -------------------------------------------------------------------------------- /ui/src/containers/footer.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | 3 | import Navbar from 'react-bootstrap/Navbar'; 4 | 5 | const Footer = () => { 6 | return ( 7 | <> 8 | 9 | © Ylem GmbH, 2024 10 | 11 | 12 | ) 13 | } 14 | 15 | export default Footer 16 | -------------------------------------------------------------------------------- /ui/src/containers/header.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | 3 | import Navbar from 'react-bootstrap/Navbar'; 4 | import Row from 'react-bootstrap/Row'; 5 | import Col from 'react-bootstrap/Col'; 6 | 7 | import Breadcrumbs from "../components/breadcrumbs.component"; 8 | 9 | import {HeaderDropdown, HeaderSearch} from './index' 10 | 11 | const Header = () => { 12 | return ( 13 | <> 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 |
25 | 26 |
27 | 28 | ) 29 | } 30 | 31 | export default Header 32 | -------------------------------------------------------------------------------- /ui/src/containers/index.js: -------------------------------------------------------------------------------- 1 | import Content from './content' 2 | import Header from './header' 3 | import Footer from './footer' 4 | import Sidebar from './sidebar' 5 | import HeaderDropdown from './headerDropdown' 6 | import HeaderSearch from './headerSearch' 7 | 8 | export { 9 | Content, 10 | Header, 11 | Footer, 12 | Sidebar, 13 | HeaderDropdown, 14 | HeaderSearch, 15 | } 16 | -------------------------------------------------------------------------------- /ui/src/containers/layout.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import Container from 'react-bootstrap/Container'; 3 | import Row from 'react-bootstrap/Row'; 4 | import Col from 'react-bootstrap/Col'; 5 | 6 | import { 7 | Header, 8 | Sidebar, 9 | Content 10 | } from './index'; 11 | 12 | const Layout = () => { 13 | 14 | return ( 15 | <> 16 | 17 | 18 | 19 | 20 | 21 | 22 |
23 | 24 | 25 | 26 | 27 | 28 | ) 29 | } 30 | 31 | export default Layout 32 | -------------------------------------------------------------------------------- /ui/src/helpers/history.js: -------------------------------------------------------------------------------- 1 | import { createBrowserHistory } from "history"; 2 | 3 | export const history = createBrowserHistory(); 4 | -------------------------------------------------------------------------------- /ui/src/images/api-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/api-i.png -------------------------------------------------------------------------------- /ui/src/images/api.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/api.png -------------------------------------------------------------------------------- /ui/src/images/aws-rds-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/aws-rds-i.png -------------------------------------------------------------------------------- /ui/src/images/clickhouse-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/clickhouse-i.png -------------------------------------------------------------------------------- /ui/src/images/elasticsearch-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/elasticsearch-i.png -------------------------------------------------------------------------------- /ui/src/images/email-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/email-i.png -------------------------------------------------------------------------------- /ui/src/images/folder.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/folder.png -------------------------------------------------------------------------------- /ui/src/images/google-bigquery-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/google-bigquery-i.png -------------------------------------------------------------------------------- /ui/src/images/google-cloud-sql-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/google-cloud-sql-i.png -------------------------------------------------------------------------------- /ui/src/images/google-generic-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/google-generic-i.png -------------------------------------------------------------------------------- /ui/src/images/google-sheets-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/google-sheets-i.png -------------------------------------------------------------------------------- /ui/src/images/hubspot-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/hubspot-i.png -------------------------------------------------------------------------------- /ui/src/images/immuta-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/immuta-i.png -------------------------------------------------------------------------------- /ui/src/images/incidentio-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/incidentio-i.png -------------------------------------------------------------------------------- /ui/src/images/info.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/info.png -------------------------------------------------------------------------------- /ui/src/images/jenkins-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/jenkins-i.png -------------------------------------------------------------------------------- /ui/src/images/jira-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/jira-i.png -------------------------------------------------------------------------------- /ui/src/images/microsoft-azure-sql-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/microsoft-azure-sql-i.png -------------------------------------------------------------------------------- /ui/src/images/mysql-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/mysql-i.png -------------------------------------------------------------------------------- /ui/src/images/opsgenie-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/opsgenie-i.png -------------------------------------------------------------------------------- /ui/src/images/planet-scale-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/planet-scale-i.png -------------------------------------------------------------------------------- /ui/src/images/postgresql-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/postgresql-i.png -------------------------------------------------------------------------------- /ui/src/images/postgresql.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/postgresql.png -------------------------------------------------------------------------------- /ui/src/images/question-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/question-i.png -------------------------------------------------------------------------------- /ui/src/images/redshift-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/redshift-i.png -------------------------------------------------------------------------------- /ui/src/images/salesforce-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/salesforce-i.png -------------------------------------------------------------------------------- /ui/src/images/slack-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/slack-i.png -------------------------------------------------------------------------------- /ui/src/images/slack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/slack.png -------------------------------------------------------------------------------- /ui/src/images/sms-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/sms-i.png -------------------------------------------------------------------------------- /ui/src/images/snowflake-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/snowflake-i.png -------------------------------------------------------------------------------- /ui/src/images/snowflake.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/snowflake.png -------------------------------------------------------------------------------- /ui/src/images/sql-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/sql-i.png -------------------------------------------------------------------------------- /ui/src/images/tableau-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/tableau-i.png -------------------------------------------------------------------------------- /ui/src/images/taskFailed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/taskFailed.png -------------------------------------------------------------------------------- /ui/src/images/taskPaused.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/taskPaused.png -------------------------------------------------------------------------------- /ui/src/images/taskPending.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/taskPending.png -------------------------------------------------------------------------------- /ui/src/images/taskRunning.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/taskRunning.gif -------------------------------------------------------------------------------- /ui/src/images/taskRunning.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/taskRunning.png -------------------------------------------------------------------------------- /ui/src/images/taskSuccess.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/taskSuccess.png -------------------------------------------------------------------------------- /ui/src/images/tour/welcome.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/tour/welcome.png -------------------------------------------------------------------------------- /ui/src/images/visualhomepage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/visualhomepage.png -------------------------------------------------------------------------------- /ui/src/images/whatsapp-i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ylem-co/ylem/10c474a04827427b745ae97d84cff57df128b7bd/ui/src/images/whatsapp-i.png -------------------------------------------------------------------------------- /ui/src/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 4 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', 5 | sans-serif; 6 | -webkit-font-smoothing: antialiased; 7 | -moz-osx-font-smoothing: grayscale; 8 | } 9 | 10 | code { 11 | font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', 12 | monospace; 13 | } 14 | -------------------------------------------------------------------------------- /ui/src/polyill.js: -------------------------------------------------------------------------------- 1 | // CustomEvent() constructor functionality in IE9, IE10, IE11 2 | (function () { 3 | 4 | if ( typeof window.CustomEvent === "function" ) return false 5 | 6 | function CustomEvent ( event, params ) { 7 | params = params || { bubbles: false, cancelable: false, detail: undefined } 8 | var evt = document.createEvent( 'CustomEvent' ) 9 | evt.initCustomEvent( event, params.bubbles, params.cancelable, params.detail ) 10 | return evt 11 | } 12 | 13 | CustomEvent.prototype = window.Event.prototype 14 | 15 | window.CustomEvent = CustomEvent 16 | })() 17 | -------------------------------------------------------------------------------- /ui/src/reducers/auth.js: -------------------------------------------------------------------------------- 1 | import { 2 | REGISTER_SUCCESS, 3 | REGISTER_FAIL, 4 | LOGIN_SUCCESS, 5 | LOGIN_FAIL, 6 | LOGOUT, 7 | } from "../actions/types"; 8 | 9 | const user = JSON.parse(localStorage.getItem("user")); 10 | 11 | const initialState = user 12 | ? { isLoggedIn: true, user } 13 | : { isLoggedIn: false, user: null }; 14 | 15 | export default function auth(state = initialState, action) { 16 | const { type, payload } = action; 17 | 18 | switch (type) { 19 | case REGISTER_SUCCESS: 20 | return { 21 | ...state, 22 | isLoggedIn: false, 23 | }; 24 | case REGISTER_FAIL: 25 | return { 26 | ...state, 27 | isLoggedIn: false, 28 | }; 29 | case LOGIN_SUCCESS: 30 | return { 31 | ...state, 32 | isLoggedIn: true, 33 | user: payload.user, 34 | }; 35 | case LOGIN_FAIL: 36 | return { 37 | ...state, 38 | isLoggedIn: false, 39 | user: null, 40 | }; 41 | case LOGOUT: 42 | return { 43 | ...state, 44 | isLoggedIn: false, 45 | user: null, 46 | }; 47 | default: 48 | return state; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /ui/src/reducers/index.js: -------------------------------------------------------------------------------- 1 | import { combineReducers } from "redux"; 2 | import auth from "./auth"; 3 | import message from "./message"; 4 | 5 | export default combineReducers({ 6 | auth, 7 | message, 8 | }); 9 | -------------------------------------------------------------------------------- /ui/src/reducers/message.js: -------------------------------------------------------------------------------- 1 | import { SET_MESSAGE, CLEAR_MESSAGE } from "../actions/types"; 2 | 3 | const initialState = {}; 4 | 5 | export default function message(state = initialState, action) { 6 | const { type, payload } = action; 7 | 8 | switch (type) { 9 | case SET_MESSAGE: 10 | return { message: payload }; 11 | 12 | case CLEAR_MESSAGE: 13 | return { message: "" }; 14 | 15 | default: 16 | return state; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /ui/src/reportWebVitals.js: -------------------------------------------------------------------------------- 1 | const reportWebVitals = onPerfEntry => { 2 | if (onPerfEntry && onPerfEntry instanceof Function) { 3 | import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { 4 | getCLS(onPerfEntry); 5 | getFID(onPerfEntry); 6 | getFCP(onPerfEntry); 7 | getLCP(onPerfEntry); 8 | getTTFB(onPerfEntry); 9 | }); 10 | } 11 | }; 12 | 13 | export default reportWebVitals; 14 | -------------------------------------------------------------------------------- /ui/src/services/invitation.service.js: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | 3 | const API_URL = "/user-api/"; 4 | 5 | class InvitationService { 6 | getPendingByOrganization = async(uuid) => { 7 | var token = localStorage.getItem("token"); 8 | 9 | return axios 10 | .get( 11 | API_URL + 'organization/' + uuid + '/pending-invitations', 12 | {}, 13 | { headers: { Authorization: 'Bearer ' + token } } 14 | ); 15 | } 16 | 17 | sendInvitations = async(uuid, emails) => { 18 | var token = localStorage.getItem("token"); 19 | 20 | return axios 21 | .post( 22 | API_URL + 'organization/' + uuid + '/invitations', 23 | {emails}, 24 | { headers: { Authorization: 'Bearer ' + token } } 25 | ); 26 | } 27 | 28 | validateInvitationKey = async(key) => { 29 | return axios 30 | .post( 31 | API_URL + 'invitations/' + key + '/validate', 32 | {}, 33 | {} 34 | ); 35 | } 36 | } 37 | 38 | export default new InvitationService(); 39 | -------------------------------------------------------------------------------- /ui/src/services/oauth.service.js: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | 3 | const API_URL = "/oauth-api/"; 4 | 5 | class OAuthService { 6 | getClients = async(uuid) => { 7 | var token = localStorage.getItem("token"); 8 | 9 | return axios 10 | .get( 11 | API_URL + 'clients/', 12 | {}, 13 | { headers: { Authorization: 'Bearer ' + token } } 14 | ); 15 | } 16 | 17 | createClient = async(name) => { 18 | var token = localStorage.getItem("token"); 19 | 20 | return axios 21 | .post( 22 | API_URL + 'clients/', 23 | { name }, 24 | { headers: { Authorization: 'Bearer ' + token } } 25 | ); 26 | } 27 | 28 | deleteClient = async(uuid) => { 29 | var token = localStorage.getItem("token") 30 | return axios 31 | .post( 32 | API_URL + 'client/' + uuid + '/delete/', 33 | {}, 34 | { headers: { Authorization: 'Bearer ' + token } } 35 | ); 36 | } 37 | } 38 | 39 | export default new OAuthService(); 40 | -------------------------------------------------------------------------------- /ui/src/services/organization.service.js: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | 3 | const API_URL = "/user-api/"; 4 | 5 | class OrganizationService { 6 | updateOrganization(uuid, name) { 7 | var token = localStorage.getItem("token") 8 | return axios 9 | .post( 10 | API_URL + 'organization/' + uuid, 11 | {name}, 12 | { headers: { Authorization: 'Bearer ' + token } } 13 | ); 14 | } 15 | 16 | getMyOrganization = async() => { 17 | var token = localStorage.getItem("token"); 18 | 19 | return axios 20 | .get( 21 | API_URL + 'my-organization', 22 | {}, 23 | { headers: { Authorization: 'Bearer ' + token } } 24 | ); 25 | } 26 | } 27 | 28 | export default new OrganizationService(); 29 | -------------------------------------------------------------------------------- /ui/src/setupTests.js: -------------------------------------------------------------------------------- 1 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 2 | // allows you to do things like: 3 | // expect(element).toHaveTextContent(/react/i) 4 | // learn more: https://github.com/testing-library/jest-dom 5 | import '@testing-library/jest-dom'; 6 | -------------------------------------------------------------------------------- /ui/src/store.js: -------------------------------------------------------------------------------- 1 | import { createStore, applyMiddleware } from "redux"; 2 | import { composeWithDevTools } from "@redux-devtools/extension"; 3 | import { thunk } from 'redux-thunk'; 4 | import rootReducer from "./reducers"; 5 | 6 | const middleware = [thunk]; 7 | 8 | const store = createStore( 9 | rootReducer, 10 | composeWithDevTools(applyMiddleware(...middleware)) 11 | ); 12 | 13 | export default store; 14 | -------------------------------------------------------------------------------- /ui/src/views/pages/page404/Page404.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import {Link} from 'react-router-dom'; 3 | 4 | const Page404 = () => { 5 | return ( 6 | <> 7 |

404

8 |

The page you were looking for was not found.

9 |

10 | But don't feel lost and forgotten, we are always there for you. Please go to the main page and start from there. 11 |

12 | 13 | ) 14 | } 15 | 16 | export default Page404 17 | --------------------------------------------------------------------------------