├── .dockerignore ├── .env ├── .gitignore ├── LICENSE ├── README.md ├── Rakefile ├── docker-compose.yml ├── document_search_engine_architecture.drawio ├── document_search_engine_logo.webp ├── elasticsearch ├── certificate │ ├── elastic-certificates.p12 │ └── elastic-stack-ca.p12 └── config │ └── elasticsearch.yml ├── hadoop.env ├── images ├── apache_kafka_1.PNG ├── apache_kafka_2.PNG ├── apache_nifi_1.PNG ├── apache_nifi_2.PNG ├── apache_nifi_3.PNG ├── apache_nifi_4.PNG ├── apache_nifi_5.PNG ├── apache_nifi_6.PNG ├── apache_nifi_7.PNG ├── consul_1.PNG ├── consul_2.PNG ├── consul_3.PNG ├── document_search_engine_architecture.jpg ├── document_search_engine_architecture.png ├── gateway_1.PNG ├── gateway_2.PNG ├── gateway_3.PNG ├── gateway_4.PNG ├── gateway_5.PNG ├── gateway_6.PNG ├── gateway_7.PNG ├── gateway_8.PNG ├── gateway_9.PNG ├── hdfs_1.PNG ├── hdfs_2.PNG ├── hdfs_3.PNG ├── keycloak_1.PNG ├── keycloak_2.PNG ├── keycloak_3.PNG ├── keycloak_4.PNG ├── keycloak_5.PNG ├── kibana_1.PNG ├── kibana_2.PNG ├── kibana_3.PNG ├── kibana_4.PNG ├── kibana_5.PNG ├── kibana_6.PNG ├── kibana_7.PNG ├── mongodb_1.PNG ├── mongodb_2.PNG ├── mongodb_3.PNG ├── mongodb_4.PNG ├── mongodb_5.PNG └── spring_cloud_oauth2_use_case.png ├── keycloak ├── Dockerfile └── docker-entrypoint-initdb.d │ ├── keycloak_db.sql │ └── restore_database.sh ├── kibana ├── certificate │ └── certificate.pem └── config │ └── kibana.yml ├── logstash ├── Dockerfile ├── certificate │ └── certificate.pem ├── config │ └── logstash.yml ├── drivers │ ├── gson-2.8.6.jar │ ├── mongo-java-driver-3.12.6.jar │ └── mongojdbc2.3.jar └── pipeline │ └── ingest_pipeline.conf ├── microservices ├── .gitignore ├── files_api_gateway │ ├── Dockerfile │ ├── pom.xml │ ├── resources │ │ ├── entrypoint.sh │ │ └── wait-for-it.sh │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── dreamsoftware │ │ │ └── documentsearchengine │ │ │ ├── FilesApiGatewayApplication.java │ │ │ └── security │ │ │ └── gateway │ │ │ └── config │ │ │ └── SecurityConfig.java │ │ └── resources │ │ └── application.yml ├── files_commons │ ├── .gitignore │ ├── .mvn │ │ └── wrapper │ │ │ ├── MavenWrapperDownloader.java │ │ │ ├── maven-wrapper.jar │ │ │ └── maven-wrapper.properties │ ├── mvnw │ ├── mvnw.cmd │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── dreamsoftware │ │ │ └── documentsearchengine │ │ │ ├── config │ │ │ ├── JacksonConfig.java │ │ │ └── SecurityConfig.java │ │ │ └── web │ │ │ └── core │ │ │ ├── APIResponse.java │ │ │ ├── ErrorResponseDTO.java │ │ │ ├── FieldErrorDTO.java │ │ │ ├── IResponseCodeTypes.java │ │ │ ├── IVersionable.java │ │ │ ├── ResponseHelper.java │ │ │ ├── ResponseStatusEnum.java │ │ │ └── SupportController.java │ │ └── resources │ │ └── application.properties ├── files_management │ ├── Dockerfile │ ├── lombok.config │ ├── pom.xml │ ├── resources │ │ └── wait-for-it.sh │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── dreamsoftware │ │ │ └── documentsearchengine │ │ │ ├── FilesManagementServiceApplication.java │ │ │ ├── client │ │ │ ├── IFilesMetadataClient.java │ │ │ └── interceptor │ │ │ │ └── FeignClientInterceptor.java │ │ │ ├── config │ │ │ ├── AppStreamsConfig.java │ │ │ ├── OpenApiConfig.java │ │ │ └── props │ │ │ │ ├── SFTPProperties.java │ │ │ │ └── UploadProperties.java │ │ │ ├── service │ │ │ ├── IFilesManagementService.java │ │ │ └── impl │ │ │ │ └── FilesManagementServiceImpl.java │ │ │ └── web │ │ │ ├── controller │ │ │ ├── FilesManagementController.java │ │ │ ├── FilesManagementResponseCodeEnum.java │ │ │ └── error │ │ │ │ ├── FilesManagementErrorController.java │ │ │ │ └── exception │ │ │ │ ├── DeleteFileException.java │ │ │ │ ├── FileAlreadyProcessedException.java │ │ │ │ └── SaveFileException.java │ │ │ └── dto │ │ │ ├── ProcessedFileDTO.java │ │ │ ├── RequestOperationToProcessedFileDTO.java │ │ │ └── RequestOperationTypeEnum.java │ │ └── resources │ │ └── application.yml ├── files_metadata │ ├── Dockerfile │ ├── nbactions.xml │ ├── pom.xml │ ├── resources │ │ └── wait-for-it.sh │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── dreamsoftware │ │ │ └── documentsearchengine │ │ │ ├── FilesMetadataServiceApplication.java │ │ │ ├── config │ │ │ └── OpenApiConfig.java │ │ │ ├── mapper │ │ │ └── ProcessedFileMapper.java │ │ │ ├── persistence │ │ │ ├── entity │ │ │ │ └── ProcessedFileEntity.java │ │ │ └── repository │ │ │ │ └── FilesProcessedRepository.java │ │ │ ├── service │ │ │ ├── IFilesProcessedService.java │ │ │ └── impl │ │ │ │ └── FilesProcessedServiceImpl.java │ │ │ └── web │ │ │ ├── controller │ │ │ ├── FilesMetadataController.java │ │ │ ├── FilesMetadataResponseCodeEnum.java │ │ │ └── error │ │ │ │ ├── FilesMetadataErrorController.java │ │ │ │ └── exception │ │ │ │ ├── FileProcessedNotFoundException.java │ │ │ │ └── NoFilesProcessedFoundException.java │ │ │ ├── dto │ │ │ └── ProcessedFileDTO.java │ │ │ └── validators │ │ │ ├── ValidObjectId.java │ │ │ └── ValidObjectIdValidator.java │ │ └── resources │ │ └── application.yml ├── files_notifications │ ├── Dockerfile │ ├── nbactions.xml │ ├── pom.xml │ ├── resources │ │ └── wait-for-it.sh │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── dreamsoftware │ │ │ └── documentsearchengine │ │ │ ├── FilesNotificationsServiceApplication.java │ │ │ ├── config │ │ │ ├── AppStreamsConfig.java │ │ │ ├── OpenApiConfig.java │ │ │ ├── WebSocketConfig.java │ │ │ └── props │ │ │ │ └── StompBrokerRelayProps.java │ │ │ ├── handler │ │ │ └── ProcessedFilesNotificationsHandler.java │ │ │ ├── mapper │ │ │ └── ProcessedFileNotificationMapper.java │ │ │ ├── persistence │ │ │ ├── entity │ │ │ │ └── ProcessedFileNotificationEntity.java │ │ │ └── repository │ │ │ │ └── ProcessedFileNotificationRepository.java │ │ │ ├── service │ │ │ ├── IProcessedFileNotificationService.java │ │ │ └── impl │ │ │ │ └── ProcessedFileNotificationServiceImpl.java │ │ │ └── web │ │ │ ├── controller │ │ │ ├── FilesNotificationsController.java │ │ │ ├── FilesNotificationsResponseCodeEnum.java │ │ │ └── error │ │ │ │ ├── FilesNotificationsErrorController.java │ │ │ │ └── exception │ │ │ │ └── NoNotificationsFoundException.java │ │ │ └── dto │ │ │ └── ProcessedFileNotificationDTO.java │ │ └── resources │ │ └── application.yml ├── files_search │ ├── Dockerfile │ ├── nbactions.xml │ ├── pom.xml │ ├── resources │ │ └── wait-for-it.sh │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── dreamsoftware │ │ │ └── documentsearchengine │ │ │ ├── FilesSearchServiceApplication.java │ │ │ ├── config │ │ │ ├── ElasticSearchClientConfig.java │ │ │ └── OpenApiConfig.java │ │ │ ├── mapper │ │ │ └── ProcessedFileMapper.java │ │ │ ├── persistance │ │ │ ├── entity │ │ │ │ └── ProcessedFileEntity.java │ │ │ └── repository │ │ │ │ └── FilesProcessedRepository.java │ │ │ ├── service │ │ │ ├── IFilesProcessedService.java │ │ │ └── impl │ │ │ │ └── FilesProcessedServiceImpl.java │ │ │ └── web │ │ │ ├── controller │ │ │ ├── FilesSearchController.java │ │ │ ├── FilesSearchResponseCodeEnum.java │ │ │ └── error │ │ │ │ ├── FilesSearchErrorController.java │ │ │ │ └── exception │ │ │ │ └── NoFilesProcessedFoundException.java │ │ │ └── dto │ │ │ └── ProcessedFileDTO.java │ │ └── resources │ │ ├── application.yml │ │ └── elastic-certificates.p12 └── pom.xml ├── nifi ├── Dockerfile ├── conf │ ├── core-site.xml │ └── hdfs-site.xml ├── flow.xml.gz └── templates │ └── Ingest_Files_Pipeline_Template_v9.xml ├── notas.txt ├── notas_logstash.txt ├── notas_microservices.txt └── sftp └── users.conf /.dockerignore: -------------------------------------------------------------------------------- 1 | #Ignore everything by default, we only want the Dockerfile and the `root` directory in the build context. 2 | ** 3 | !Dockerfile 4 | !root -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | MONGO_ROOT_USER=devroot 2 | MONGO_ROOT_PASSWORD=devroot 3 | MONGOEXPRESS_LOGIN=dev 4 | MONGOEXPRESS_PASSWORD=dev 5 | MONGO_DB=files -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | target/ 3 | !.mvn/wrapper/maven-wrapper.jar 4 | !**/src/main/**/target/ 5 | !**/src/test/**/target/ 6 | 7 | ### STS ### 8 | .apt_generated 9 | .classpath 10 | .factorypath 11 | .project 12 | .settings 13 | .springBeans 14 | .sts4-cache 15 | 16 | ### IntelliJ IDEA ### 17 | .idea 18 | *.iws 19 | *.iml 20 | *.ipr 21 | 22 | ### NetBeans ### 23 | /nbproject/private/ 24 | /nbbuild/ 25 | /dist/ 26 | /nbdist/ 27 | /.nb-gradle/ 28 | build/ 29 | !**/src/main/**/build/ 30 | !**/src/test/**/build/ 31 | 32 | ### VS Code ### 33 | .vscode/ 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Sergio Sánchez Sánchez 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | task default: %w[deploy] 2 | 3 | 4 | desc "Deploys Architecture and launches all services and daemons needed to properly work" 5 | task :deploy => [ 6 | :cleaning_environment_task, 7 | :start, 8 | :status] do 9 | puts "Deploying services..." 10 | end 11 | 12 | desc "UnDeploy Architecture" 13 | task :undeploy => [:status] do 14 | puts "Undeploy Services" 15 | puts `docker-compose down -v 2>&1` 16 | end 17 | 18 | desc "Start Containers" 19 | task :start => [ :check_docker_task, :login, :check_deployment_file_task ] do 20 | puts "Start Containers" 21 | puts `docker-compose up -d --remove-orphans` 22 | end 23 | 24 | desc "Stop Containers" 25 | task :stop => [ :check_docker_task ] do 26 | puts "Stop Containers" 27 | puts `docker-compose stop 2>&1` 28 | puts `docker-compose rm -f 2>&1` 29 | end 30 | 31 | 32 | desc "Status Containers" 33 | task :status do 34 | puts "Show Containers Status" 35 | puts `docker-compose ps 2>&1` 36 | end 37 | 38 | 39 | desc "Cleaning Evironment Task" 40 | task :cleaning_environment_task do 41 | puts "Cleaning Environment" 42 | puts `docker image prune -af` 43 | puts `docker volume prune -f 2>&1` 44 | end 45 | 46 | 47 | desc "Check Docker and Docker Compose Task" 48 | task :check_docker_task do 49 | puts "Check Docker and Docker Compose ..." 50 | if which('docker') && which('docker-compose') 51 | show_docker_version 52 | show_docker_compose_version 53 | else 54 | raise "Please check that Docker and Docker Compose are visible and accessible in the PATH" 55 | end 56 | end 57 | 58 | 59 | desc "Authenticating with existing credentials" 60 | task :login do 61 | puts `docker login 2>&1` 62 | end 63 | 64 | desc "Check Deployment File" 65 | task :check_deployment_file_task do 66 | puts "Check Deployment File ..." 67 | raise "Deployment file not found, please check availability" unless File.file?("docker-compose.yml") 68 | puts "Deployment File OK" 69 | end 70 | 71 | 72 | 73 | ## Utils Functions 74 | 75 | def show_docker_version 76 | puts `docker version 2>&1` 77 | end 78 | 79 | def show_docker_compose_version 80 | puts `docker-compose version 2>&1` 81 | end 82 | 83 | # Cross-platform way of finding an executable in the $PATH. 84 | # which('ruby') #=> /usr/bin/ruby 85 | def which(cmd) 86 | exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : [''] 87 | ENV['PATH'].split(File::PATH_SEPARATOR).each do |path| 88 | exts.each { |ext| 89 | exe = File.join(path, "#{cmd}#{ext}") 90 | return exe if File.executable?(exe) && !File.directory?(exe) 91 | } 92 | end 93 | return nil 94 | end 95 | -------------------------------------------------------------------------------- /document_search_engine_logo.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/document_search_engine_logo.webp -------------------------------------------------------------------------------- /elasticsearch/certificate/elastic-certificates.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/elasticsearch/certificate/elastic-certificates.p12 -------------------------------------------------------------------------------- /elasticsearch/certificate/elastic-stack-ca.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/elasticsearch/certificate/elastic-stack-ca.p12 -------------------------------------------------------------------------------- /elasticsearch/config/elasticsearch.yml: -------------------------------------------------------------------------------- 1 | cluster.name: ssanchez-elasticsearch-cluster 2 | network.host: 0.0.0.0 3 | xpack.security.enabled: true 4 | xpack.security.transport.ssl.enabled: true 5 | xpack.security.transport.ssl.keystore.type: PKCS12 6 | xpack.security.transport.ssl.truststore.type: PKCS12 7 | xpack.security.transport.ssl.keystore.path: elastic-certificates.p12 8 | xpack.security.transport.ssl.truststore.path: elastic-certificates.p12 9 | xpack.security.transport.ssl.keystore.password: ssanchez00 10 | xpack.security.transport.ssl.truststore.password: ssanchez00 11 | xpack.security.transport.ssl.verification_mode: certificate 12 | 13 | xpack.security.http.ssl.enabled: true 14 | xpack.security.http.ssl.keystore.type: PKCS12 15 | xpack.security.http.ssl.truststore.type: PKCS12 16 | xpack.security.http.ssl.keystore.path: elastic-certificates.p12 17 | xpack.security.http.ssl.truststore.path: elastic-certificates.p12 18 | xpack.security.http.ssl.keystore.password: ssanchez00 19 | xpack.security.http.ssl.truststore.password: ssanchez00 20 | xpack.security.http.ssl.verification_mode: certificate 21 | 22 | -------------------------------------------------------------------------------- /hadoop.env: -------------------------------------------------------------------------------- 1 | CORE_CONF_fs_defaultFS=hdfs://namenode:9000 2 | CORE_CONF_hadoop_http_staticuser_user=root 3 | CORE_CONF_hadoop_proxyuser_hue_hosts=* 4 | CORE_CONF_hadoop_proxyuser_hue_groups=* 5 | CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec 6 | 7 | HDFS_CONF_dfs_webhdfs_enabled=true 8 | HDFS_CONF_dfs_permissions_enabled=false 9 | HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false 10 | 11 | YARN_CONF_yarn_log___aggregation___enable=true 12 | YARN_CONF_yarn_log_server_url=http://historyserver:8188/applicationhistory/logs/ 13 | YARN_CONF_yarn_resourcemanager_recovery_enabled=true 14 | YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore 15 | YARN_CONF_yarn_resourcemanager_scheduler_class=org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler 16 | YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___mb=8192 17 | YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___vcores=4 18 | YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate 19 | YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true 20 | YARN_CONF_yarn_resourcemanager_hostname=resourcemanager 21 | YARN_CONF_yarn_resourcemanager_address=resourcemanager:8032 22 | YARN_CONF_yarn_resourcemanager_scheduler_address=resourcemanager:8030 23 | YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031 24 | YARN_CONF_yarn_timeline___service_enabled=true 25 | YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true 26 | YARN_CONF_yarn_timeline___service_hostname=historyserver 27 | YARN_CONF_mapreduce_map_output_compress=true 28 | YARN_CONF_mapred_map_output_compress_codec=org.apache.hadoop.io.compress.SnappyCodec 29 | YARN_CONF_yarn_nodemanager_resource_memory___mb=16384 30 | YARN_CONF_yarn_nodemanager_resource_cpu___vcores=8 31 | YARN_CONF_yarn_nodemanager_disk___health___checker_max___disk___utilization___per___disk___percentage=98.5 32 | YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs 33 | YARN_CONF_yarn_nodemanager_aux___services=mapreduce_shuffle 34 | 35 | MAPRED_CONF_mapreduce_framework_name=yarn 36 | MAPRED_CONF_mapred_child_java_opts=-Xmx4096m 37 | MAPRED_CONF_mapreduce_map_memory_mb=4096 38 | MAPRED_CONF_mapreduce_reduce_memory_mb=8192 39 | MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m 40 | MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m 41 | MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/ 42 | MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/ 43 | MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/ 44 | -------------------------------------------------------------------------------- /images/apache_kafka_1.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/apache_kafka_1.PNG -------------------------------------------------------------------------------- /images/apache_kafka_2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/apache_kafka_2.PNG -------------------------------------------------------------------------------- /images/apache_nifi_1.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/apache_nifi_1.PNG -------------------------------------------------------------------------------- /images/apache_nifi_2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/apache_nifi_2.PNG -------------------------------------------------------------------------------- /images/apache_nifi_3.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/apache_nifi_3.PNG -------------------------------------------------------------------------------- /images/apache_nifi_4.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/apache_nifi_4.PNG -------------------------------------------------------------------------------- /images/apache_nifi_5.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/apache_nifi_5.PNG -------------------------------------------------------------------------------- /images/apache_nifi_6.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/apache_nifi_6.PNG -------------------------------------------------------------------------------- /images/apache_nifi_7.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/apache_nifi_7.PNG -------------------------------------------------------------------------------- /images/consul_1.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/consul_1.PNG -------------------------------------------------------------------------------- /images/consul_2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/consul_2.PNG -------------------------------------------------------------------------------- /images/consul_3.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/consul_3.PNG -------------------------------------------------------------------------------- /images/document_search_engine_architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/document_search_engine_architecture.jpg -------------------------------------------------------------------------------- /images/document_search_engine_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/document_search_engine_architecture.png -------------------------------------------------------------------------------- /images/gateway_1.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/gateway_1.PNG -------------------------------------------------------------------------------- /images/gateway_2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/gateway_2.PNG -------------------------------------------------------------------------------- /images/gateway_3.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/gateway_3.PNG -------------------------------------------------------------------------------- /images/gateway_4.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/gateway_4.PNG -------------------------------------------------------------------------------- /images/gateway_5.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/gateway_5.PNG -------------------------------------------------------------------------------- /images/gateway_6.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/gateway_6.PNG -------------------------------------------------------------------------------- /images/gateway_7.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/gateway_7.PNG -------------------------------------------------------------------------------- /images/gateway_8.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/gateway_8.PNG -------------------------------------------------------------------------------- /images/gateway_9.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/gateway_9.PNG -------------------------------------------------------------------------------- /images/hdfs_1.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/hdfs_1.PNG -------------------------------------------------------------------------------- /images/hdfs_2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/hdfs_2.PNG -------------------------------------------------------------------------------- /images/hdfs_3.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/hdfs_3.PNG -------------------------------------------------------------------------------- /images/keycloak_1.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/keycloak_1.PNG -------------------------------------------------------------------------------- /images/keycloak_2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/keycloak_2.PNG -------------------------------------------------------------------------------- /images/keycloak_3.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/keycloak_3.PNG -------------------------------------------------------------------------------- /images/keycloak_4.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/keycloak_4.PNG -------------------------------------------------------------------------------- /images/keycloak_5.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/keycloak_5.PNG -------------------------------------------------------------------------------- /images/kibana_1.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/kibana_1.PNG -------------------------------------------------------------------------------- /images/kibana_2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/kibana_2.PNG -------------------------------------------------------------------------------- /images/kibana_3.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/kibana_3.PNG -------------------------------------------------------------------------------- /images/kibana_4.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/kibana_4.PNG -------------------------------------------------------------------------------- /images/kibana_5.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/kibana_5.PNG -------------------------------------------------------------------------------- /images/kibana_6.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/kibana_6.PNG -------------------------------------------------------------------------------- /images/kibana_7.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/kibana_7.PNG -------------------------------------------------------------------------------- /images/mongodb_1.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/mongodb_1.PNG -------------------------------------------------------------------------------- /images/mongodb_2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/mongodb_2.PNG -------------------------------------------------------------------------------- /images/mongodb_3.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/mongodb_3.PNG -------------------------------------------------------------------------------- /images/mongodb_4.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/mongodb_4.PNG -------------------------------------------------------------------------------- /images/mongodb_5.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/mongodb_5.PNG -------------------------------------------------------------------------------- /images/spring_cloud_oauth2_use_case.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/images/spring_cloud_oauth2_use_case.png -------------------------------------------------------------------------------- /keycloak/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM postgres 2 | 3 | COPY ./docker-entrypoint-initdb.d/ /docker-entrypoint-initdb.d/ 4 | -------------------------------------------------------------------------------- /keycloak/docker-entrypoint-initdb.d/restore_database.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd /docker-entrypoint-initdb.d/ 4 | 5 | export PGPASSWORD=ssanchez00 6 | echo "Restoring Keycloak Database" 7 | psql -U keycloak -p 5432 -d keycloak < "keycloak_db.sql" -------------------------------------------------------------------------------- /kibana/certificate/certificate.pem: -------------------------------------------------------------------------------- 1 | Bag Attributes 2 | friendlyName: ca 3 | localKeyID: 54 69 6D 65 20 31 36 30 32 39 33 30 35 38 34 30 34 38 4 | subject=CN = Elastic Certificate Tool Autogenerated CA 5 | 6 | issuer=CN = Elastic Certificate Tool Autogenerated CA 7 | 8 | -----BEGIN CERTIFICATE----- 9 | MIIDSTCCAjGgAwIBAgIUWKBzaaujWnYRWGCsNp2sGPU1GzMwDQYJKoZIhvcNAQEL 10 | BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l 11 | cmF0ZWQgQ0EwHhcNMjAxMDE3MTAyOTM4WhcNMjMxMDE3MTAyOTM4WjA0MTIwMAYD 12 | VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC 13 | ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAIZZHqX/18v9cF9eJXiSe9Gd 14 | /FwblR7iPT2GIL3q/p4rmvTSatmOsOr2rbrSAj0mEgNuPTAKDHOH/eMQi6xX3jOg 15 | 1VjYl/a379lEAtDFTOFs4+bZgNPOCYrK24XYYhvqleaPb2uuJCp1ZIScYUO+vTf2 16 | WeEEKxD9D2TfxS4LwYc4jzsLVNvgzC+hXR7YDRE+8r6toI6g5SszNNo4WmZ0/DcV 17 | 6lfmJIO5F1rcN13H9DWvsrxBek+GSm0xEesIVCsOvFY832nsEI8em8Bzlayx1nCW 18 | mTA1F2l4xWFOlT78bKMrA5VszKfFvsf7hwjMGyGHPLFCYJW7cbYWiMWgfIgRWZMC 19 | AwEAAaNTMFEwHQYDVR0OBBYEFN2k0tq9a6AZo0YHk09eVckptkaNMB8GA1UdIwQY 20 | MBaAFN2k0tq9a6AZo0YHk09eVckptkaNMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI 21 | hvcNAQELBQADggEBAGg/4S4hhc7YVL5kLh7VQAaA+q4UpB7qAtRvTeUzcyuEbZ6X 22 | B5pfEVunPJCN20J9kYCPCZPADdvyWRbf9N5NL+bEvHR+LNK0OP87wV3qmCHA+piE 23 | ByqAuxyYZXp1UrErm8gW0ggEn5JEm2Lch0xXpKmqdNviZaHgHolO/LIH3S3PIIpZ 24 | Myt4KGhqc/FeawABy8CmEEk6L1dvizyZc73EX9hcnDSLhp7KP6ECqbImCkpmYGrN 25 | V33KQVdO4zgCcN5X9Rcz6rDe+bm9973uQ1gIM0rLkSOd1HqNcn256qgJ9iLCsGiK 26 | 1vDCGyme8+G5YdrqBWAmpB528fWo9dAxCTkz4cg= 27 | -----END CERTIFICATE----- 28 | -------------------------------------------------------------------------------- /kibana/config/kibana.yml: -------------------------------------------------------------------------------- 1 | server.name: kibana 2 | server.host: "0.0.0.0" 3 | elasticsearch.hosts: [ "https://elasticsearch:9200" ] 4 | elasticsearch.username: kibana 5 | elasticsearch.password: ssanchez00 6 | elasticsearch.ssl.certificateAuthorities: /etc/kibana/keys/certificate.pem 7 | elasticsearch.ssl.verificationMode: certificate 8 | xpack.monitoring.elasticsearch.hosts: [ "https://elasticsearch:9200" ] 9 | xpack.monitoring.enabled: true 10 | xpack.monitoring.elasticsearch.username: elastic 11 | xpack.monitoring.elasticsearch.password: ssanchez00 -------------------------------------------------------------------------------- /logstash/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.elastic.co/logstash/logstash:7.9.2 2 | 3 | RUN mkdir /usr/share/logstash/drivers 4 | COPY ./drivers/* /usr/share/logstash/drivers/ 5 | 6 | # Plugins 7 | RUN logstash-plugin install logstash-integration-jdbc 8 | RUN logstash-plugin install logstash-output-elasticsearch 9 | -------------------------------------------------------------------------------- /logstash/certificate/certificate.pem: -------------------------------------------------------------------------------- 1 | Bag Attributes 2 | friendlyName: ca 3 | localKeyID: 54 69 6D 65 20 31 36 30 32 39 33 30 35 38 34 30 34 38 4 | subject=CN = Elastic Certificate Tool Autogenerated CA 5 | 6 | issuer=CN = Elastic Certificate Tool Autogenerated CA 7 | 8 | -----BEGIN CERTIFICATE----- 9 | MIIDSTCCAjGgAwIBAgIUWKBzaaujWnYRWGCsNp2sGPU1GzMwDQYJKoZIhvcNAQEL 10 | BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l 11 | cmF0ZWQgQ0EwHhcNMjAxMDE3MTAyOTM4WhcNMjMxMDE3MTAyOTM4WjA0MTIwMAYD 12 | VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC 13 | ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAIZZHqX/18v9cF9eJXiSe9Gd 14 | /FwblR7iPT2GIL3q/p4rmvTSatmOsOr2rbrSAj0mEgNuPTAKDHOH/eMQi6xX3jOg 15 | 1VjYl/a379lEAtDFTOFs4+bZgNPOCYrK24XYYhvqleaPb2uuJCp1ZIScYUO+vTf2 16 | WeEEKxD9D2TfxS4LwYc4jzsLVNvgzC+hXR7YDRE+8r6toI6g5SszNNo4WmZ0/DcV 17 | 6lfmJIO5F1rcN13H9DWvsrxBek+GSm0xEesIVCsOvFY832nsEI8em8Bzlayx1nCW 18 | mTA1F2l4xWFOlT78bKMrA5VszKfFvsf7hwjMGyGHPLFCYJW7cbYWiMWgfIgRWZMC 19 | AwEAAaNTMFEwHQYDVR0OBBYEFN2k0tq9a6AZo0YHk09eVckptkaNMB8GA1UdIwQY 20 | MBaAFN2k0tq9a6AZo0YHk09eVckptkaNMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI 21 | hvcNAQELBQADggEBAGg/4S4hhc7YVL5kLh7VQAaA+q4UpB7qAtRvTeUzcyuEbZ6X 22 | B5pfEVunPJCN20J9kYCPCZPADdvyWRbf9N5NL+bEvHR+LNK0OP87wV3qmCHA+piE 23 | ByqAuxyYZXp1UrErm8gW0ggEn5JEm2Lch0xXpKmqdNviZaHgHolO/LIH3S3PIIpZ 24 | Myt4KGhqc/FeawABy8CmEEk6L1dvizyZc73EX9hcnDSLhp7KP6ECqbImCkpmYGrN 25 | V33KQVdO4zgCcN5X9Rcz6rDe+bm9973uQ1gIM0rLkSOd1HqNcn256qgJ9iLCsGiK 26 | 1vDCGyme8+G5YdrqBWAmpB528fWo9dAxCTkz4cg= 27 | -----END CERTIFICATE----- 28 | -------------------------------------------------------------------------------- /logstash/config/logstash.yml: -------------------------------------------------------------------------------- 1 | http.host: "0.0.0.0" 2 | log.level: debug 3 | xpack.monitoring.elasticsearch.hosts: [ "elasticsearch:9200" ] 4 | xpack.monitoring.enabled: true 5 | xpack.monitoring.elasticsearch.username: elastic 6 | xpack.monitoring.elasticsearch.password: ssanchez00 7 | xpack.monitoring.elasticsearch.ssl.certificate_authority: "/etc/logstash/keys/certificate.pem" 8 | xpack.monitoring.elasticsearch.ssl.verification_mode: none 9 | xpack.monitoring.elasticsearch.sniffing: false 10 | xpack.monitoring.collection.interval: 60s -------------------------------------------------------------------------------- /logstash/drivers/gson-2.8.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/logstash/drivers/gson-2.8.6.jar -------------------------------------------------------------------------------- /logstash/drivers/mongo-java-driver-3.12.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/logstash/drivers/mongo-java-driver-3.12.6.jar -------------------------------------------------------------------------------- /logstash/drivers/mongojdbc2.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/logstash/drivers/mongojdbc2.3.jar -------------------------------------------------------------------------------- /logstash/pipeline/ingest_pipeline.conf: -------------------------------------------------------------------------------- 1 | input { 2 | jdbc { 3 | jdbc_driver_library => "/usr/share/logstash/drivers/mongojdbc2.3.jar" 4 | jdbc_driver_class => "com.dbschema.MongoJdbcDriver" 5 | jdbc_connection_string => "jdbc:mongodb://devroot:devroot@mongo:27017/files?authSource=admin" 6 | jdbc_user => "devroot" 7 | jdbc_password => "devroot" 8 | schedule => "*/30 * * * * *" 9 | statement => "db.processed_files.find({ 'document.processed_at' : {'$gte': :sql_last_value}},{'_id': false});" 10 | } 11 | } 12 | 13 | output { 14 | stdout { 15 | codec => rubydebug 16 | } 17 | elasticsearch { 18 | action => "create" 19 | index => "processed_files" 20 | hosts => ["elasticsearch:9200"] 21 | user => "elastic" 22 | password => "ssanchez00" 23 | ssl => true 24 | ssl_certificate_verification => false 25 | cacert => "/etc/logstash/keys/certificate.pem" 26 | } 27 | } 28 | 29 | -------------------------------------------------------------------------------- /microservices/.gitignore: -------------------------------------------------------------------------------- 1 | # Project exclude paths 2 | /callme/target/ 3 | /gateway/target/ -------------------------------------------------------------------------------- /microservices/files_api_gateway/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu 2 | 3 | # Add Maintainer Info 4 | LABEL description="Document Search Engine API Gateway Service" 5 | 6 | RUN apt-get update \ 7 | && apt-get install -y firefox \ 8 | openssh-server \ 9 | xauth \ 10 | && mkdir /var/run/sshd \ 11 | && mkdir /root/.ssh \ 12 | && chmod 700 /root/.ssh \ 13 | && sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config \ 14 | && sed -i "s/^.*X11Forwarding.*$/X11Forwarding yes/" /etc/ssh/sshd_config \ 15 | && sed -i "s/^.*X11UseLocalhost.*$/X11UseLocalhost no/" /etc/ssh/sshd_config \ 16 | && grep "^X11UseLocalhost" /etc/ssh/sshd_config || echo "X11UseLocalhost no" >> /etc/ssh/sshd_config 17 | 18 | RUN echo 'root:ssanchez00' | chpasswd 19 | 20 | ## Install OpenJDK 8 21 | RUN apt-get install -y --no-install-recommends software-properties-common 22 | RUN add-apt-repository -y ppa:openjdk-r/ppa 23 | RUN apt-get update 24 | RUN apt-get install -y openjdk-8-jdk 25 | RUN apt-get install -y openjdk-8-jre 26 | RUN update-alternatives --config java 27 | RUN update-alternatives --config javac 28 | 29 | RUN apt-get install -y netcat 30 | RUN mkdir -p /usr/src/app 31 | WORKDIR /usr/src/app 32 | COPY resources/wait-for-it.sh wait-for-it.sh 33 | COPY resources/entrypoint.sh entrypoint.sh 34 | COPY target/files_api_gateway_service.jar app.jar 35 | 36 | RUN apt-get install -y dos2unix 37 | RUN dos2unix wait-for-it.sh 38 | RUN chmod +x wait-for-it.sh 39 | RUN dos2unix entrypoint.sh 40 | RUN chmod +x entrypoint.sh 41 | RUN uname -a 42 | RUN pwd 43 | RUN ls -al 44 | 45 | 46 | EXPOSE 22 47 | 48 | CMD ["./entrypoint.sh"] -------------------------------------------------------------------------------- /microservices/files_api_gateway/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 4.0.0 7 | com.dreamsoftware 8 | document_search_engine_gateway 9 | 0.0.1-SNAPSHOT 10 | jar 11 | files_api_gateway_service 12 | Document Search Engine API Gateway 13 | 14 | 15 | com.dreamsoftware 16 | document_search_engine 17 | 0.0.1-SNAPSHOT 18 | ../ 19 | 20 | 21 | 22 | 1.4.8 23 | 24 | 25 | 26 | 27 | org.springframework.boot 28 | spring-boot-starter-oauth2-client 29 | 30 | 31 | org.springframework.cloud 32 | spring-cloud-starter-security 33 | 34 | 35 | org.springframework.cloud 36 | spring-cloud-starter-gateway 37 | 38 | 39 | org.springdoc 40 | springdoc-openapi-webflux-core 41 | ${springdoc.openapi.webflux} 42 | 43 | 44 | org.springdoc 45 | springdoc-openapi-webflux-ui 46 | ${springdoc.openapi.webflux} 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | org.springframework.boot 55 | spring-boot-maven-plugin 56 | 57 | files_api_gateway_service 58 | 59 | 60 | 61 | 62 | com.spotify 63 | dockerfile-maven-plugin 64 | 1.4.13 65 | 66 | 67 | build-image 68 | package 69 | 70 | build 71 | 72 | 73 | 74 | 75 | 76 | 77 | ssanchez11/files_api_gateway_service 78 | ${project.version} 79 | 80 | files_api_gateway_service 81 | 82 | false 83 | true 84 | 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /microservices/files_api_gateway/resources/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "starting ssh as root" 4 | service ssh start && service ssh status 5 | 6 | cd /usr/src/app 7 | echo 'waiting for 300 seconds for consul-server:8500 to be accessable before starting application' 8 | ./wait-for-it.sh -t 300 consul:8500 -- java -jar app.jar -------------------------------------------------------------------------------- /microservices/files_api_gateway/resources/wait-for-it.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | TIMEOUT=15 4 | QUIET=0 5 | 6 | echoerr() { 7 | if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi 8 | } 9 | 10 | usage() { 11 | exitcode="$1" 12 | cat << USAGE >&2 13 | Usage: 14 | $cmdname host:port [-t timeout] [-- command args] 15 | -q | --quiet Do not output any status messages 16 | -t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout 17 | -- COMMAND ARGS Execute command with args after the test finishes 18 | USAGE 19 | exit "$exitcode" 20 | } 21 | 22 | wait_for() { 23 | for i in `seq $TIMEOUT` ; do 24 | nc -z "$HOST" "$PORT" > /dev/null 2>&1 25 | 26 | result=$? 27 | if [ $result -eq 0 ] ; then 28 | if [ $# -gt 0 ] ; then 29 | exec "$@" 30 | fi 31 | exit 0 32 | fi 33 | sleep 1 34 | done 35 | echo "Operation timed out" >&2 36 | exit 1 37 | } 38 | 39 | while [ $# -gt 0 ] 40 | do 41 | case "$1" in 42 | *:* ) 43 | HOST=$(printf "%s\n" "$1"| cut -d : -f 1) 44 | PORT=$(printf "%s\n" "$1"| cut -d : -f 2) 45 | shift 1 46 | ;; 47 | -q | --quiet) 48 | QUIET=1 49 | shift 1 50 | ;; 51 | -t) 52 | TIMEOUT="$2" 53 | if [ "$TIMEOUT" = "" ]; then break; fi 54 | shift 2 55 | ;; 56 | --timeout=*) 57 | TIMEOUT="${1#*=}" 58 | shift 1 59 | ;; 60 | --) 61 | shift 62 | break 63 | ;; 64 | --help) 65 | usage 0 66 | ;; 67 | *) 68 | echoerr "Unknown argument: $1" 69 | usage 1 70 | ;; 71 | esac 72 | done 73 | 74 | if [ "$HOST" = "" -o "$PORT" = "" ]; then 75 | echoerr "Error: you need to provide a host and port to test." 76 | usage 2 77 | fi 78 | 79 | wait_for "$@" -------------------------------------------------------------------------------- /microservices/files_api_gateway/src/main/java/com/dreamsoftware/documentsearchengine/FilesApiGatewayApplication.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.boot.SpringApplication; 6 | import org.springframework.boot.autoconfigure.SpringBootApplication; 7 | import org.springframework.security.oauth2.client.OAuth2AuthorizedClient; 8 | import org.springframework.security.oauth2.client.annotation.RegisteredOAuth2AuthorizedClient; 9 | import org.springframework.web.bind.annotation.GetMapping; 10 | import org.springframework.web.bind.annotation.RestController; 11 | import org.springframework.web.server.WebSession; 12 | import reactor.core.publisher.Mono; 13 | 14 | @SpringBootApplication 15 | @RestController 16 | public class FilesApiGatewayApplication { 17 | 18 | private static final Logger LOGGER = LoggerFactory.getLogger(FilesApiGatewayApplication.class); 19 | 20 | public static void main(String[] args) { 21 | SpringApplication.run(FilesApiGatewayApplication.class, args); 22 | } 23 | 24 | @GetMapping(value = "/token") 25 | public Mono getHome(@RegisteredOAuth2AuthorizedClient OAuth2AuthorizedClient authorizedClient) { 26 | return Mono.just(authorizedClient.getAccessToken().getTokenValue()); 27 | } 28 | 29 | @GetMapping("/") 30 | public Mono index(WebSession session) { 31 | return Mono.just(session.getId()); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /microservices/files_api_gateway/src/main/java/com/dreamsoftware/documentsearchengine/security/gateway/config/SecurityConfig.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.security.gateway.config; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.security.config.annotation.web.reactive.EnableWebFluxSecurity; 6 | import org.springframework.security.config.web.server.ServerHttpSecurity; 7 | import org.springframework.security.web.server.SecurityWebFilterChain; 8 | import static org.springframework.security.config.Customizer.withDefaults; 9 | 10 | @Configuration 11 | @EnableWebFluxSecurity 12 | public class SecurityConfig { 13 | 14 | @Bean 15 | public SecurityWebFilterChain provideSpringSecurityFilterChain(ServerHttpSecurity http) { 16 | http.authorizeExchange(exchanges -> exchanges.anyExchange().authenticated()) 17 | .oauth2Login(withDefaults()); 18 | http.csrf().disable(); 19 | return http.build(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /microservices/files_api_gateway/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | application: 3 | name: files-api-gateway-service 4 | security: 5 | oauth2: 6 | client: 7 | provider: 8 | keycloak: 9 | token-uri: http://keycloak:8080/auth/realms/document_search_engine/protocol/openid-connect/token 10 | authorization-uri: http://keycloak:8080/auth/realms/document_search_engine/protocol/openid-connect/auth 11 | userinfo-uri: http://keycloak:8080/auth/realms/document_search_engine/protocol/openid-connect/userinfo 12 | user-name-attribute: preferred_username 13 | registration: 14 | files-management-client: 15 | provider: keycloak 16 | client-id: files-management-client 17 | client-secret: 23c6be5d-0aed-421b-a245-9f13ee5412c9 18 | authorization-grant-type: authorization_code 19 | redirect-uri: "{baseUrl}/login/oauth2/code/keycloak" 20 | files-viewing-client: 21 | provider: keycloak 22 | client-id: files-viewing-client 23 | client-secret: 8d238a8c-6f0c-4935-b394-da4b4901aec1 24 | authorization-grant-type: authorization_code 25 | redirect-uri: "{baseUrl}/login/oauth2/code/keycloak" 26 | cloud: 27 | consul: 28 | enabled: true 29 | host: consul-server 30 | port: 8500 31 | discovery: 32 | register: false 33 | registerHealthCheck: false 34 | gateway: 35 | discovery: 36 | locator: 37 | enabled: true 38 | default-filters: 39 | - TokenRelay 40 | routes: 41 | - id: files-metadata-service 42 | uri: lb://files-metadata-service 43 | predicates: 44 | - Path=/metadata/** 45 | filters: 46 | - RemoveRequestHeader=Cookie 47 | - RewritePath=/metadata/(?.*), /api/v1/metadata/$\{path} 48 | - id: files-metadata-service-swagger 49 | uri: lb://files-metadata-service 50 | predicates: 51 | - Path=/swagger-metadata-ui.html 52 | filters: 53 | - RemoveRequestHeader=Cookie 54 | - RewritePath=/swagger-metadata-ui.html, /swagger-ui.html 55 | - id: files-search-service 56 | uri: lb://files-search-service 57 | predicates: 58 | - Path=/search/** 59 | filters: 60 | - RemoveRequestHeader=Cookie 61 | - RewritePath=/search/(?.*), /api/v1/search/$\{path} 62 | - id: files-search-service-swagger 63 | uri: lb://files-search-service 64 | predicates: 65 | - Path=/swagger-search-ui.html 66 | filters: 67 | - RemoveRequestHeader=Cookie 68 | - RewritePath=/swagger-search-ui.html, /swagger-ui.html 69 | - id: files-management-service 70 | uri: lb://files-management-service 71 | predicates: 72 | - Path=/management/** 73 | filters: 74 | - RemoveRequestHeader=Cookie 75 | - RewritePath=/management/(?.*), /api/v1/management/$\{path} 76 | - id: files-management-service-swagger 77 | uri: lb://files-management-service 78 | predicates: 79 | - Path=/swagger-management-ui.html 80 | filters: 81 | - RemoveRequestHeader=Cookie 82 | - RewritePath=/swagger-management-ui.html, /swagger-ui.html 83 | - id: files-notifications-service 84 | uri: lb://files-notifications-service 85 | predicates: 86 | - Path=/notifications/** 87 | filters: 88 | - RemoveRequestHeader=Cookie 89 | - RewritePath=/notifications/(?.*), /api/v1/notifications/$\{path} 90 | - id: files-notifications-service-swagger 91 | uri: lb://files-notifications-service 92 | predicates: 93 | - Path=/swagger-management-ui.html 94 | filters: 95 | - RemoveRequestHeader=Cookie 96 | - RewritePath=/swagger-management-ui.html, /swagger-ui.html 97 | - id: openapi 98 | uri: http://localhost:${server.port} 99 | predicates: 100 | - Path=/v3/api-docs/** 101 | filters: 102 | - RewritePath=/v3/api-docs/(?.*), /$\{path}/v3/api-docs 103 | 104 | 105 | server.port: 8080 106 | 107 | logging.level: 108 | org.springframework.cloud.gateway: DEBUG 109 | org.springframework.security: DEBUG 110 | org.springframework.web.reactive.function.client: DEBUG -------------------------------------------------------------------------------- /microservices/files_commons/.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | target/ 3 | !.mvn/wrapper/maven-wrapper.jar 4 | !**/src/main/**/target/ 5 | !**/src/test/**/target/ 6 | 7 | ### STS ### 8 | .apt_generated 9 | .classpath 10 | .factorypath 11 | .project 12 | .settings 13 | .springBeans 14 | .sts4-cache 15 | 16 | ### IntelliJ IDEA ### 17 | .idea 18 | *.iws 19 | *.iml 20 | *.ipr 21 | 22 | ### NetBeans ### 23 | /nbproject/private/ 24 | /nbbuild/ 25 | /dist/ 26 | /nbdist/ 27 | /.nb-gradle/ 28 | build/ 29 | !**/src/main/**/build/ 30 | !**/src/test/**/build/ 31 | 32 | ### VS Code ### 33 | .vscode/ 34 | -------------------------------------------------------------------------------- /microservices/files_commons/.mvn/wrapper/MavenWrapperDownloader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2007-present the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | import java.net.*; 17 | import java.io.*; 18 | import java.nio.channels.*; 19 | import java.util.Properties; 20 | 21 | public class MavenWrapperDownloader { 22 | 23 | private static final String WRAPPER_VERSION = "0.5.6"; 24 | /** 25 | * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. 26 | */ 27 | private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/" 28 | + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar"; 29 | 30 | /** 31 | * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to 32 | * use instead of the default one. 33 | */ 34 | private static final String MAVEN_WRAPPER_PROPERTIES_PATH = 35 | ".mvn/wrapper/maven-wrapper.properties"; 36 | 37 | /** 38 | * Path where the maven-wrapper.jar will be saved to. 39 | */ 40 | private static final String MAVEN_WRAPPER_JAR_PATH = 41 | ".mvn/wrapper/maven-wrapper.jar"; 42 | 43 | /** 44 | * Name of the property which should be used to override the default download url for the wrapper. 45 | */ 46 | private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; 47 | 48 | public static void main(String args[]) { 49 | System.out.println("- Downloader started"); 50 | File baseDirectory = new File(args[0]); 51 | System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); 52 | 53 | // If the maven-wrapper.properties exists, read it and check if it contains a custom 54 | // wrapperUrl parameter. 55 | File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); 56 | String url = DEFAULT_DOWNLOAD_URL; 57 | if(mavenWrapperPropertyFile.exists()) { 58 | FileInputStream mavenWrapperPropertyFileInputStream = null; 59 | try { 60 | mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); 61 | Properties mavenWrapperProperties = new Properties(); 62 | mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); 63 | url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); 64 | } catch (IOException e) { 65 | System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); 66 | } finally { 67 | try { 68 | if(mavenWrapperPropertyFileInputStream != null) { 69 | mavenWrapperPropertyFileInputStream.close(); 70 | } 71 | } catch (IOException e) { 72 | // Ignore ... 73 | } 74 | } 75 | } 76 | System.out.println("- Downloading from: " + url); 77 | 78 | File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); 79 | if(!outputFile.getParentFile().exists()) { 80 | if(!outputFile.getParentFile().mkdirs()) { 81 | System.out.println( 82 | "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'"); 83 | } 84 | } 85 | System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); 86 | try { 87 | downloadFileFromURL(url, outputFile); 88 | System.out.println("Done"); 89 | System.exit(0); 90 | } catch (Throwable e) { 91 | System.out.println("- Error downloading"); 92 | e.printStackTrace(); 93 | System.exit(1); 94 | } 95 | } 96 | 97 | private static void downloadFileFromURL(String urlString, File destination) throws Exception { 98 | if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) { 99 | String username = System.getenv("MVNW_USERNAME"); 100 | char[] password = System.getenv("MVNW_PASSWORD").toCharArray(); 101 | Authenticator.setDefault(new Authenticator() { 102 | @Override 103 | protected PasswordAuthentication getPasswordAuthentication() { 104 | return new PasswordAuthentication(username, password); 105 | } 106 | }); 107 | } 108 | URL website = new URL(urlString); 109 | ReadableByteChannel rbc; 110 | rbc = Channels.newChannel(website.openStream()); 111 | FileOutputStream fos = new FileOutputStream(destination); 112 | fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); 113 | fos.close(); 114 | rbc.close(); 115 | } 116 | 117 | } 118 | -------------------------------------------------------------------------------- /microservices/files_commons/.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/microservices/files_commons/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /microservices/files_commons/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip 2 | wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar 3 | -------------------------------------------------------------------------------- /microservices/files_commons/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | com.dreamsoftware 6 | document_search_engine_files_commons 7 | 0.0.1-SNAPSHOT 8 | jar 9 | file_commons 10 | Document Search Engine Files Commons 11 | 12 | 13 | com.dreamsoftware 14 | document_search_engine 15 | 0.0.1-SNAPSHOT 16 | ../ 17 | 18 | 19 | 20 | 1.3.9 21 | 22 | 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-starter-web 27 | 28 | 29 | org.springdoc 30 | springdoc-openapi-ui 31 | ${springdoc.open.api} 32 | 33 | 34 | org.springframework.cloud 35 | spring-cloud-starter-openfeign 36 | 37 | 38 | org.springframework.boot 39 | spring-boot-starter-security 40 | 41 | 42 | org.springframework.security 43 | spring-security-oauth2-resource-server 44 | 45 | 46 | org.springframework.security 47 | spring-security-oauth2-jose 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /microservices/files_commons/src/main/java/com/dreamsoftware/documentsearchengine/config/JacksonConfig.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.config; 2 | 3 | import com.fasterxml.jackson.annotation.JsonInclude.Include; 4 | import com.fasterxml.jackson.core.JsonGenerator; 5 | import com.fasterxml.jackson.databind.DeserializationFeature; 6 | import com.fasterxml.jackson.databind.ObjectMapper; 7 | import com.fasterxml.jackson.databind.SerializationFeature; 8 | import com.fasterxml.jackson.databind.node.JsonNodeFactory; 9 | import org.springframework.context.annotation.Bean; 10 | import org.springframework.context.annotation.Configuration; 11 | 12 | /** 13 | * 14 | * @author ssanchez 15 | */ 16 | @Configuration 17 | public class JacksonConfig { 18 | 19 | @Bean 20 | public ObjectMapper provideObjectMapper() { 21 | return new ObjectMapper() 22 | .setDefaultPropertyInclusion(Include.NON_NULL) 23 | .setNodeFactory(JsonNodeFactory.withExactBigDecimals(true)) 24 | .enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS) 25 | .enable(JsonGenerator.Feature.WRITE_BIGDECIMAL_AS_PLAIN) 26 | .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) 27 | .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /microservices/files_commons/src/main/java/com/dreamsoftware/documentsearchengine/config/SecurityConfig.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.config; 2 | 3 | import org.springframework.context.annotation.Configuration; 4 | import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity; 5 | import org.springframework.security.config.annotation.web.builders.HttpSecurity; 6 | import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; 7 | import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; 8 | import org.springframework.security.config.annotation.web.configurers.oauth2.server.resource.OAuth2ResourceServerConfigurer; 9 | 10 | /** 11 | * 12 | * @author ssanchez 13 | */ 14 | @Configuration 15 | @EnableWebSecurity 16 | @EnableGlobalMethodSecurity(prePostEnabled = true) 17 | public class SecurityConfig extends WebSecurityConfigurerAdapter { 18 | 19 | private static final String[] AUTH_WHITELIST_SWAGGER_V2 = { 20 | // -- swagger ui 21 | "/v2/api-docs", 22 | "/swagger-resources/**", 23 | "/configuration/ui", 24 | "/configuration/security", 25 | "/swagger-ui.html", 26 | "/webjars/**" 27 | }; 28 | 29 | private static final String[] AUTH_WHITELIST_SWAGGER_V3 = { 30 | // -- swagger ui 31 | "/v2/api-docs", 32 | "/v2/api-docs/**", 33 | "/v3/api-docs", 34 | "/v3/api-docs/**", 35 | "/swagger-resources/**", 36 | "/swagger-ui/**",}; 37 | 38 | @Override 39 | protected void configure(HttpSecurity http) throws Exception { 40 | http.authorizeRequests(authorize 41 | -> authorize.antMatchers("/actuator/**") 42 | .permitAll() 43 | .antMatchers(AUTH_WHITELIST_SWAGGER_V2).permitAll() 44 | .antMatchers(AUTH_WHITELIST_SWAGGER_V3).permitAll() 45 | .anyRequest() 46 | .authenticated() 47 | ) 48 | .oauth2ResourceServer(OAuth2ResourceServerConfigurer::jwt); 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /microservices/files_commons/src/main/java/com/dreamsoftware/documentsearchengine/web/core/APIResponse.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.core; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Builder; 6 | import lombok.Data; 7 | import lombok.NoArgsConstructor; 8 | import org.springframework.http.HttpStatus; 9 | 10 | @Data 11 | @Builder 12 | @AllArgsConstructor 13 | @NoArgsConstructor 14 | public class APIResponse { 15 | 16 | /** 17 | * Category 18 | */ 19 | @JsonProperty("category") 20 | private String category; 21 | 22 | /** 23 | * Code 24 | */ 25 | @JsonProperty("code") 26 | private Long code; 27 | 28 | /** 29 | * Response code name 30 | */ 31 | @JsonProperty("code_name") 32 | private String codeName; 33 | 34 | /** 35 | * Response Status 36 | */ 37 | @JsonProperty("status") 38 | private ResponseStatusEnum status; 39 | 40 | /** 41 | * Response Http Status 42 | */ 43 | @JsonProperty("http_status") 44 | private HttpStatus httpStatusCode; 45 | 46 | /** 47 | * Response Info Url 48 | */ 49 | @JsonProperty("info_url") 50 | private String infoUrl; 51 | 52 | /** 53 | * Response Data 54 | */ 55 | @JsonProperty("data") 56 | private T data; 57 | 58 | } 59 | -------------------------------------------------------------------------------- /microservices/files_commons/src/main/java/com/dreamsoftware/documentsearchengine/web/core/ErrorResponseDTO.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.core; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import java.util.List; 5 | import lombok.AllArgsConstructor; 6 | import lombok.Builder; 7 | import lombok.Data; 8 | import lombok.NoArgsConstructor; 9 | 10 | /** 11 | * 12 | * @author ssanchez 13 | */ 14 | @Data 15 | @Builder 16 | @AllArgsConstructor 17 | @NoArgsConstructor 18 | public class ErrorResponseDTO { 19 | 20 | /** 21 | * Field Errors 22 | */ 23 | @JsonProperty("errors") 24 | private List fieldErrors; 25 | 26 | /** 27 | * Add Field Error 28 | * 29 | * @param path 30 | * @param message 31 | */ 32 | public void addFieldError(String path, String message) { 33 | fieldErrors.add(new FieldErrorDTO(path, message)); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /microservices/files_commons/src/main/java/com/dreamsoftware/documentsearchengine/web/core/FieldErrorDTO.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.core; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Builder; 6 | import lombok.Data; 7 | import lombok.NoArgsConstructor; 8 | 9 | /** 10 | * 11 | * @author ssanchez 12 | */ 13 | @Data 14 | @Builder 15 | @AllArgsConstructor 16 | @NoArgsConstructor 17 | public class FieldErrorDTO { 18 | 19 | /** 20 | * Field Name 21 | */ 22 | @JsonProperty("field") 23 | private String field; 24 | 25 | /** 26 | * Field Message Error 27 | */ 28 | @JsonProperty("message") 29 | private String message; 30 | 31 | } 32 | -------------------------------------------------------------------------------- /microservices/files_commons/src/main/java/com/dreamsoftware/documentsearchengine/web/core/IResponseCodeTypes.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.core; 2 | 3 | /** 4 | * 5 | * @author ssanchez 6 | */ 7 | public interface IResponseCodeTypes { 8 | 9 | /** 10 | * Response Code 11 | * 12 | * @return 13 | */ 14 | Long getResponseCode(); 15 | 16 | /** 17 | * Get Category Name 18 | * 19 | * @return 20 | */ 21 | String getCategoryName(); 22 | 23 | /** 24 | * Get Code Name 25 | * 26 | * @return 27 | */ 28 | String getCodeName(); 29 | 30 | } 31 | -------------------------------------------------------------------------------- /microservices/files_commons/src/main/java/com/dreamsoftware/documentsearchengine/web/core/IVersionable.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.core; 2 | 3 | /** 4 | * 5 | * @author ssanchez 6 | */ 7 | public interface IVersionable { 8 | 9 | /** 10 | * 11 | * @return 12 | */ 13 | String getVersion(); 14 | } 15 | -------------------------------------------------------------------------------- /microservices/files_commons/src/main/java/com/dreamsoftware/documentsearchengine/web/core/ResponseStatusEnum.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.core; 2 | 3 | /** 4 | * 5 | * @author ssanchez 6 | */ 7 | public enum ResponseStatusEnum { 8 | SUCCESS, ERROR; 9 | } 10 | -------------------------------------------------------------------------------- /microservices/files_commons/src/main/java/com/dreamsoftware/documentsearchengine/web/core/SupportController.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.core; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | 5 | /** 6 | * 7 | * @author ssanchez 8 | */ 9 | public class SupportController { 10 | 11 | /** 12 | * Response Helper 13 | */ 14 | @Autowired 15 | protected ResponseHelper responseHelper; 16 | 17 | } 18 | -------------------------------------------------------------------------------- /microservices/files_commons/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /microservices/files_management/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-jdk-alpine 2 | # Add Maintainer Info 3 | LABEL description="Document Search Engine Files Management Service" 4 | # Args for image 5 | ARG PORT=8080 6 | 7 | RUN apk update && apk upgrade 8 | RUN ln -s /bin/bash /usr/bin 9 | RUN mkdir -p /usr/src/app 10 | WORKDIR /usr/src/app 11 | 12 | 13 | COPY resources/wait-for-it.sh wait-for-it.sh 14 | COPY target/file_management_service.jar app.jar 15 | 16 | RUN dos2unix wait-for-it.sh 17 | RUN chmod +x wait-for-it.sh 18 | RUN uname -a 19 | RUN pwd 20 | RUN ls -al 21 | 22 | EXPOSE ${PORT} 23 | 24 | CMD ["sh", "-c", "echo 'waiting for 300 seconds for consul-server:8500 to be accessable before starting application' && ./wait-for-it.sh -t 300 consul:8500 -- java -jar app.jar"] -------------------------------------------------------------------------------- /microservices/files_management/lombok.config: -------------------------------------------------------------------------------- 1 | lombok.copyableAnnotations += org.springframework.beans.factory.annotation.Qualifier 2 | -------------------------------------------------------------------------------- /microservices/files_management/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 4.0.0 7 | com.dreamsoftware 8 | document_search_engine_files_management 9 | 0.0.1-SNAPSHOT 10 | jar 11 | file_management_service 12 | Document Search Engine Files Management 13 | 14 | 15 | com.dreamsoftware 16 | document_search_engine 17 | 0.0.1-SNAPSHOT 18 | ../ 19 | 20 | 21 | 22 | 23 | ${project.groupId} 24 | document_search_engine_files_commons 25 | ${project.version} 26 | 27 | 28 | 29 | com.hierynomus 30 | sshj 31 | 0.27.0 32 | 33 | 34 | 35 | org.springframework.cloud 36 | spring-cloud-starter-stream-kafka 37 | 38 | 39 | 40 | org.springframework.cloud 41 | spring-cloud-starter-oauth2 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | org.springframework.boot 50 | spring-boot-maven-plugin 51 | 52 | file_management_service 53 | 54 | 55 | 56 | 57 | com.spotify 58 | dockerfile-maven-plugin 59 | 1.4.13 60 | 61 | 62 | build-image 63 | package 64 | 65 | build 66 | 67 | 68 | 69 | 70 | 71 | 72 | ssanchez11/file_management_service 73 | ${project.version} 74 | 75 | file_management_service 76 | 77 | false 78 | true 79 | 80 | 81 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /microservices/files_management/resources/wait-for-it.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | TIMEOUT=15 4 | QUIET=0 5 | 6 | echoerr() { 7 | if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi 8 | } 9 | 10 | usage() { 11 | exitcode="$1" 12 | cat << USAGE >&2 13 | Usage: 14 | $cmdname host:port [-t timeout] [-- command args] 15 | -q | --quiet Do not output any status messages 16 | -t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout 17 | -- COMMAND ARGS Execute command with args after the test finishes 18 | USAGE 19 | exit "$exitcode" 20 | } 21 | 22 | wait_for() { 23 | for i in `seq $TIMEOUT` ; do 24 | nc -z "$HOST" "$PORT" > /dev/null 2>&1 25 | 26 | result=$? 27 | if [ $result -eq 0 ] ; then 28 | if [ $# -gt 0 ] ; then 29 | exec "$@" 30 | fi 31 | exit 0 32 | fi 33 | sleep 1 34 | done 35 | echo "Operation timed out" >&2 36 | exit 1 37 | } 38 | 39 | while [ $# -gt 0 ] 40 | do 41 | case "$1" in 42 | *:* ) 43 | HOST=$(printf "%s\n" "$1"| cut -d : -f 1) 44 | PORT=$(printf "%s\n" "$1"| cut -d : -f 2) 45 | shift 1 46 | ;; 47 | -q | --quiet) 48 | QUIET=1 49 | shift 1 50 | ;; 51 | -t) 52 | TIMEOUT="$2" 53 | if [ "$TIMEOUT" = "" ]; then break; fi 54 | shift 2 55 | ;; 56 | --timeout=*) 57 | TIMEOUT="${1#*=}" 58 | shift 1 59 | ;; 60 | --) 61 | shift 62 | break 63 | ;; 64 | --help) 65 | usage 0 66 | ;; 67 | *) 68 | echoerr "Unknown argument: $1" 69 | usage 1 70 | ;; 71 | esac 72 | done 73 | 74 | if [ "$HOST" = "" -o "$PORT" = "" ]; then 75 | echoerr "Error: you need to provide a host and port to test." 76 | usage 2 77 | fi 78 | 79 | wait_for "$@" -------------------------------------------------------------------------------- /microservices/files_management/src/main/java/com/dreamsoftware/documentsearchengine/FilesManagementServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine; 2 | 3 | import com.dreamsoftware.documentsearchengine.config.AppStreamsConfig; 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | import org.springframework.cloud.openfeign.EnableFeignClients; 7 | import org.springframework.cloud.stream.annotation.EnableBinding; 8 | 9 | @SpringBootApplication 10 | @EnableFeignClients 11 | @EnableBinding(AppStreamsConfig.class) 12 | public class FilesManagementServiceApplication { 13 | 14 | public static void main(String[] args) { 15 | SpringApplication.run(FilesManagementServiceApplication.class, args); 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /microservices/files_management/src/main/java/com/dreamsoftware/documentsearchengine/client/IFilesMetadataClient.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.client; 2 | 3 | import com.dreamsoftware.documentsearchengine.web.core.APIResponse; 4 | import com.dreamsoftware.documentsearchengine.web.dto.ProcessedFileDTO; 5 | import org.springframework.cloud.openfeign.FeignClient; 6 | import org.springframework.web.bind.annotation.GetMapping; 7 | import org.springframework.web.bind.annotation.PathVariable; 8 | 9 | /** 10 | * 11 | * @author ssanchez 12 | */ 13 | @FeignClient(name = "files-metadata-service") 14 | public interface IFilesMetadataClient { 15 | 16 | /** 17 | * Get Processed File By Name 18 | * 19 | * @param name 20 | * @return 21 | */ 22 | @GetMapping(path = "/api/v1/metadata/name/{name}", consumes = "application/json") 23 | APIResponse getProcessedFileByName(@PathVariable("name") String name); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /microservices/files_management/src/main/java/com/dreamsoftware/documentsearchengine/client/interceptor/FeignClientInterceptor.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.client.interceptor; 2 | 3 | import feign.RequestInterceptor; 4 | import feign.RequestTemplate; 5 | import lombok.extern.slf4j.Slf4j; 6 | import org.springframework.security.core.Authentication; 7 | import org.springframework.security.core.context.SecurityContextHolder; 8 | import org.springframework.security.oauth2.provider.authentication.OAuth2AuthenticationDetails; 9 | import org.springframework.stereotype.Component; 10 | import org.springframework.security.web.authentication.WebAuthenticationDetails; 11 | import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationToken; 12 | 13 | /** 14 | * Feign Client Interceptor 15 | * 16 | * @author ssanchez 17 | */ 18 | @Component 19 | @Slf4j 20 | public class FeignClientInterceptor implements RequestInterceptor { 21 | 22 | private static final String AUTHORIZATION_HEADER = "Authorization"; 23 | private static final String TOKEN_TYPE = "Bearer"; 24 | 25 | @Override 26 | public void apply(RequestTemplate requestTemplate) { 27 | log.debug("FeignClientInterceptor -> apply CALLED"); 28 | final Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); 29 | if (authentication != null && authentication instanceof JwtAuthenticationToken) { 30 | final JwtAuthenticationToken jwtAuthToken = (JwtAuthenticationToken) authentication; 31 | requestTemplate.header(AUTHORIZATION_HEADER, String.format("%s %s", TOKEN_TYPE, jwtAuthToken.getToken().getTokenValue())); 32 | } 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /microservices/files_management/src/main/java/com/dreamsoftware/documentsearchengine/config/AppStreamsConfig.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.config; 2 | 3 | import org.springframework.cloud.stream.annotation.Output; 4 | import org.springframework.messaging.MessageChannel; 5 | 6 | /** 7 | * 8 | * @author ssanchez 9 | */ 10 | public interface AppStreamsConfig { 11 | 12 | String FILES_PROCESSED_OPERATIONS_CHANNEL = "files-processed-operations"; 13 | 14 | /** 15 | * 16 | * @return 17 | */ 18 | @Output(FILES_PROCESSED_OPERATIONS_CHANNEL) 19 | MessageChannel outboundFilesProcessedOperations(); 20 | } 21 | -------------------------------------------------------------------------------- /microservices/files_management/src/main/java/com/dreamsoftware/documentsearchengine/config/OpenApiConfig.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.config; 2 | 3 | import io.swagger.v3.oas.models.Components; 4 | import io.swagger.v3.oas.models.ExternalDocumentation; 5 | import io.swagger.v3.oas.models.OpenAPI; 6 | import io.swagger.v3.oas.models.info.Info; 7 | import io.swagger.v3.oas.models.info.License; 8 | import io.swagger.v3.oas.models.security.SecurityRequirement; 9 | import io.swagger.v3.oas.models.security.SecurityScheme; 10 | import java.util.Arrays; 11 | import org.springframework.context.annotation.Bean; 12 | import org.springframework.context.annotation.Configuration; 13 | 14 | /** 15 | * 16 | * @author ssanchez 17 | */ 18 | @Configuration 19 | public class OpenApiConfig { 20 | 21 | @Bean 22 | public OpenAPI springShopOpenAPI() { 23 | return new OpenAPI() 24 | .components(new Components().addSecuritySchemes("bearer-jwt", 25 | new SecurityScheme() 26 | .description("Auth Header with Json Web Token format") 27 | .type(SecurityScheme.Type.HTTP) 28 | .scheme("bearer") 29 | .bearerFormat("JWT") 30 | .in(SecurityScheme.In.HEADER).name("Authorization"))) 31 | .addSecurityItem( 32 | new SecurityRequirement().addList("bearer-jwt", Arrays.asList("read", "write"))) 33 | .info(new Info().title("Files Management REST API") 34 | .description("Provides a hypermedia-driven REST API for Apps") 35 | .version("v0.0.1-SNAPSHOT") 36 | .license(new License().name("Apache 2.0").url("http://springdoc.org"))) 37 | .externalDocs(new ExternalDocumentation() 38 | .description("Provides a hypermedia-driven REST API for Apps") 39 | .url("https://springshop.wiki.github.org/docs") 40 | ); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /microservices/files_management/src/main/java/com/dreamsoftware/documentsearchengine/config/props/SFTPProperties.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.config.props; 2 | 3 | import lombok.Data; 4 | import org.springframework.beans.factory.annotation.Value; 5 | import org.springframework.stereotype.Component; 6 | 7 | /** 8 | * 9 | * @author ssanchez 10 | */ 11 | @Data 12 | @Component 13 | public class SFTPProperties { 14 | 15 | /** 16 | * HostName of the server 17 | */ 18 | @Value("${sftp.hostname}") 19 | private String hostName; 20 | 21 | /** 22 | * Port of the server 23 | */ 24 | @Value("${sftp.port}") 25 | private Integer port; 26 | 27 | /** 28 | * UserName to login 29 | */ 30 | @Value("${sftp.username}") 31 | private String username; 32 | 33 | /** 34 | * Password to login 35 | */ 36 | @Value("${sftp.password}") 37 | private String password; 38 | 39 | /** 40 | * Remote Folder 41 | */ 42 | @Value("${sftp.remoteFolder}") 43 | private String remoteFolder; 44 | 45 | /** 46 | * Get Connection String 47 | * 48 | * @return 49 | */ 50 | public String getConnectionString() { 51 | return "sftp://" + username + ":" + password + "@" + hostName + "/" + remoteFolder + "/"; 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /microservices/files_management/src/main/java/com/dreamsoftware/documentsearchengine/config/props/UploadProperties.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.config.props; 2 | 3 | import lombok.Data; 4 | import org.springframework.beans.factory.annotation.Value; 5 | import org.springframework.stereotype.Component; 6 | 7 | /** 8 | * 9 | * @author ssanchez 10 | */ 11 | @Component 12 | @Data 13 | public class UploadProperties { 14 | 15 | @Value("${uploads.directory}") 16 | private String uploadsDirectory; 17 | 18 | } 19 | -------------------------------------------------------------------------------- /microservices/files_management/src/main/java/com/dreamsoftware/documentsearchengine/service/IFilesManagementService.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.service; 2 | 3 | import org.springframework.web.multipart.MultipartFile; 4 | 5 | /** 6 | * 7 | * @author ssanchez 8 | */ 9 | public interface IFilesManagementService { 10 | 11 | /** 12 | * Save File 13 | * 14 | * @param uploadFile 15 | */ 16 | void save(final MultipartFile uploadFile); 17 | 18 | /** 19 | * Delete 20 | * 21 | * @param name 22 | */ 23 | void delete(final String name); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /microservices/files_management/src/main/java/com/dreamsoftware/documentsearchengine/web/controller/FilesManagementController.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.controller; 2 | 3 | import com.dreamsoftware.documentsearchengine.service.IFilesManagementService; 4 | import com.dreamsoftware.documentsearchengine.web.controller.error.exception.DeleteFileException; 5 | import com.dreamsoftware.documentsearchengine.web.controller.error.exception.SaveFileException; 6 | import com.dreamsoftware.documentsearchengine.web.core.APIResponse; 7 | import com.dreamsoftware.documentsearchengine.web.core.SupportController; 8 | import io.swagger.v3.oas.annotations.Operation; 9 | import io.swagger.v3.oas.annotations.Parameter; 10 | import io.swagger.v3.oas.annotations.tags.Tag; 11 | import javax.validation.Valid; 12 | import lombok.RequiredArgsConstructor; 13 | import org.springframework.http.HttpStatus; 14 | import org.springframework.http.MediaType; 15 | import org.springframework.http.ResponseEntity; 16 | import org.springframework.security.access.prepost.PreAuthorize; 17 | import org.springframework.validation.annotation.Validated; 18 | import org.springframework.web.bind.annotation.PathVariable; 19 | import org.springframework.web.bind.annotation.RequestMapping; 20 | import org.springframework.web.bind.annotation.RequestMethod; 21 | import org.springframework.web.bind.annotation.RequestPart; 22 | import org.springframework.web.bind.annotation.RestController; 23 | import org.springframework.web.multipart.MultipartFile; 24 | 25 | /** 26 | * Files Management Controller 27 | * 28 | * @author ssanchez 29 | */ 30 | @RestController 31 | @Validated 32 | @RequestMapping("/api/v1/management/") 33 | @Tag(name = "files_management", description = "/api/v1/management/ (Code Response interval -> 3XX)") 34 | @RequiredArgsConstructor 35 | public class FilesManagementController extends SupportController { 36 | 37 | /** 38 | * Files Management Service 39 | */ 40 | private final IFilesManagementService filesManagementService; 41 | 42 | /** 43 | * Save File 44 | * 45 | * @param uploadFile 46 | * @return 47 | * @throws Throwable 48 | */ 49 | @PreAuthorize("hasAuthority('SCOPE_FILES_MANAGEMENT')") 50 | @Operation(summary = "SAVE_FILE", description = "Save a new File") 51 | @RequestMapping(value = "/", method = RequestMethod.POST, 52 | consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) 53 | public ResponseEntity> saveFile( 54 | @Valid @RequestPart(name = "file", required = true) MultipartFile uploadFile) throws Throwable { 55 | 56 | try { 57 | 58 | // Save File 59 | filesManagementService.save(uploadFile); 60 | 61 | // Create and Send Response 62 | return responseHelper.createAndSendResponse( 63 | FilesManagementResponseCodeEnum.FILE_SAVED_SUCCESSFULLY, HttpStatus.OK, 64 | "File Saved Successfully"); 65 | 66 | } catch (final Exception ex) { 67 | throw new SaveFileException(ex); 68 | } 69 | } 70 | 71 | /** 72 | * Delete By Name 73 | * 74 | * @param name 75 | * @return 76 | */ 77 | @PreAuthorize("hasAuthority('SCOPE_FILES_MANAGEMENT')") 78 | @Operation(summary = "DELETE_FILE", description = "Delete File") 79 | @RequestMapping(value = "/{name}", method = RequestMethod.DELETE) 80 | public ResponseEntity> deleteByName( 81 | @Parameter(name = "name", description = "Processed File Name", required = true) 82 | @PathVariable final String name) { 83 | 84 | try { 85 | 86 | filesManagementService.delete(name); 87 | 88 | return responseHelper.createAndSendResponse( 89 | FilesManagementResponseCodeEnum.FILE_DELETED_SUCCESSFULLY, HttpStatus.OK, 90 | "File delete successfully"); 91 | 92 | } catch (final Exception ex) { 93 | throw new DeleteFileException(ex); 94 | } 95 | 96 | } 97 | 98 | } 99 | -------------------------------------------------------------------------------- /microservices/files_management/src/main/java/com/dreamsoftware/documentsearchengine/web/controller/FilesManagementResponseCodeEnum.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.controller; 2 | 3 | import com.dreamsoftware.documentsearchengine.web.core.IResponseCodeTypes; 4 | 5 | /** 6 | * 7 | * @author ssanchez 8 | */ 9 | public enum FilesManagementResponseCodeEnum implements IResponseCodeTypes { 10 | 11 | FILE_SAVED_SUCCESSFULLY(300L), 12 | SAVE_FILE_ERROR(301L), 13 | VALIDATION_ERROR(302L), 14 | FILE_ALREADY_PROCESSED(303L), 15 | DELETE_FILE_ERROR(304L), 16 | FILE_DELETED_SUCCESSFULLY(305L); 17 | 18 | private final Long code; 19 | 20 | public static final String CATEGORY_NAME = "FILES_MANAGEMENT"; 21 | 22 | private FilesManagementResponseCodeEnum(Long code) { 23 | this.code = code; 24 | } 25 | 26 | @Override 27 | public Long getResponseCode() { 28 | return code; 29 | } 30 | 31 | @Override 32 | public String getCategoryName() { 33 | return CATEGORY_NAME; 34 | } 35 | 36 | @Override 37 | public String getCodeName() { 38 | return name(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /microservices/files_management/src/main/java/com/dreamsoftware/documentsearchengine/web/controller/error/FilesManagementErrorController.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.controller.error; 2 | 3 | import com.dreamsoftware.documentsearchengine.web.controller.FilesManagementResponseCodeEnum; 4 | import com.dreamsoftware.documentsearchengine.web.controller.error.exception.DeleteFileException; 5 | import com.dreamsoftware.documentsearchengine.web.controller.error.exception.FileAlreadyProcessedException; 6 | import com.dreamsoftware.documentsearchengine.web.controller.error.exception.SaveFileException; 7 | import com.dreamsoftware.documentsearchengine.web.core.APIResponse; 8 | import com.dreamsoftware.documentsearchengine.web.core.ErrorResponseDTO; 9 | import com.dreamsoftware.documentsearchengine.web.core.FieldErrorDTO; 10 | import com.dreamsoftware.documentsearchengine.web.core.SupportController; 11 | import java.util.List; 12 | import java.util.stream.Collectors; 13 | import javax.servlet.http.HttpServletRequest; 14 | import javax.validation.ConstraintViolationException; 15 | import org.slf4j.Logger; 16 | import org.slf4j.LoggerFactory; 17 | import org.springframework.core.Ordered; 18 | import org.springframework.core.annotation.Order; 19 | import org.springframework.http.HttpStatus; 20 | import org.springframework.http.ResponseEntity; 21 | import org.springframework.web.bind.annotation.ControllerAdvice; 22 | import org.springframework.web.bind.annotation.ExceptionHandler; 23 | import org.springframework.web.bind.annotation.ResponseBody; 24 | 25 | /** 26 | * 27 | * @author ssanchez 28 | */ 29 | @ControllerAdvice 30 | @Order(Ordered.HIGHEST_PRECEDENCE) 31 | public class FilesManagementErrorController extends SupportController { 32 | 33 | private static final Logger logger = LoggerFactory.getLogger(FilesManagementErrorController.class); 34 | 35 | /** 36 | * Handler for Constraint Violation Exception 37 | * 38 | * @param ex 39 | * @return 40 | */ 41 | @ExceptionHandler(ConstraintViolationException.class) 42 | @ResponseBody 43 | public ResponseEntity> handleConstraintViolationException(ConstraintViolationException ex) { 44 | 45 | List fieldErrors = ex.getConstraintViolations().stream() 46 | .map(constraintViolation -> FieldErrorDTO.builder() 47 | .field(constraintViolation.getPropertyPath().toString()) 48 | .message(constraintViolation.getMessage()) 49 | .build()) 50 | .collect(Collectors.toList()); 51 | 52 | return responseHelper.createAndSendErrorResponse( 53 | FilesManagementResponseCodeEnum.VALIDATION_ERROR, 54 | HttpStatus.BAD_REQUEST, fieldErrors); 55 | 56 | } 57 | 58 | /** 59 | * 60 | * @param ex 61 | * @param request 62 | * @return 63 | */ 64 | @ExceptionHandler(SaveFileException.class) 65 | @ResponseBody 66 | protected ResponseEntity> handleSaveFileException(SaveFileException ex, HttpServletRequest request) { 67 | ex.printStackTrace(); 68 | logger.debug("SaveFileException -> " + ex.getMessage()); 69 | 70 | return responseHelper.createAndSendErrorResponse(FilesManagementResponseCodeEnum.SAVE_FILE_ERROR, 71 | HttpStatus.INTERNAL_SERVER_ERROR, "Save File Error"); 72 | } 73 | 74 | /** 75 | * Handler for File already processed exception 76 | * 77 | * @param ex 78 | * @param request 79 | * @return 80 | */ 81 | @ExceptionHandler(FileAlreadyProcessedException.class) 82 | @ResponseBody 83 | protected ResponseEntity> handleFileAlreadyProcessedException(FileAlreadyProcessedException ex, HttpServletRequest request) { 84 | logger.debug("FileAlreadyProcessedException -> " + ex.getMessage()); 85 | 86 | return responseHelper.createAndSendErrorResponse(FilesManagementResponseCodeEnum.FILE_ALREADY_PROCESSED, 87 | HttpStatus.INTERNAL_SERVER_ERROR, "File Already Processed"); 88 | } 89 | 90 | /** 91 | * Handler for Delete file exception 92 | * 93 | * @param ex 94 | * @param request 95 | * @return 96 | */ 97 | @ExceptionHandler(DeleteFileException.class) 98 | @ResponseBody 99 | protected ResponseEntity> handleDeleteFileException(DeleteFileException ex, HttpServletRequest request) { 100 | logger.debug("DeleteFileException -> " + ex.getMessage()); 101 | 102 | return responseHelper.createAndSendErrorResponse(FilesManagementResponseCodeEnum.DELETE_FILE_ERROR, 103 | HttpStatus.INTERNAL_SERVER_ERROR, "An error ocurred when try to delete this processed file"); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /microservices/files_management/src/main/java/com/dreamsoftware/documentsearchengine/web/controller/error/exception/DeleteFileException.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.controller.error.exception; 2 | 3 | /** 4 | * 5 | * @author ssanchez 6 | */ 7 | public class DeleteFileException extends RuntimeException { 8 | 9 | public DeleteFileException(Exception ex) { 10 | super(ex); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /microservices/files_management/src/main/java/com/dreamsoftware/documentsearchengine/web/controller/error/exception/FileAlreadyProcessedException.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.controller.error.exception; 2 | 3 | /** 4 | * 5 | * @author ssanchez 6 | */ 7 | public class FileAlreadyProcessedException extends RuntimeException { 8 | 9 | public FileAlreadyProcessedException() { 10 | super(); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /microservices/files_management/src/main/java/com/dreamsoftware/documentsearchengine/web/controller/error/exception/SaveFileException.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.controller.error.exception; 2 | 3 | /** 4 | * 5 | * @author ssanchez 6 | */ 7 | public class SaveFileException extends RuntimeException { 8 | 9 | public SaveFileException(Exception ex) { 10 | super(ex); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /microservices/files_management/src/main/java/com/dreamsoftware/documentsearchengine/web/dto/ProcessedFileDTO.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.dto; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Data; 6 | import lombok.EqualsAndHashCode; 7 | import lombok.NoArgsConstructor; 8 | 9 | /** 10 | * 11 | * @author ssanchez 12 | */ 13 | @Data 14 | @EqualsAndHashCode(callSuper = false) 15 | @AllArgsConstructor 16 | @NoArgsConstructor 17 | public class ProcessedFileDTO { 18 | 19 | /** 20 | * File Id 21 | */ 22 | @JsonProperty("id") 23 | private String id; 24 | 25 | /** 26 | * File UUID 27 | */ 28 | @JsonProperty("uuid") 29 | private String uuid; 30 | 31 | /** 32 | * File Name 33 | */ 34 | @JsonProperty("name") 35 | private String name; 36 | 37 | /** 38 | * File Permissions 39 | */ 40 | @JsonProperty("permissions") 41 | private String permissions; 42 | 43 | /** 44 | * File Mime Type 45 | */ 46 | @JsonProperty("mime_type") 47 | private String mimeType; 48 | 49 | /** 50 | * File Mime Extension 51 | */ 52 | @JsonProperty("mime_extension") 53 | private String mimeExtension; 54 | 55 | /** 56 | * File Created At 57 | */ 58 | @JsonProperty("created_at") 59 | private String createdAt; 60 | 61 | /** 62 | * File Language 63 | */ 64 | @JsonProperty("language") 65 | private String language; 66 | 67 | /** 68 | * File Author 69 | */ 70 | @JsonProperty("author") 71 | private String author; 72 | 73 | /** 74 | * File Creator 75 | */ 76 | @JsonProperty("creator") 77 | private String creator; 78 | 79 | /** 80 | * File Producer 81 | */ 82 | @JsonProperty("producer") 83 | private String producer; 84 | 85 | /** 86 | * File Last Modified Time 87 | */ 88 | @JsonProperty("last_modified_time") 89 | private String lastModifiedTime; 90 | 91 | /** 92 | * File Processed At 93 | */ 94 | @JsonProperty("processed_at") 95 | private String processedAt; 96 | 97 | } 98 | -------------------------------------------------------------------------------- /microservices/files_management/src/main/java/com/dreamsoftware/documentsearchengine/web/dto/RequestOperationToProcessedFileDTO.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.dto; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Builder; 6 | import lombok.Data; 7 | import lombok.EqualsAndHashCode; 8 | import lombok.NoArgsConstructor; 9 | 10 | /** 11 | * 12 | * @author ssanchez 13 | */ 14 | @Data 15 | @Builder 16 | @EqualsAndHashCode(callSuper = false) 17 | @AllArgsConstructor 18 | @NoArgsConstructor 19 | public class RequestOperationToProcessedFileDTO { 20 | 21 | /** 22 | * File Id 23 | */ 24 | @JsonProperty("id") 25 | private String id; 26 | 27 | /** 28 | * File Name 29 | */ 30 | @JsonProperty("name") 31 | private String filename; 32 | 33 | /** 34 | * Operation 35 | */ 36 | @JsonProperty("operation") 37 | private RequestOperationTypeEnum operation; 38 | 39 | } 40 | -------------------------------------------------------------------------------- /microservices/files_management/src/main/java/com/dreamsoftware/documentsearchengine/web/dto/RequestOperationTypeEnum.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.dto; 2 | 3 | /** 4 | * 5 | * @author ssanchez 6 | */ 7 | public enum RequestOperationTypeEnum { 8 | DELETE 9 | } 10 | -------------------------------------------------------------------------------- /microservices/files_management/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | application: 3 | name: files-management-service 4 | security: 5 | oauth2: 6 | resourceserver: 7 | jwt: 8 | issuer-uri: http://keycloak:8080/auth/realms/document_search_engine 9 | cloud: 10 | consul: 11 | host: consul-server 12 | port: 8500 13 | discovery: 14 | port: 8080 15 | prefer-ip-address: true 16 | healthCheckPath: /actuator/health 17 | stream: 18 | kafka: 19 | binder: 20 | brokers: kafka:9092 21 | requiredAcks: 1 22 | auto-create-topics: false 23 | configuration: 24 | auto: 25 | offset.reset: latest 26 | bindings: 27 | files-processed-operations: 28 | consumer: 29 | autoCommitOffset: false 30 | bindings: 31 | processed-files-state: 32 | group: files-management 33 | 34 | ## SFTP Config 35 | sftp: 36 | hostname: sftp 37 | port: 22 38 | username: ssanchez 39 | password: ssanchez00 40 | remoteFolder: uploads/ 41 | 42 | ## Uploads Config 43 | uploads: 44 | directory: /opt/uploads 45 | 46 | ## Spring Open API 47 | springdoc: 48 | swagger-ui: 49 | path: /swagger-ui.html 50 | api-docs: 51 | path: /v3/api-docs 52 | 53 | logging.level: 54 | org.springframework.cloud.gateway: DEBUG 55 | org.springframework.security: DEBUG 56 | com.dreamsoftware.documentsearchengine: DEBUG 57 | org.springframework.web.reactive.function.client: TRACE 58 | 59 | server.port: 8080 -------------------------------------------------------------------------------- /microservices/files_metadata/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-jdk-alpine 2 | # Add Maintainer Info 3 | LABEL description="Document Search Engine Files Metadata Service" 4 | # Args for image 5 | ARG PORT=8080 6 | 7 | RUN apk update && apk upgrade 8 | RUN ln -s /bin/bash /usr/bin 9 | RUN mkdir -p /usr/src/app 10 | WORKDIR /usr/src/app 11 | 12 | 13 | COPY resources/wait-for-it.sh wait-for-it.sh 14 | COPY target/file_metadata_service.jar app.jar 15 | 16 | RUN dos2unix wait-for-it.sh 17 | RUN chmod +x wait-for-it.sh 18 | RUN uname -a 19 | RUN pwd 20 | RUN ls -al 21 | 22 | EXPOSE ${PORT} 23 | 24 | CMD ["sh", "-c", "echo 'waiting for 300 seconds for consul-agent:8500 to be accessable before starting application' && ./wait-for-it.sh -t 300 consul:8500 -- java -jar app.jar"] -------------------------------------------------------------------------------- /microservices/files_metadata/nbactions.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | run 5 | 6 | jar 7 | 8 | 9 | process-classes 10 | org.codehaus.mojo:exec-maven-plugin:1.5.0:exec 11 | 12 | 13 | -classpath %classpath ${packageClassName} 14 | java 15 | 16 | 17 | 18 | debug 19 | 20 | jar 21 | 22 | 23 | process-classes 24 | org.codehaus.mojo:exec-maven-plugin:1.5.0:exec 25 | 26 | 27 | -agentlib:jdwp=transport=dt_socket,server=n,address=${jpda.address} -classpath %classpath ${packageClassName} 28 | java 29 | true 30 | 31 | 32 | 33 | profile 34 | 35 | jar 36 | 37 | 38 | process-classes 39 | org.codehaus.mojo:exec-maven-plugin:1.5.0:exec 40 | 41 | 42 | -classpath %classpath ${packageClassName} 43 | java 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /microservices/files_metadata/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 4.0.0 7 | com.dreamsoftware 8 | document_search_engine_files_metadata 9 | 0.0.1-SNAPSHOT 10 | jar 11 | file_metadata_service 12 | Document Search Engine Files Metadata Service 13 | 14 | 15 | com.dreamsoftware 16 | document_search_engine 17 | 0.0.1-SNAPSHOT 18 | ../ 19 | 20 | 21 | 22 | 23 | 24 | 25 | ${project.groupId} 26 | document_search_engine_files_commons 27 | ${project.version} 28 | 29 | 30 | 31 | org.springframework.boot 32 | spring-boot-starter-data-mongodb 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | org.springframework.boot 41 | spring-boot-maven-plugin 42 | 43 | file_metadata_service 44 | 45 | 46 | 47 | com.spotify 48 | dockerfile-maven-plugin 49 | 1.4.13 50 | 51 | 52 | build-image 53 | package 54 | 55 | build 56 | 57 | 58 | 59 | 60 | 61 | 62 | ssanchez11/file_metadata_service 63 | ${project.version} 64 | 65 | file_metadata_service 66 | 67 | false 68 | true 69 | 70 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /microservices/files_metadata/resources/wait-for-it.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | TIMEOUT=15 4 | QUIET=0 5 | 6 | echoerr() { 7 | if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi 8 | } 9 | 10 | usage() { 11 | exitcode="$1" 12 | cat << USAGE >&2 13 | Usage: 14 | $cmdname host:port [-t timeout] [-- command args] 15 | -q | --quiet Do not output any status messages 16 | -t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout 17 | -- COMMAND ARGS Execute command with args after the test finishes 18 | USAGE 19 | exit "$exitcode" 20 | } 21 | 22 | wait_for() { 23 | for i in `seq $TIMEOUT` ; do 24 | nc -z "$HOST" "$PORT" > /dev/null 2>&1 25 | 26 | result=$? 27 | if [ $result -eq 0 ] ; then 28 | if [ $# -gt 0 ] ; then 29 | exec "$@" 30 | fi 31 | exit 0 32 | fi 33 | sleep 1 34 | done 35 | echo "Operation timed out" >&2 36 | exit 1 37 | } 38 | 39 | while [ $# -gt 0 ] 40 | do 41 | case "$1" in 42 | *:* ) 43 | HOST=$(printf "%s\n" "$1"| cut -d : -f 1) 44 | PORT=$(printf "%s\n" "$1"| cut -d : -f 2) 45 | shift 1 46 | ;; 47 | -q | --quiet) 48 | QUIET=1 49 | shift 1 50 | ;; 51 | -t) 52 | TIMEOUT="$2" 53 | if [ "$TIMEOUT" = "" ]; then break; fi 54 | shift 2 55 | ;; 56 | --timeout=*) 57 | TIMEOUT="${1#*=}" 58 | shift 1 59 | ;; 60 | --) 61 | shift 62 | break 63 | ;; 64 | --help) 65 | usage 0 66 | ;; 67 | *) 68 | echoerr "Unknown argument: $1" 69 | usage 1 70 | ;; 71 | esac 72 | done 73 | 74 | if [ "$HOST" = "" -o "$PORT" = "" ]; then 75 | echoerr "Error: you need to provide a host and port to test." 76 | usage 2 77 | fi 78 | 79 | wait_for "$@" -------------------------------------------------------------------------------- /microservices/files_metadata/src/main/java/com/dreamsoftware/documentsearchengine/FilesMetadataServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine; 2 | 3 | import io.swagger.v3.oas.annotations.OpenAPIDefinition; 4 | import io.swagger.v3.oas.annotations.info.Info; 5 | import org.springframework.boot.SpringApplication; 6 | import org.springframework.boot.autoconfigure.SpringBootApplication; 7 | 8 | @SpringBootApplication 9 | @OpenAPIDefinition(info 10 | = @Info( 11 | title = "Files Metadata API", 12 | version = "1.0", 13 | description = "Document Search Engine - Files Metadata API v1.0" 14 | ) 15 | ) 16 | public class FilesMetadataServiceApplication { 17 | 18 | public static void main(String[] args) { 19 | SpringApplication.run(FilesMetadataServiceApplication.class, args); 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /microservices/files_metadata/src/main/java/com/dreamsoftware/documentsearchengine/config/OpenApiConfig.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.config; 2 | 3 | import io.swagger.v3.oas.models.Components; 4 | import io.swagger.v3.oas.models.ExternalDocumentation; 5 | import io.swagger.v3.oas.models.OpenAPI; 6 | import io.swagger.v3.oas.models.info.Info; 7 | import io.swagger.v3.oas.models.info.License; 8 | import io.swagger.v3.oas.models.security.SecurityRequirement; 9 | import io.swagger.v3.oas.models.security.SecurityScheme; 10 | import java.util.Arrays; 11 | import org.springframework.context.annotation.Bean; 12 | import org.springframework.context.annotation.Configuration; 13 | 14 | /** 15 | * 16 | * @author ssanchez 17 | */ 18 | @Configuration 19 | public class OpenApiConfig { 20 | 21 | @Bean 22 | public OpenAPI springShopOpenAPI() { 23 | return new OpenAPI() 24 | .components(new Components().addSecuritySchemes("bearer-jwt", 25 | new SecurityScheme() 26 | .description("Auth Header with Json Web Token format") 27 | .type(SecurityScheme.Type.HTTP) 28 | .scheme("bearer") 29 | .bearerFormat("JWT") 30 | .in(SecurityScheme.In.HEADER).name("Authorization"))) 31 | .addSecurityItem( 32 | new SecurityRequirement().addList("bearer-jwt", Arrays.asList("read", "write"))) 33 | .info(new Info().title("Files Metadata REST API") 34 | .description("Provides a hypermedia-driven REST API for Apps") 35 | .version("v0.0.1-SNAPSHOT") 36 | .license(new License().name("Apache 2.0").url("http://springdoc.org"))) 37 | .externalDocs(new ExternalDocumentation() 38 | .description("Provides a hypermedia-driven REST API for Apps") 39 | .url("https://springshop.wiki.github.org/docs") 40 | ); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /microservices/files_metadata/src/main/java/com/dreamsoftware/documentsearchengine/mapper/ProcessedFileMapper.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.mapper; 2 | 3 | import com.dreamsoftware.documentsearchengine.persistence.entity.ProcessedFileEntity; 4 | import com.dreamsoftware.documentsearchengine.web.dto.ProcessedFileDTO; 5 | import java.io.UnsupportedEncodingException; 6 | import java.net.URLEncoder; 7 | import java.nio.charset.StandardCharsets; 8 | import java.util.List; 9 | import java.util.logging.Level; 10 | import java.util.logging.Logger; 11 | import org.mapstruct.IterableMapping; 12 | import org.mapstruct.Mapper; 13 | import org.mapstruct.Mapping; 14 | import org.mapstruct.Mappings; 15 | import org.mapstruct.Named; 16 | 17 | /** 18 | * 19 | * @author ssanchez 20 | */ 21 | @Mapper(unmappedTargetPolicy = org.mapstruct.ReportingPolicy.IGNORE) 22 | public abstract class ProcessedFileMapper { 23 | 24 | /** 25 | * 26 | * @param processedFileEntity 27 | * @return 28 | */ 29 | @Mappings({ 30 | @Mapping(expression = "java(processedFileEntity.getId().toString())", target = "id"), 31 | @Mapping(expression = "java(createWebhdfsUrlForFilename(processedFileEntity))", target = "url") 32 | }) 33 | @Named("entityToDTO") 34 | public abstract ProcessedFileDTO entityToDTO(ProcessedFileEntity processedFileEntity); 35 | 36 | /** 37 | * 38 | * @param processedFileEntityList 39 | * @return 40 | */ 41 | @IterableMapping(qualifiedByName = "entityToDTO") 42 | public abstract List entityToDTO(List processedFileEntityList); 43 | 44 | /** 45 | * 46 | * @param processedFileEntity 47 | * @return 48 | */ 49 | protected String createWebhdfsUrlForFilename(final ProcessedFileEntity processedFileEntity) { 50 | try { 51 | return String.format("http://namenode:9870/webhdfs/v1/uploads/%s?op=OPEN", URLEncoder.encode(processedFileEntity.getName().replace("_", "."), "UTF-8").replace("+", "%20")); 52 | } catch (UnsupportedEncodingException ex) { 53 | throw new RuntimeException(ex); 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /microservices/files_metadata/src/main/java/com/dreamsoftware/documentsearchengine/persistence/entity/ProcessedFileEntity.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.persistence.entity; 2 | 3 | import java.util.Date; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Data; 6 | import lombok.EqualsAndHashCode; 7 | import lombok.NoArgsConstructor; 8 | import org.bson.types.ObjectId; 9 | import org.springframework.data.annotation.Id; 10 | import org.springframework.data.mongodb.core.mapping.Document; 11 | import org.springframework.data.mongodb.core.mapping.Field; 12 | 13 | /** 14 | * 15 | * @author ssanchez 16 | */ 17 | @Data 18 | @EqualsAndHashCode(callSuper = false) 19 | @AllArgsConstructor 20 | @NoArgsConstructor 21 | @Document(collection = ProcessedFileEntity.COLLECTION_NAME) 22 | public class ProcessedFileEntity { 23 | 24 | public final static String COLLECTION_NAME = "processed_files"; 25 | 26 | /** 27 | * Id 28 | */ 29 | @Id 30 | private ObjectId id; 31 | 32 | /** 33 | * File Name 34 | */ 35 | @Field("attrs.filename") 36 | private String name; 37 | 38 | /** 39 | * File Permissions 40 | */ 41 | @Field("attrs.file_permissions") 42 | private String permissions; 43 | 44 | /** 45 | * File Mime Type 46 | */ 47 | @Field("attrs.mime_type") 48 | private String mimeType; 49 | 50 | /** 51 | * File Mime Extension 52 | */ 53 | @Field("attrs.mime_extension") 54 | private String mimeExtension; 55 | 56 | /** 57 | * File UUID 58 | */ 59 | @Field("attrs.uuid") 60 | private String uuid; 61 | 62 | /** 63 | * File Created At 64 | */ 65 | @Field("metadata.created") 66 | private String createdAt; 67 | 68 | /** 69 | * File Language 70 | */ 71 | @Field("metadata.language") 72 | private String language; 73 | 74 | /** 75 | * File Author 76 | */ 77 | @Field("metadata.Author") 78 | private String author; 79 | 80 | /** 81 | * File Creator 82 | */ 83 | @Field("metadata.creator") 84 | private String creator; 85 | 86 | /** 87 | * File Producer 88 | */ 89 | @Field("metadata.producer") 90 | private String producer; 91 | /** 92 | * File Last Modified Time 93 | */ 94 | @Field("attrs.file_lastModifiedTime") 95 | private String lastModifiedTime; 96 | 97 | /** 98 | * File Processed At 99 | */ 100 | @Field("document.processed_at") 101 | private String processedAt; 102 | 103 | } 104 | -------------------------------------------------------------------------------- /microservices/files_metadata/src/main/java/com/dreamsoftware/documentsearchengine/persistence/repository/FilesProcessedRepository.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.persistence.repository; 2 | 3 | import com.dreamsoftware.documentsearchengine.persistence.entity.ProcessedFileEntity; 4 | import java.util.Optional; 5 | import org.bson.types.ObjectId; 6 | import org.springframework.data.mongodb.repository.MongoRepository; 7 | import org.springframework.stereotype.Repository; 8 | 9 | /** 10 | * 11 | * @author ssanchez 12 | */ 13 | @Repository 14 | public interface FilesProcessedRepository extends MongoRepository { 15 | 16 | /** 17 | * Find One By Name 18 | * 19 | * @param name 20 | * @return 21 | */ 22 | Optional findOneByName(final String name); 23 | 24 | } 25 | -------------------------------------------------------------------------------- /microservices/files_metadata/src/main/java/com/dreamsoftware/documentsearchengine/service/IFilesProcessedService.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.service; 2 | 3 | import com.dreamsoftware.documentsearchengine.web.dto.ProcessedFileDTO; 4 | import java.util.Optional; 5 | import org.bson.types.ObjectId; 6 | import org.springframework.data.domain.Page; 7 | import org.springframework.data.domain.Pageable; 8 | 9 | /** 10 | * 11 | * @author ssanchez 12 | */ 13 | public interface IFilesProcessedService { 14 | 15 | /** 16 | * 17 | * @param page 18 | * @param size 19 | * @return 20 | */ 21 | Page findPaginated(final Integer page, final Integer size); 22 | 23 | /** 24 | * 25 | * @param pageable 26 | * @return 27 | */ 28 | Page findPaginated(final Pageable pageable); 29 | 30 | /** 31 | * 32 | * @param id 33 | * @return 34 | */ 35 | Optional findById(final ObjectId id); 36 | 37 | /** 38 | * 39 | * @param name 40 | * @return 41 | */ 42 | Optional findByName(final String name); 43 | 44 | } 45 | -------------------------------------------------------------------------------- /microservices/files_metadata/src/main/java/com/dreamsoftware/documentsearchengine/service/impl/FilesProcessedServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.service.impl; 2 | 3 | import com.dreamsoftware.documentsearchengine.mapper.ProcessedFileMapper; 4 | import com.dreamsoftware.documentsearchengine.persistence.repository.FilesProcessedRepository; 5 | import com.dreamsoftware.documentsearchengine.service.IFilesProcessedService; 6 | import com.dreamsoftware.documentsearchengine.web.dto.ProcessedFileDTO; 7 | import java.util.Optional; 8 | import lombok.RequiredArgsConstructor; 9 | import org.bson.types.ObjectId; 10 | import org.springframework.data.domain.Page; 11 | import org.springframework.data.domain.PageRequest; 12 | import org.springframework.data.domain.Pageable; 13 | import org.springframework.stereotype.Service; 14 | import org.springframework.util.Assert; 15 | 16 | /** 17 | * 18 | * @author ssanchez 19 | */ 20 | @Service("filesProcessedService") 21 | @RequiredArgsConstructor 22 | public class FilesProcessedServiceImpl implements IFilesProcessedService { 23 | 24 | private final ProcessedFileMapper processedFileMapper; 25 | private final FilesProcessedRepository filesProcessedRepository; 26 | 27 | /** 28 | * 29 | * @param page 30 | * @param size 31 | * @return 32 | */ 33 | @Override 34 | public Page findPaginated(final Integer page, final Integer size) { 35 | Assert.notNull(page, "Page can not be null"); 36 | Assert.notNull(size, "Size can not be null"); 37 | return findPaginated(PageRequest.of(page, size)); 38 | } 39 | 40 | /** 41 | * 42 | * @param pageable 43 | * @return 44 | */ 45 | @Override 46 | public Page findPaginated(final Pageable pageable) { 47 | Assert.notNull(pageable, "Pageable can not be null"); 48 | 49 | return filesProcessedRepository.findAll(pageable) 50 | .map(processedFileMapper::entityToDTO); 51 | } 52 | 53 | /** 54 | * 55 | * @param id 56 | * @return 57 | */ 58 | @Override 59 | public Optional findById(ObjectId id) { 60 | Assert.notNull(id, "File id can not be null"); 61 | 62 | return filesProcessedRepository.findById(id) 63 | .map(processedFileMapper::entityToDTO); 64 | } 65 | 66 | /** 67 | * Find By Name 68 | * 69 | * @param name 70 | * @return 71 | */ 72 | @Override 73 | public Optional findByName(final String name) { 74 | Assert.notNull(name, "File name can not be null"); 75 | 76 | return filesProcessedRepository.findOneByName(name) 77 | .map(processedFileMapper::entityToDTO); 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /microservices/files_metadata/src/main/java/com/dreamsoftware/documentsearchengine/web/controller/FilesMetadataResponseCodeEnum.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.controller; 2 | 3 | import com.dreamsoftware.documentsearchengine.web.core.IResponseCodeTypes; 4 | 5 | /** 6 | * 7 | * @author ssanchez 8 | */ 9 | public enum FilesMetadataResponseCodeEnum implements IResponseCodeTypes { 10 | 11 | GET_FILES_PROCESSED(100L), 12 | GET_PROCESSED_FILE_DETAIL(101L), 13 | NO_FILES_PROCESSED_FOUND(102L), 14 | PROCESSED_FILE_NOT_FOUND(103L), 15 | VALIDATION_ERROR(104L); 16 | 17 | private final Long code; 18 | 19 | public static final String CATEGORY_NAME = "FILES_METADATA"; 20 | 21 | private FilesMetadataResponseCodeEnum(Long code) { 22 | this.code = code; 23 | } 24 | 25 | @Override 26 | public Long getResponseCode() { 27 | return code; 28 | } 29 | 30 | @Override 31 | public String getCategoryName() { 32 | return CATEGORY_NAME; 33 | } 34 | 35 | @Override 36 | public String getCodeName() { 37 | return name(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /microservices/files_metadata/src/main/java/com/dreamsoftware/documentsearchengine/web/controller/error/FilesMetadataErrorController.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.controller.error; 2 | 3 | import com.dreamsoftware.documentsearchengine.web.controller.FilesMetadataResponseCodeEnum; 4 | import com.dreamsoftware.documentsearchengine.web.controller.error.exception.FileProcessedNotFoundException; 5 | import com.dreamsoftware.documentsearchengine.web.controller.error.exception.NoFilesProcessedFoundException; 6 | import com.dreamsoftware.documentsearchengine.web.core.APIResponse; 7 | import com.dreamsoftware.documentsearchengine.web.core.ErrorResponseDTO; 8 | import com.dreamsoftware.documentsearchengine.web.core.FieldErrorDTO; 9 | import com.dreamsoftware.documentsearchengine.web.core.SupportController; 10 | import java.util.List; 11 | import java.util.stream.Collectors; 12 | import javax.servlet.http.HttpServletRequest; 13 | import javax.validation.ConstraintViolationException; 14 | import org.springframework.core.Ordered; 15 | import org.springframework.core.annotation.Order; 16 | import org.springframework.http.HttpStatus; 17 | import org.springframework.http.ResponseEntity; 18 | import org.springframework.web.bind.annotation.ControllerAdvice; 19 | import org.springframework.web.bind.annotation.ExceptionHandler; 20 | import org.springframework.web.bind.annotation.ResponseBody; 21 | 22 | /** 23 | * 24 | * @author ssanchez 25 | */ 26 | @ControllerAdvice 27 | @Order(Ordered.HIGHEST_PRECEDENCE) 28 | public class FilesMetadataErrorController extends SupportController { 29 | 30 | /** 31 | * Handler for Constraint Violation Exception 32 | * 33 | * @param ex 34 | * @return 35 | */ 36 | @ExceptionHandler(ConstraintViolationException.class) 37 | @ResponseBody 38 | public ResponseEntity> handleConstraintViolationException(ConstraintViolationException ex) { 39 | 40 | List fieldErrors = ex.getConstraintViolations().stream() 41 | .map(constraintViolation -> FieldErrorDTO.builder() 42 | .field(constraintViolation.getPropertyPath().toString()) 43 | .message(constraintViolation.getMessage()) 44 | .build()) 45 | .collect(Collectors.toList()); 46 | 47 | return responseHelper.createAndSendErrorResponse( 48 | FilesMetadataResponseCodeEnum.VALIDATION_ERROR, 49 | HttpStatus.BAD_REQUEST, fieldErrors); 50 | 51 | } 52 | 53 | /** 54 | * 55 | * @param ex 56 | * @param request 57 | * @return 58 | */ 59 | @ExceptionHandler(FileProcessedNotFoundException.class) 60 | @ResponseBody 61 | protected ResponseEntity> handleFileProcessedNotFoundException(FileProcessedNotFoundException ex, HttpServletRequest request) { 62 | return responseHelper.createAndSendErrorResponse(FilesMetadataResponseCodeEnum.PROCESSED_FILE_NOT_FOUND, 63 | HttpStatus.NOT_FOUND, "File Proccessed Not Found"); 64 | } 65 | 66 | /** 67 | * 68 | * @param ex 69 | * @param request 70 | * @return 71 | */ 72 | @ExceptionHandler(NoFilesProcessedFoundException.class) 73 | @ResponseBody 74 | protected ResponseEntity> handleNoFilesProcessedFoundException(NoFilesProcessedFoundException ex, HttpServletRequest request) { 75 | return responseHelper.createAndSendErrorResponse(FilesMetadataResponseCodeEnum.NO_FILES_PROCESSED_FOUND, 76 | HttpStatus.NOT_FOUND, "No Files Processed."); 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /microservices/files_metadata/src/main/java/com/dreamsoftware/documentsearchengine/web/controller/error/exception/FileProcessedNotFoundException.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.controller.error.exception; 2 | 3 | /** 4 | * 5 | * @author ssanchez 6 | */ 7 | public class FileProcessedNotFoundException extends RuntimeException { 8 | } 9 | -------------------------------------------------------------------------------- /microservices/files_metadata/src/main/java/com/dreamsoftware/documentsearchengine/web/controller/error/exception/NoFilesProcessedFoundException.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.controller.error.exception; 2 | 3 | /** 4 | * 5 | * @author ssanchez 6 | */ 7 | public class NoFilesProcessedFoundException extends RuntimeException { 8 | } 9 | -------------------------------------------------------------------------------- /microservices/files_metadata/src/main/java/com/dreamsoftware/documentsearchengine/web/dto/ProcessedFileDTO.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.dto; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Data; 6 | import lombok.EqualsAndHashCode; 7 | import lombok.NoArgsConstructor; 8 | 9 | /** 10 | * 11 | * @author ssanchez 12 | */ 13 | @Data 14 | @EqualsAndHashCode(callSuper = false) 15 | @AllArgsConstructor 16 | @NoArgsConstructor 17 | public class ProcessedFileDTO { 18 | 19 | /** 20 | * File Id 21 | */ 22 | @JsonProperty("id") 23 | private String id; 24 | 25 | /** 26 | * File UUID 27 | */ 28 | @JsonProperty("uuid") 29 | private String uuid; 30 | 31 | /** 32 | * File Name 33 | */ 34 | @JsonProperty("name") 35 | private String name; 36 | 37 | /** 38 | * File URL 39 | */ 40 | @JsonProperty("url") 41 | private String url; 42 | 43 | /** 44 | * File Permissions 45 | */ 46 | @JsonProperty("permissions") 47 | private String permissions; 48 | 49 | /** 50 | * File Mime Type 51 | */ 52 | @JsonProperty("mime_type") 53 | private String mimeType; 54 | 55 | /** 56 | * File Mime Extension 57 | */ 58 | @JsonProperty("mime_extension") 59 | private String mimeExtension; 60 | 61 | /** 62 | * File Created At 63 | */ 64 | @JsonProperty("created_at") 65 | private String createdAt; 66 | 67 | /** 68 | * File Language 69 | */ 70 | @JsonProperty("language") 71 | private String language; 72 | 73 | /** 74 | * File Author 75 | */ 76 | @JsonProperty("author") 77 | private String author; 78 | 79 | /** 80 | * File Creator 81 | */ 82 | @JsonProperty("creator") 83 | private String creator; 84 | 85 | /** 86 | * File Producer 87 | */ 88 | @JsonProperty("producer") 89 | private String producer; 90 | 91 | /** 92 | * File Last Modified Time 93 | */ 94 | @JsonProperty("last_modified_time") 95 | private String lastModifiedTime; 96 | 97 | /** 98 | * File Processed At 99 | */ 100 | @JsonProperty("processed_at") 101 | private String processedAt; 102 | 103 | } 104 | -------------------------------------------------------------------------------- /microservices/files_metadata/src/main/java/com/dreamsoftware/documentsearchengine/web/validators/ValidObjectId.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.validators; 2 | 3 | import static java.lang.annotation.ElementType.ANNOTATION_TYPE; 4 | import static java.lang.annotation.ElementType.CONSTRUCTOR; 5 | import static java.lang.annotation.ElementType.FIELD; 6 | import static java.lang.annotation.ElementType.METHOD; 7 | import static java.lang.annotation.ElementType.PARAMETER; 8 | import static java.lang.annotation.ElementType.TYPE_USE; 9 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 10 | 11 | import java.lang.annotation.Documented; 12 | import java.lang.annotation.Retention; 13 | import java.lang.annotation.Target; 14 | 15 | import javax.validation.Constraint; 16 | import javax.validation.Payload; 17 | import javax.validation.constraints.NotNull; 18 | 19 | @Target({METHOD, FIELD, ANNOTATION_TYPE, CONSTRUCTOR, PARAMETER, TYPE_USE}) 20 | @Retention(RUNTIME) 21 | @Constraint(validatedBy = ValidObjectIdValidator.class) 22 | @NotNull 23 | @Documented 24 | public @interface ValidObjectId { 25 | 26 | String message() default "{constraints.valid.objectid}"; 27 | 28 | Class[] groups() default {}; 29 | 30 | Class[] payload() default {}; 31 | } 32 | -------------------------------------------------------------------------------- /microservices/files_metadata/src/main/java/com/dreamsoftware/documentsearchengine/web/validators/ValidObjectIdValidator.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.validators; 2 | 3 | import javax.validation.ConstraintValidator; 4 | import javax.validation.ConstraintValidatorContext; 5 | import org.bson.types.ObjectId; 6 | 7 | /** 8 | * Valid Object Id Validator 9 | * 10 | * @author ssanchez 11 | */ 12 | public class ValidObjectIdValidator implements ConstraintValidator { 13 | 14 | @Override 15 | public void initialize(ValidObjectId constraintAnnotation) { 16 | } 17 | 18 | @Override 19 | public boolean isValid(String id, ConstraintValidatorContext cvc) { 20 | return ObjectId.isValid(id); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /microservices/files_metadata/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | application: 3 | name: files-metadata-service 4 | security: 5 | oauth2: 6 | resourceserver: 7 | jwt: 8 | issuer-uri: http://keycloak:8080/auth/realms/document_search_engine 9 | cloud: 10 | consul: 11 | host: consul-server 12 | port: 8500 13 | discovery: 14 | port: 8080 15 | prefer-ip-address: true 16 | healthCheckPath: /actuator/health 17 | data: 18 | mongodb: 19 | authentication-database: admin 20 | database: files 21 | host: mongo 22 | port: 27017 23 | username: devroot 24 | password: devroot 25 | 26 | ## Spring Open API 27 | springdoc: 28 | swagger-ui: 29 | path: /swagger-ui.html 30 | api-docs: 31 | path: /v3/api-docs 32 | 33 | logging.level: 34 | org.springframework.cloud.gateway: DEBUG 35 | org.springframework.security: DEBUG 36 | com.dreamsoftware.documentsearchengine: DEBUG 37 | org.springframework.web.reactive.function.client: TRACE 38 | 39 | server.port: 8080 -------------------------------------------------------------------------------- /microservices/files_notifications/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-jdk-alpine 2 | # Add Maintainer Info 3 | LABEL description="Document Search Engine Files Notification Service" 4 | # Args for image 5 | ARG PORT=8080 6 | 7 | RUN apk update && apk upgrade 8 | RUN ln -s /bin/bash /usr/bin 9 | RUN mkdir -p /usr/src/app 10 | WORKDIR /usr/src/app 11 | 12 | 13 | COPY resources/wait-for-it.sh wait-for-it.sh 14 | COPY target/file_notifications_service.jar app.jar 15 | 16 | RUN dos2unix wait-for-it.sh 17 | RUN chmod +x wait-for-it.sh 18 | RUN uname -a 19 | RUN pwd 20 | RUN ls -al 21 | 22 | EXPOSE ${PORT} 23 | 24 | CMD ["sh", "-c", "echo 'waiting for 300 seconds for consul-agent:8500 to be accessable before starting application' && ./wait-for-it.sh -t 300 consul:8500 -- java -jar app.jar"] -------------------------------------------------------------------------------- /microservices/files_notifications/nbactions.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | run 5 | 6 | jar 7 | 8 | 9 | process-classes 10 | org.codehaus.mojo:exec-maven-plugin:1.5.0:exec 11 | 12 | 13 | -classpath %classpath ${packageClassName} 14 | java 15 | 16 | 17 | 18 | debug 19 | 20 | jar 21 | 22 | 23 | process-classes 24 | org.codehaus.mojo:exec-maven-plugin:1.5.0:exec 25 | 26 | 27 | -agentlib:jdwp=transport=dt_socket,server=n,address=${jpda.address} -classpath %classpath ${packageClassName} 28 | java 29 | true 30 | 31 | 32 | 33 | profile 34 | 35 | jar 36 | 37 | 38 | process-classes 39 | org.codehaus.mojo:exec-maven-plugin:1.5.0:exec 40 | 41 | 42 | -classpath %classpath ${packageClassName} 43 | java 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /microservices/files_notifications/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 4.0.0 7 | com.dreamsoftware 8 | document_search_engine_files_notifications 9 | 0.0.1-SNAPSHOT 10 | jar 11 | file_notifications_service 12 | Document Search Engine Files Notification Service 13 | 14 | 15 | com.dreamsoftware 16 | document_search_engine 17 | 0.0.1-SNAPSHOT 18 | ../ 19 | 20 | 21 | 22 | 23 | ${project.groupId} 24 | document_search_engine_files_commons 25 | ${project.version} 26 | 27 | 28 | 29 | org.springframework.boot 30 | spring-boot-starter-data-mongodb 31 | 32 | 33 | 34 | org.springframework.cloud 35 | spring-cloud-stream 36 | 37 | 38 | org.springframework.cloud 39 | spring-cloud-starter-stream-kafka 40 | 41 | 42 | 43 | org.springframework.boot 44 | spring-boot-starter-websocket 45 | 46 | 47 | org.springframework.boot 48 | spring-boot-starter-reactor-netty 49 | 50 | 51 | 52 | 53 | 54 | 55 | org.springframework.boot 56 | spring-boot-maven-plugin 57 | 58 | file_notifications_service 59 | 60 | 61 | 62 | com.spotify 63 | dockerfile-maven-plugin 64 | 1.4.13 65 | 66 | 67 | build-image 68 | package 69 | 70 | build 71 | 72 | 73 | 74 | 75 | 76 | 77 | ssanchez11/file_notifications_service 78 | ${project.version} 79 | 80 | file_notifications_service 81 | 82 | false 83 | true 84 | 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /microservices/files_notifications/resources/wait-for-it.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | TIMEOUT=15 4 | QUIET=0 5 | 6 | echoerr() { 7 | if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi 8 | } 9 | 10 | usage() { 11 | exitcode="$1" 12 | cat << USAGE >&2 13 | Usage: 14 | $cmdname host:port [-t timeout] [-- command args] 15 | -q | --quiet Do not output any status messages 16 | -t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout 17 | -- COMMAND ARGS Execute command with args after the test finishes 18 | USAGE 19 | exit "$exitcode" 20 | } 21 | 22 | wait_for() { 23 | for i in `seq $TIMEOUT` ; do 24 | nc -z "$HOST" "$PORT" > /dev/null 2>&1 25 | 26 | result=$? 27 | if [ $result -eq 0 ] ; then 28 | if [ $# -gt 0 ] ; then 29 | exec "$@" 30 | fi 31 | exit 0 32 | fi 33 | sleep 1 34 | done 35 | echo "Operation timed out" >&2 36 | exit 1 37 | } 38 | 39 | while [ $# -gt 0 ] 40 | do 41 | case "$1" in 42 | *:* ) 43 | HOST=$(printf "%s\n" "$1"| cut -d : -f 1) 44 | PORT=$(printf "%s\n" "$1"| cut -d : -f 2) 45 | shift 1 46 | ;; 47 | -q | --quiet) 48 | QUIET=1 49 | shift 1 50 | ;; 51 | -t) 52 | TIMEOUT="$2" 53 | if [ "$TIMEOUT" = "" ]; then break; fi 54 | shift 2 55 | ;; 56 | --timeout=*) 57 | TIMEOUT="${1#*=}" 58 | shift 1 59 | ;; 60 | --) 61 | shift 62 | break 63 | ;; 64 | --help) 65 | usage 0 66 | ;; 67 | *) 68 | echoerr "Unknown argument: $1" 69 | usage 1 70 | ;; 71 | esac 72 | done 73 | 74 | if [ "$HOST" = "" -o "$PORT" = "" ]; then 75 | echoerr "Error: you need to provide a host and port to test." 76 | usage 2 77 | fi 78 | 79 | wait_for "$@" -------------------------------------------------------------------------------- /microservices/files_notifications/src/main/java/com/dreamsoftware/documentsearchengine/FilesNotificationsServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine; 2 | 3 | import com.dreamsoftware.documentsearchengine.config.AppStreamsConfig; 4 | import io.swagger.v3.oas.annotations.OpenAPIDefinition; 5 | import io.swagger.v3.oas.annotations.info.Info; 6 | import org.springframework.boot.SpringApplication; 7 | import org.springframework.boot.autoconfigure.SpringBootApplication; 8 | import org.springframework.cloud.stream.annotation.EnableBinding; 9 | 10 | @SpringBootApplication 11 | @OpenAPIDefinition(info 12 | = @Info( 13 | title = "Files Notifications API", 14 | version = "1.0", 15 | description = "Document Search Engine - Files Notifications API v1.0" 16 | ) 17 | ) 18 | @EnableBinding(AppStreamsConfig.class) 19 | public class FilesNotificationsServiceApplication { 20 | 21 | public static void main(String[] args) { 22 | SpringApplication.run(FilesNotificationsServiceApplication.class, args); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /microservices/files_notifications/src/main/java/com/dreamsoftware/documentsearchengine/config/AppStreamsConfig.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.config; 2 | 3 | import org.springframework.cloud.stream.annotation.Input; 4 | import org.springframework.messaging.MessageChannel; 5 | 6 | /** 7 | * 8 | * @author ssanchez 9 | */ 10 | public interface AppStreamsConfig { 11 | 12 | String PROCESSED_FILES_CHANNEL = "processed-files-state"; 13 | 14 | /** 15 | * Input Channel definition 16 | * 17 | * @return 18 | */ 19 | @Input(PROCESSED_FILES_CHANNEL) 20 | MessageChannel inputProcessedFilesNotifications(); 21 | } 22 | -------------------------------------------------------------------------------- /microservices/files_notifications/src/main/java/com/dreamsoftware/documentsearchengine/config/OpenApiConfig.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.config; 2 | 3 | import io.swagger.v3.oas.models.Components; 4 | import io.swagger.v3.oas.models.ExternalDocumentation; 5 | import io.swagger.v3.oas.models.OpenAPI; 6 | import io.swagger.v3.oas.models.info.Info; 7 | import io.swagger.v3.oas.models.info.License; 8 | import io.swagger.v3.oas.models.security.SecurityRequirement; 9 | import io.swagger.v3.oas.models.security.SecurityScheme; 10 | import java.util.Arrays; 11 | import org.springframework.context.annotation.Bean; 12 | import org.springframework.context.annotation.Configuration; 13 | 14 | /** 15 | * 16 | * @author ssanchez 17 | */ 18 | @Configuration 19 | public class OpenApiConfig { 20 | 21 | @Bean 22 | public OpenAPI springShopOpenAPI() { 23 | return new OpenAPI() 24 | .components(new Components().addSecuritySchemes("bearer-jwt", 25 | new SecurityScheme() 26 | .description("Auth Header with Json Web Token format") 27 | .type(SecurityScheme.Type.HTTP) 28 | .scheme("bearer") 29 | .bearerFormat("JWT") 30 | .in(SecurityScheme.In.HEADER).name("Authorization"))) 31 | .addSecurityItem( 32 | new SecurityRequirement().addList("bearer-jwt", Arrays.asList("read", "write"))) 33 | .info(new Info().title("Files Notifications REST API") 34 | .description("Provides a hypermedia-driven REST API for Apps") 35 | .version("v0.0.1-SNAPSHOT") 36 | .license(new License().name("Apache 2.0").url("http://springdoc.org"))) 37 | .externalDocs(new ExternalDocumentation() 38 | .description("Provides a hypermedia-driven REST API for Apps") 39 | .url("https://springshop.wiki.github.org/docs") 40 | ); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /microservices/files_notifications/src/main/java/com/dreamsoftware/documentsearchengine/config/WebSocketConfig.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.config; 2 | 3 | import javax.annotation.PostConstruct; 4 | import lombok.RequiredArgsConstructor; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.context.annotation.Configuration; 8 | import org.springframework.messaging.simp.config.MessageBrokerRegistry; 9 | import org.springframework.web.socket.config.annotation.EnableWebSocketMessageBroker; 10 | import org.springframework.web.socket.config.annotation.StompEndpointRegistry; 11 | import org.springframework.web.socket.config.annotation.WebSocketMessageBrokerConfigurer; 12 | import com.dreamsoftware.documentsearchengine.config.props.StompBrokerRelayProps; 13 | 14 | /** 15 | * 16 | * @author ssanchez 17 | */ 18 | @Configuration 19 | @EnableWebSocketMessageBroker 20 | @RequiredArgsConstructor 21 | public class WebSocketConfig implements WebSocketMessageBrokerConfigurer { 22 | 23 | private static final Logger logger = LoggerFactory.getLogger(WebSocketConfig.class); 24 | 25 | /** 26 | * Broker Relay Properties 27 | */ 28 | private final StompBrokerRelayProps brokerRelayProps; 29 | 30 | /** 31 | * Configure Message Broker 32 | * 33 | * @param registry 34 | */ 35 | @Override 36 | public void configureMessageBroker(MessageBrokerRegistry registry) { 37 | registry 38 | .enableStompBrokerRelay("/queue", "/topic") 39 | .setUserDestinationBroadcast(brokerRelayProps.getDestinationBroadcast()) 40 | .setUserRegistryBroadcast(brokerRelayProps.getRegistryBroadcast()) 41 | .setRelayHost(brokerRelayProps.getHost()) 42 | .setRelayPort(brokerRelayProps.getPort()) 43 | .setClientLogin(brokerRelayProps.getClientUsername()) 44 | .setClientPasscode(brokerRelayProps.getClientPasscode()) 45 | .setSystemLogin(brokerRelayProps.getClientUsername()) 46 | .setSystemPasscode(brokerRelayProps.getClientPasscode()); 47 | } 48 | 49 | /** 50 | * Register Endpoints 51 | * 52 | * @param registry 53 | */ 54 | @Override 55 | public void registerStompEndpoints(StompEndpointRegistry registry) { 56 | registry.addEndpoint("/notifications").setAllowedOrigins("*"); 57 | registry.addEndpoint("/notifications").setAllowedOrigins("*").withSockJS(); 58 | } 59 | 60 | @PostConstruct 61 | protected void onInit() { 62 | logger.debug("Stomp Host -> " + brokerRelayProps.getHost()); 63 | logger.debug("Stomp Port -> " + brokerRelayProps.getPort()); 64 | logger.debug("Stomp Client Username -> " + brokerRelayProps.getClientUsername()); 65 | logger.debug("Stomp Client Passcode -> " + brokerRelayProps.getClientPasscode()); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /microservices/files_notifications/src/main/java/com/dreamsoftware/documentsearchengine/config/props/StompBrokerRelayProps.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.config.props; 2 | 3 | import lombok.Data; 4 | import org.springframework.beans.factory.annotation.Value; 5 | import org.springframework.stereotype.Component; 6 | 7 | /** 8 | * 9 | * @author ssanchez 10 | */ 11 | @Data 12 | @Component 13 | public class StompBrokerRelayProps { 14 | 15 | /** 16 | * Broker Relay Destination Broadcast 17 | */ 18 | @Value("${stomp.destination.broadcast}") 19 | private String destinationBroadcast; 20 | 21 | /** 22 | * Broker Relay Registry Broadcast 23 | */ 24 | @Value("${stomp.registry.broadcast}") 25 | private String registryBroadcast; 26 | 27 | /** 28 | * Broker Relay Host 29 | */ 30 | @Value("${stomp.host}") 31 | private String host; 32 | 33 | /** 34 | * Broker Relay Port 35 | */ 36 | @Value("${stomp.port}") 37 | private int port; 38 | 39 | /** 40 | * Broker Relay Client Username 41 | */ 42 | @Value("${stomp.client.username}") 43 | private String clientUsername; 44 | 45 | /** 46 | * Broker Relay Client Passcode 47 | */ 48 | @Value("${stomp.client.passcode}") 49 | private String clientPasscode; 50 | 51 | } 52 | -------------------------------------------------------------------------------- /microservices/files_notifications/src/main/java/com/dreamsoftware/documentsearchengine/handler/ProcessedFilesNotificationsHandler.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.handler; 2 | 3 | import lombok.RequiredArgsConstructor; 4 | import lombok.extern.slf4j.Slf4j; 5 | import org.springframework.cloud.stream.annotation.StreamListener; 6 | import org.springframework.integration.IntegrationMessageHeaderAccessor; 7 | import org.springframework.kafka.support.Acknowledgment; 8 | import org.springframework.kafka.support.KafkaHeaders; 9 | import org.springframework.messaging.handler.annotation.Header; 10 | import org.springframework.messaging.handler.annotation.Payload; 11 | import org.springframework.stereotype.Component; 12 | import com.dreamsoftware.documentsearchengine.web.dto.ProcessedFileNotificationDTO; 13 | import com.dreamsoftware.documentsearchengine.config.AppStreamsConfig; 14 | import com.dreamsoftware.documentsearchengine.service.IProcessedFileNotificationService; 15 | 16 | /** 17 | * 18 | * @author ssanchez 19 | */ 20 | @Slf4j 21 | @RequiredArgsConstructor 22 | @Component 23 | public class ProcessedFilesNotificationsHandler { 24 | 25 | private final IProcessedFileNotificationService processedFileNotificationService; 26 | 27 | /** 28 | * 29 | * @param processedFileNotification 30 | * @param topic 31 | * @param partition 32 | * @param offset 33 | * @param acknowledgment 34 | * @param deliveryAttempt 35 | */ 36 | @StreamListener(AppStreamsConfig.PROCESSED_FILES_CHANNEL) 37 | public void onNewProcessedFileNotification( 38 | @Payload final ProcessedFileNotificationDTO processedFileNotification, 39 | @Header(KafkaHeaders.RECEIVED_TOPIC) String topic, 40 | @Header(KafkaHeaders.RECEIVED_PARTITION_ID) Integer partition, 41 | @Header(KafkaHeaders.OFFSET) Long offset, 42 | @Header(KafkaHeaders.ACKNOWLEDGMENT) Acknowledgment acknowledgment, 43 | @Header(IntegrationMessageHeaderAccessor.DELIVERY_ATTEMPT) Integer deliveryAttempt) { 44 | 45 | log.info("onNewProcessedFileNotification for file '{}' with state '{}' received from bus. topic: {}, partition: {}, offset: {}, deliveryAttempt: {}", 46 | processedFileNotification.getFilename(), processedFileNotification.getFileState(), topic, partition, offset, deliveryAttempt); 47 | 48 | try { 49 | processedFileNotificationService.save(processedFileNotification); 50 | // commit offset 51 | if (acknowledgment != null) { 52 | acknowledgment.acknowledge(); 53 | log.info("Commit Offsets ..."); 54 | } 55 | } catch (final Exception ex) { 56 | log.error("Collect Processed Files Notifications Exception -> " + ex.getMessage()); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /microservices/files_notifications/src/main/java/com/dreamsoftware/documentsearchengine/mapper/ProcessedFileNotificationMapper.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.mapper; 2 | 3 | import com.dreamsoftware.documentsearchengine.persistence.entity.ProcessedFileNotificationEntity; 4 | import com.dreamsoftware.documentsearchengine.web.dto.ProcessedFileNotificationDTO; 5 | import java.util.List; 6 | import org.mapstruct.IterableMapping; 7 | import org.mapstruct.Mapper; 8 | import org.mapstruct.Mappings; 9 | import org.mapstruct.Named; 10 | 11 | /** 12 | * 13 | * @author ssanchez 14 | */ 15 | @Mapper(unmappedTargetPolicy = org.mapstruct.ReportingPolicy.IGNORE) 16 | public abstract class ProcessedFileNotificationMapper { 17 | 18 | /** 19 | * 20 | * @param processedFileNotificationEntity 21 | * @return 22 | */ 23 | @Mappings({}) 24 | @Named("entityToDTO") 25 | public abstract ProcessedFileNotificationDTO entityToDTO(ProcessedFileNotificationEntity processedFileNotificationEntity); 26 | 27 | /** 28 | * 29 | * @param processedFileNotificationDTO 30 | * @return 31 | */ 32 | @Mappings({}) 33 | @Named("DTOtoEntity") 34 | public abstract ProcessedFileNotificationEntity dtoToEntity(ProcessedFileNotificationDTO processedFileNotificationDTO); 35 | 36 | /** 37 | * 38 | * @param processedFileNotificationEntityList 39 | * @return 40 | */ 41 | @IterableMapping(qualifiedByName = "entityToDTO") 42 | public abstract List entityToDTO(List processedFileNotificationEntityList); 43 | } 44 | -------------------------------------------------------------------------------- /microservices/files_notifications/src/main/java/com/dreamsoftware/documentsearchengine/persistence/entity/ProcessedFileNotificationEntity.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.persistence.entity; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.EqualsAndHashCode; 6 | import lombok.NoArgsConstructor; 7 | import org.bson.types.ObjectId; 8 | import org.springframework.data.annotation.Id; 9 | import org.springframework.data.mongodb.core.mapping.Document; 10 | import org.springframework.data.mongodb.core.mapping.Field; 11 | 12 | /** 13 | * 14 | * @author ssanchez 15 | */ 16 | @Data 17 | @EqualsAndHashCode(callSuper = false) 18 | @AllArgsConstructor 19 | @NoArgsConstructor 20 | @Document(collection = ProcessedFileNotificationEntity.COLLECTION_NAME) 21 | public class ProcessedFileNotificationEntity { 22 | 23 | public final static String COLLECTION_NAME = "processed_files_notification"; 24 | 25 | /** 26 | * Id 27 | */ 28 | @Id 29 | private ObjectId id; 30 | 31 | /** 32 | * File Name 33 | */ 34 | @Field("file_name") 35 | private String filename; 36 | 37 | /** 38 | * File Last Modified Time 39 | */ 40 | @Field("file_last_modified_time") 41 | private String fileLastModifiedTime; 42 | 43 | /** 44 | * File Mime Extension 45 | */ 46 | @Field("file_mime_extension") 47 | private String fileMimeExtension; 48 | 49 | /** 50 | * File Mime Type 51 | */ 52 | @Field("file_mime_type") 53 | private String fileMimeType; 54 | 55 | /** 56 | * File HDFS Path 57 | */ 58 | @Field("file_hdfs_path") 59 | private String fileHdfsPath; 60 | 61 | /** 62 | * File State 63 | */ 64 | @Field("file_state") 65 | private String fileState; 66 | 67 | } 68 | -------------------------------------------------------------------------------- /microservices/files_notifications/src/main/java/com/dreamsoftware/documentsearchengine/persistence/repository/ProcessedFileNotificationRepository.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.persistence.repository; 2 | 3 | import com.dreamsoftware.documentsearchengine.persistence.entity.ProcessedFileNotificationEntity; 4 | import org.bson.types.ObjectId; 5 | import org.springframework.data.mongodb.repository.MongoRepository; 6 | import org.springframework.stereotype.Repository; 7 | 8 | /** 9 | * 10 | * @author ssanchez 11 | */ 12 | @Repository 13 | public interface ProcessedFileNotificationRepository extends MongoRepository { 14 | 15 | } 16 | -------------------------------------------------------------------------------- /microservices/files_notifications/src/main/java/com/dreamsoftware/documentsearchengine/service/IProcessedFileNotificationService.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.service; 2 | 3 | import com.dreamsoftware.documentsearchengine.web.dto.ProcessedFileNotificationDTO; 4 | import java.util.Optional; 5 | import org.bson.types.ObjectId; 6 | import org.springframework.data.domain.Page; 7 | import org.springframework.data.domain.Pageable; 8 | 9 | /** 10 | * 11 | * @author ssanchez 12 | */ 13 | public interface IProcessedFileNotificationService { 14 | 15 | /** 16 | * 17 | * @param page 18 | * @param size 19 | * @return 20 | */ 21 | Page findPaginated(final Integer page, final Integer size); 22 | 23 | /** 24 | * 25 | * @param pageable 26 | * @return 27 | */ 28 | Page findPaginated(final Pageable pageable); 29 | 30 | /** 31 | * 32 | * @param id 33 | * @return 34 | */ 35 | Optional findById(final ObjectId id); 36 | 37 | /** 38 | * 39 | * @param processedFileNotification 40 | */ 41 | void save(final ProcessedFileNotificationDTO processedFileNotification); 42 | 43 | } 44 | -------------------------------------------------------------------------------- /microservices/files_notifications/src/main/java/com/dreamsoftware/documentsearchengine/service/impl/ProcessedFileNotificationServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.service.impl; 2 | 3 | import com.dreamsoftware.documentsearchengine.mapper.ProcessedFileNotificationMapper; 4 | import com.dreamsoftware.documentsearchengine.persistence.repository.ProcessedFileNotificationRepository; 5 | import com.dreamsoftware.documentsearchengine.service.IProcessedFileNotificationService; 6 | import com.dreamsoftware.documentsearchengine.web.dto.ProcessedFileNotificationDTO; 7 | import java.util.Optional; 8 | import lombok.RequiredArgsConstructor; 9 | import org.bson.types.ObjectId; 10 | import org.springframework.data.domain.Page; 11 | import org.springframework.data.domain.PageRequest; 12 | import org.springframework.data.domain.Pageable; 13 | import org.springframework.stereotype.Service; 14 | import org.springframework.util.Assert; 15 | import com.dreamsoftware.documentsearchengine.persistence.entity.ProcessedFileNotificationEntity; 16 | 17 | /** 18 | * 19 | * @author ssanchez 20 | */ 21 | @Service("processedFileService") 22 | @RequiredArgsConstructor 23 | public class ProcessedFileNotificationServiceImpl implements IProcessedFileNotificationService { 24 | 25 | private final ProcessedFileNotificationMapper processedFileNotificationMapper; 26 | private final ProcessedFileNotificationRepository processedFileNotificationRepository; 27 | 28 | /** 29 | * 30 | * @param page 31 | * @param size 32 | * @return 33 | */ 34 | @Override 35 | public Page findPaginated(final Integer page, final Integer size) { 36 | Assert.notNull(page, "Page can not be null"); 37 | Assert.notNull(size, "Size can not be null"); 38 | return findPaginated(PageRequest.of(page, size)); 39 | } 40 | 41 | /** 42 | * 43 | * @param pageable 44 | * @return 45 | */ 46 | @Override 47 | public Page findPaginated(final Pageable pageable) { 48 | Assert.notNull(pageable, "Pageable can not be null"); 49 | 50 | return processedFileNotificationRepository.findAll(pageable) 51 | .map(processedFileNotificationMapper::entityToDTO); 52 | } 53 | 54 | /** 55 | * 56 | * @param id 57 | * @return 58 | */ 59 | @Override 60 | public Optional findById(ObjectId id) { 61 | Assert.notNull(id, "File id can not be null"); 62 | 63 | return processedFileNotificationRepository.findById(id) 64 | .map(processedFileNotificationMapper::entityToDTO); 65 | } 66 | 67 | /** 68 | * 69 | * @param processedFileNotification 70 | */ 71 | @Override 72 | public void save(final ProcessedFileNotificationDTO processedFileNotification) { 73 | Assert.notNull(processedFileNotification, "Processed File Notification can not be null"); 74 | 75 | final ProcessedFileNotificationEntity processedFileNotificationToSave 76 | = processedFileNotificationMapper.dtoToEntity(processedFileNotification); 77 | 78 | processedFileNotificationRepository.save(processedFileNotificationToSave); 79 | 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /microservices/files_notifications/src/main/java/com/dreamsoftware/documentsearchengine/web/controller/FilesNotificationsController.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.controller; 2 | 3 | import com.dreamsoftware.documentsearchengine.service.IProcessedFileNotificationService; 4 | import com.dreamsoftware.documentsearchengine.web.controller.error.exception.NoNotificationsFoundException; 5 | import com.dreamsoftware.documentsearchengine.web.core.APIResponse; 6 | import com.dreamsoftware.documentsearchengine.web.core.ErrorResponseDTO; 7 | import com.dreamsoftware.documentsearchengine.web.core.SupportController; 8 | import com.dreamsoftware.documentsearchengine.web.dto.ProcessedFileNotificationDTO; 9 | import io.swagger.v3.oas.annotations.Operation; 10 | import io.swagger.v3.oas.annotations.media.Content; 11 | import io.swagger.v3.oas.annotations.media.Schema; 12 | import io.swagger.v3.oas.annotations.responses.ApiResponse; 13 | import io.swagger.v3.oas.annotations.responses.ApiResponses; 14 | import io.swagger.v3.oas.annotations.tags.Tag; 15 | import lombok.RequiredArgsConstructor; 16 | import org.springframework.data.domain.Page; 17 | import org.springframework.http.HttpStatus; 18 | import org.springframework.http.MediaType; 19 | import org.springframework.http.ResponseEntity; 20 | import org.springframework.security.access.prepost.PreAuthorize; 21 | import org.springframework.validation.annotation.Validated; 22 | import org.springframework.web.bind.annotation.RequestMapping; 23 | import org.springframework.web.bind.annotation.RequestMethod; 24 | import org.springframework.web.bind.annotation.RequestParam; 25 | import org.springframework.web.bind.annotation.RestController; 26 | 27 | /** 28 | * Files Metadata Controller 29 | * 30 | * @author ssanchez 31 | */ 32 | @RestController 33 | @Validated 34 | @RequestMapping("/api/v1/notifications/") 35 | @Tag(name = "files_notifications", description = "/api/v1/notifications/ (Code Response interval -> 4XX)") 36 | @RequiredArgsConstructor 37 | public class FilesNotificationsController extends SupportController { 38 | 39 | private final IProcessedFileNotificationService processedFileNotificationService; 40 | 41 | /** 42 | * 43 | * @param page 44 | * @param size 45 | * @return 46 | * @throws Throwable 47 | */ 48 | @Operation(summary = "GET_NOTIFICATIONS - GetProcessed files notifications", description = " Get Notifications", tags = {"files_notifications"}) 49 | @ApiResponses(value = { 50 | @ApiResponse(responseCode = "200", description = "Notifications List", 51 | content = @Content( 52 | schema = @Schema(implementation = Page.class))), 53 | @ApiResponse(responseCode = "404", description = "No Notifications found", 54 | content = @Content( 55 | schema = @Schema(implementation = ErrorResponseDTO.class))) 56 | }) 57 | @PreAuthorize("hasAuthority('SCOPE_FILES_NOTIFICATIONS')") 58 | @RequestMapping(value = "/", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) 59 | public ResponseEntity>> getNotifications( 60 | @RequestParam(name = "page", required = false, defaultValue = "0") final Integer page, 61 | @RequestParam(name = "size", required = false, defaultValue = "20") final Integer size) throws Throwable { 62 | 63 | final Page filesProcessedPage = processedFileNotificationService.findPaginated(page, size); 64 | 65 | if (!filesProcessedPage.hasContent()) { 66 | throw new NoNotificationsFoundException(); 67 | } 68 | 69 | return responseHelper.createAndSendResponse( 70 | FilesNotificationsResponseCodeEnum.GET_NOTIFICATIONS, 71 | HttpStatus.OK, filesProcessedPage); 72 | 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /microservices/files_notifications/src/main/java/com/dreamsoftware/documentsearchengine/web/controller/FilesNotificationsResponseCodeEnum.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.controller; 2 | 3 | import com.dreamsoftware.documentsearchengine.web.core.IResponseCodeTypes; 4 | 5 | /** 6 | * 7 | * @author ssanchez 8 | */ 9 | public enum FilesNotificationsResponseCodeEnum implements IResponseCodeTypes { 10 | 11 | GET_NOTIFICATIONS(400L), 12 | NO_NOTIFICATIONS_FOUND(401L); 13 | 14 | private final Long code; 15 | 16 | public static final String CATEGORY_NAME = "FILES_NOTIFICATIONS"; 17 | 18 | private FilesNotificationsResponseCodeEnum(Long code) { 19 | this.code = code; 20 | } 21 | 22 | @Override 23 | public Long getResponseCode() { 24 | return code; 25 | } 26 | 27 | @Override 28 | public String getCategoryName() { 29 | return CATEGORY_NAME; 30 | } 31 | 32 | @Override 33 | public String getCodeName() { 34 | return name(); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /microservices/files_notifications/src/main/java/com/dreamsoftware/documentsearchengine/web/controller/error/FilesNotificationsErrorController.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.controller.error; 2 | 3 | import com.dreamsoftware.documentsearchengine.web.controller.FilesNotificationsResponseCodeEnum; 4 | import com.dreamsoftware.documentsearchengine.web.controller.error.exception.NoNotificationsFoundException; 5 | import com.dreamsoftware.documentsearchengine.web.core.APIResponse; 6 | import com.dreamsoftware.documentsearchengine.web.core.ErrorResponseDTO; 7 | import com.dreamsoftware.documentsearchengine.web.core.SupportController; 8 | import javax.servlet.http.HttpServletRequest; 9 | import org.springframework.core.Ordered; 10 | import org.springframework.core.annotation.Order; 11 | import org.springframework.http.HttpStatus; 12 | import org.springframework.http.ResponseEntity; 13 | import org.springframework.web.bind.annotation.ControllerAdvice; 14 | import org.springframework.web.bind.annotation.ExceptionHandler; 15 | import org.springframework.web.bind.annotation.ResponseBody; 16 | 17 | /** 18 | * 19 | * @author ssanchez 20 | */ 21 | @ControllerAdvice 22 | @Order(Ordered.HIGHEST_PRECEDENCE) 23 | public class FilesNotificationsErrorController extends SupportController { 24 | 25 | /** 26 | * 27 | * @param ex 28 | * @param request 29 | * @return 30 | */ 31 | @ExceptionHandler(NoNotificationsFoundException.class) 32 | @ResponseBody 33 | protected ResponseEntity> handleNoFilesProcessedFoundException(NoNotificationsFoundException ex, HttpServletRequest request) { 34 | return responseHelper.createAndSendErrorResponse(FilesNotificationsResponseCodeEnum.NO_NOTIFICATIONS_FOUND, 35 | HttpStatus.NOT_FOUND, "No Notifications found."); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /microservices/files_notifications/src/main/java/com/dreamsoftware/documentsearchengine/web/controller/error/exception/NoNotificationsFoundException.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.controller.error.exception; 2 | 3 | /** 4 | * 5 | * @author ssanchez 6 | */ 7 | public class NoNotificationsFoundException extends RuntimeException { 8 | } 9 | -------------------------------------------------------------------------------- /microservices/files_notifications/src/main/java/com/dreamsoftware/documentsearchengine/web/dto/ProcessedFileNotificationDTO.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.dto; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Data; 6 | import lombok.EqualsAndHashCode; 7 | import lombok.NoArgsConstructor; 8 | 9 | /** 10 | * 11 | * @author ssanchez 12 | */ 13 | @Data 14 | @EqualsAndHashCode(callSuper = false) 15 | @AllArgsConstructor 16 | @NoArgsConstructor 17 | public class ProcessedFileNotificationDTO { 18 | 19 | /** 20 | * File Name 21 | */ 22 | @JsonProperty("file_name") 23 | private String filename; 24 | 25 | /** 26 | * File Last Modified Time 27 | */ 28 | @JsonProperty("file_last_modified_time") 29 | private String fileLastModifiedTime; 30 | 31 | /** 32 | * File Mime Extension 33 | */ 34 | @JsonProperty("file_mime_extension") 35 | private String fileMimeExtension; 36 | 37 | /** 38 | * File Mime Type 39 | */ 40 | @JsonProperty("file_mime_type") 41 | private String fileMimeType; 42 | 43 | /** 44 | * File HDFS Path 45 | */ 46 | @JsonProperty("file_hdfs_path") 47 | private String fileHdfsPath; 48 | 49 | /** 50 | * File State 51 | */ 52 | @JsonProperty("file_state") 53 | private String fileState; 54 | 55 | } 56 | -------------------------------------------------------------------------------- /microservices/files_notifications/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | application: 3 | name: files-notifications-service 4 | security: 5 | oauth2: 6 | resourceserver: 7 | jwt: 8 | issuer-uri: http://keycloak:8080/auth/realms/document_search_engine 9 | cloud: 10 | consul: 11 | host: consul-server 12 | port: 8500 13 | discovery: 14 | port: 8080 15 | prefer-ip-address: true 16 | healthCheckPath: /actuator/health 17 | stream: 18 | kafka: 19 | binder: 20 | brokers: kafka:9092 21 | requiredAcks: 1 22 | auto-create-topics: false 23 | configuration: 24 | auto: 25 | offset.reset: latest 26 | bindings: 27 | processed-files-state: 28 | consumer: 29 | autoCommitOffset: false 30 | bindings: 31 | processed-files-state: 32 | group: notifications-collector 33 | data: 34 | mongodb: 35 | authentication-database: admin 36 | database: files 37 | host: mongo 38 | port: 27017 39 | username: devroot 40 | password: devroot 41 | 42 | ## Spring Open API 43 | springdoc: 44 | swagger-ui: 45 | path: /swagger-ui.html 46 | api-docs: 47 | path: /v3/api-docs 48 | 49 | ## Stomp Configuration 50 | stomp: 51 | destination.broadcast: /topic/unresolved.user.dest 52 | registry.broadcast: /topic/registry.broadcast 53 | host: rabbitmq-stomp 54 | port: 61613 55 | client: 56 | username: guest 57 | passcode: guest 58 | 59 | logging.level: 60 | org.springframework.cloud.gateway: DEBUG 61 | org.springframework.security: DEBUG 62 | com.dreamsoftware.documentsearchengine: DEBUG 63 | org.springframework.web.reactive.function.client: TRACE 64 | 65 | server.port: 8080 -------------------------------------------------------------------------------- /microservices/files_search/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-jdk-alpine 2 | # Add Maintainer Info 3 | LABEL description="Document Search Engine Files Search Service" 4 | # Args for image 5 | ARG PORT=8080 6 | 7 | RUN apk update && apk upgrade 8 | RUN ln -s /bin/bash /usr/bin 9 | RUN mkdir -p /usr/src/app 10 | WORKDIR /usr/src/app 11 | 12 | 13 | COPY resources/wait-for-it.sh wait-for-it.sh 14 | COPY target/file_search_service.jar app.jar 15 | 16 | RUN dos2unix wait-for-it.sh 17 | RUN chmod +x wait-for-it.sh 18 | RUN uname -a 19 | RUN pwd 20 | RUN ls -al 21 | 22 | EXPOSE ${PORT} 23 | 24 | CMD ["sh", "-c", "echo 'waiting for 300 seconds for consul-agent:8500 to be accessable before starting application' && ./wait-for-it.sh -t 300 consul:8500 -- java -jar app.jar"] -------------------------------------------------------------------------------- /microservices/files_search/nbactions.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | run 5 | 6 | jar 7 | 8 | 9 | process-classes 10 | org.codehaus.mojo:exec-maven-plugin:1.5.0:exec 11 | 12 | 13 | -classpath %classpath com.dreamsoftware.documentsearchengine.FilesSearchServiceApplication 14 | java 15 | 16 | 17 | 18 | debug 19 | 20 | jar 21 | 22 | 23 | process-classes 24 | org.codehaus.mojo:exec-maven-plugin:1.5.0:exec 25 | 26 | 27 | -agentlib:jdwp=transport=dt_socket,server=n,address=${jpda.address} -classpath %classpath com.dreamsoftware.documentsearchengine.FilesSearchServiceApplication 28 | java 29 | true 30 | 31 | 32 | 33 | profile 34 | 35 | jar 36 | 37 | 38 | process-classes 39 | org.codehaus.mojo:exec-maven-plugin:1.5.0:exec 40 | 41 | 42 | -classpath %classpath com.dreamsoftware.documentsearchengine.FilesSearchServiceApplication 43 | java 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /microservices/files_search/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 4.0.0 7 | com.dreamsoftware 8 | document_search_engine_search 9 | 0.0.1-SNAPSHOT 10 | jar 11 | file_search_service 12 | Document Search Engine Files Search Service 13 | 14 | 15 | com.dreamsoftware 16 | document_search_engine 17 | 0.0.1-SNAPSHOT 18 | ../ 19 | 20 | 21 | 22 | 23 | ${project.groupId} 24 | document_search_engine_files_commons 25 | ${project.version} 26 | 27 | 28 | 29 | org.springframework.boot 30 | spring-boot-starter-data-elasticsearch 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | org.springframework.boot 40 | spring-boot-maven-plugin 41 | 42 | file_search_service 43 | 44 | 45 | 46 | 47 | com.spotify 48 | dockerfile-maven-plugin 49 | 1.4.13 50 | 51 | 52 | build-image 53 | package 54 | 55 | build 56 | 57 | 58 | 59 | 60 | 61 | 62 | ssanchez11/file_search_service 63 | ${project.version} 64 | 65 | file_search_service 66 | 67 | false 68 | true 69 | 70 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /microservices/files_search/resources/wait-for-it.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | TIMEOUT=15 4 | QUIET=0 5 | 6 | echoerr() { 7 | if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi 8 | } 9 | 10 | usage() { 11 | exitcode="$1" 12 | cat << USAGE >&2 13 | Usage: 14 | $cmdname host:port [-t timeout] [-- command args] 15 | -q | --quiet Do not output any status messages 16 | -t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout 17 | -- COMMAND ARGS Execute command with args after the test finishes 18 | USAGE 19 | exit "$exitcode" 20 | } 21 | 22 | wait_for() { 23 | for i in `seq $TIMEOUT` ; do 24 | nc -z "$HOST" "$PORT" > /dev/null 2>&1 25 | 26 | result=$? 27 | if [ $result -eq 0 ] ; then 28 | if [ $# -gt 0 ] ; then 29 | exec "$@" 30 | fi 31 | exit 0 32 | fi 33 | sleep 1 34 | done 35 | echo "Operation timed out" >&2 36 | exit 1 37 | } 38 | 39 | while [ $# -gt 0 ] 40 | do 41 | case "$1" in 42 | *:* ) 43 | HOST=$(printf "%s\n" "$1"| cut -d : -f 1) 44 | PORT=$(printf "%s\n" "$1"| cut -d : -f 2) 45 | shift 1 46 | ;; 47 | -q | --quiet) 48 | QUIET=1 49 | shift 1 50 | ;; 51 | -t) 52 | TIMEOUT="$2" 53 | if [ "$TIMEOUT" = "" ]; then break; fi 54 | shift 2 55 | ;; 56 | --timeout=*) 57 | TIMEOUT="${1#*=}" 58 | shift 1 59 | ;; 60 | --) 61 | shift 62 | break 63 | ;; 64 | --help) 65 | usage 0 66 | ;; 67 | *) 68 | echoerr "Unknown argument: $1" 69 | usage 1 70 | ;; 71 | esac 72 | done 73 | 74 | if [ "$HOST" = "" -o "$PORT" = "" ]; then 75 | echoerr "Error: you need to provide a host and port to test." 76 | usage 2 77 | fi 78 | 79 | wait_for "$@" -------------------------------------------------------------------------------- /microservices/files_search/src/main/java/com/dreamsoftware/documentsearchengine/FilesSearchServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class FilesSearchServiceApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(FilesSearchServiceApplication.class, args); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /microservices/files_search/src/main/java/com/dreamsoftware/documentsearchengine/config/ElasticSearchClientConfig.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.config; 2 | 3 | import java.security.KeyStore; 4 | import java.security.cert.CertificateException; 5 | import java.security.cert.X509Certificate; 6 | import java.time.Duration; 7 | import javax.net.ssl.KeyManagerFactory; 8 | import javax.net.ssl.SSLContext; 9 | import javax.net.ssl.SSLSession; 10 | import javax.net.ssl.TrustManager; 11 | import javax.net.ssl.X509TrustManager; 12 | import org.elasticsearch.client.RestHighLevelClient; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | import org.springframework.beans.factory.annotation.Value; 16 | import org.springframework.context.annotation.Configuration; 17 | import org.springframework.data.elasticsearch.client.ClientConfiguration; 18 | import org.springframework.data.elasticsearch.client.RestClients; 19 | import org.springframework.data.elasticsearch.config.AbstractElasticsearchConfiguration; 20 | import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; 21 | 22 | /** 23 | * 24 | * @author ssanchez 25 | */ 26 | @Configuration 27 | @EnableElasticsearchRepositories 28 | public class ElasticSearchClientConfig extends AbstractElasticsearchConfiguration { 29 | 30 | private static final Logger logger = LoggerFactory.getLogger(ElasticSearchClientConfig.class); 31 | 32 | @Value("${elasticsearch.host}") 33 | private String host; 34 | 35 | @Value("${elasticsearch.cert.file}") 36 | private String certFile; 37 | 38 | @Value("${elasticsearch.cert.password}") 39 | private String certPassword; 40 | 41 | @Value("${elasticsearch.user.name}") 42 | private String userName; 43 | 44 | @Value("${elasticsearch.user.pass}") 45 | private String userPassword; 46 | 47 | /** 48 | * Provide the elasticsearch REST client 49 | * 50 | * @return 51 | */ 52 | @Override 53 | public RestHighLevelClient elasticsearchClient() { 54 | final ClientConfiguration clientConfiguration = ClientConfiguration.builder() 55 | .connectedTo(host) // set the address of the Elasticsearch cluster 56 | .usingSsl(createSSLContext(), (String s, SSLSession sslSession) -> true) // use the SSLContext with the client cert 57 | .withConnectTimeout(Duration.ofSeconds(5)) 58 | .withSocketTimeout(Duration.ofSeconds(3)) 59 | .withBasicAuth(userName, userPassword) // use the headers for authentication 60 | .build(); 61 | return RestClients.create(clientConfiguration).rest(); 62 | } 63 | 64 | private SSLContext createSSLContext() { 65 | try { 66 | 67 | // Init PKCS12 Key Store 68 | KeyStore ks = KeyStore.getInstance("PKCS12"); 69 | ks.load(getClass().getClassLoader().getResourceAsStream(certFile), 70 | certPassword.toCharArray()); 71 | 72 | KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); 73 | keyManagerFactory.init(ks, certPassword.toCharArray()); 74 | 75 | TrustManager tm = new X509TrustManager() { 76 | @Override 77 | public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { 78 | } 79 | 80 | @Override 81 | public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { 82 | } 83 | 84 | @Override 85 | public X509Certificate[] getAcceptedIssuers() { 86 | return null; 87 | } 88 | }; 89 | 90 | SSLContext sslContext = SSLContext.getInstance("TLS"); 91 | sslContext.init(keyManagerFactory.getKeyManagers(), new TrustManager[]{tm}, null); 92 | 93 | return sslContext; 94 | 95 | } catch (Exception e) { 96 | logger.debug("cannot create SSLContext -> " + e.getMessage()); 97 | } 98 | return null; 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /microservices/files_search/src/main/java/com/dreamsoftware/documentsearchengine/config/OpenApiConfig.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.config; 2 | 3 | import io.swagger.v3.oas.models.Components; 4 | import io.swagger.v3.oas.models.ExternalDocumentation; 5 | import io.swagger.v3.oas.models.OpenAPI; 6 | import io.swagger.v3.oas.models.info.Info; 7 | import io.swagger.v3.oas.models.info.License; 8 | import io.swagger.v3.oas.models.security.SecurityRequirement; 9 | import io.swagger.v3.oas.models.security.SecurityScheme; 10 | import java.util.Arrays; 11 | import org.springframework.context.annotation.Bean; 12 | import org.springframework.context.annotation.Configuration; 13 | 14 | /** 15 | * 16 | * @author ssanchez 17 | */ 18 | @Configuration 19 | public class OpenApiConfig { 20 | 21 | @Bean 22 | public OpenAPI springShopOpenAPI() { 23 | return new OpenAPI() 24 | .components(new Components().addSecuritySchemes("bearer-jwt", 25 | new SecurityScheme() 26 | .description("Auth Header with Json Web Token format") 27 | .type(SecurityScheme.Type.HTTP) 28 | .scheme("bearer") 29 | .bearerFormat("JWT") 30 | .in(SecurityScheme.In.HEADER).name("Authorization"))) 31 | .addSecurityItem( 32 | new SecurityRequirement().addList("bearer-jwt", Arrays.asList("read", "write"))) 33 | .info(new Info().title("Files Search REST API") 34 | .description("Provides a hypermedia-driven REST API for Apps") 35 | .version("v0.0.1-SNAPSHOT") 36 | .license(new License().name("Apache 2.0").url("http://springdoc.org"))) 37 | .externalDocs(new ExternalDocumentation() 38 | .description("Provides a hypermedia-driven REST API for Apps") 39 | .url("https://springshop.wiki.github.org/docs") 40 | ); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /microservices/files_search/src/main/java/com/dreamsoftware/documentsearchengine/mapper/ProcessedFileMapper.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.mapper; 2 | 3 | import com.dreamsoftware.documentsearchengine.persistance.entity.ProcessedFileEntity; 4 | import com.dreamsoftware.documentsearchengine.web.dto.ProcessedFileDTO; 5 | import java.io.UnsupportedEncodingException; 6 | import java.net.URLEncoder; 7 | import java.nio.charset.StandardCharsets; 8 | import java.util.List; 9 | import org.mapstruct.IterableMapping; 10 | import org.mapstruct.Mapper; 11 | import org.mapstruct.Mapping; 12 | import org.mapstruct.Mappings; 13 | import org.mapstruct.Named; 14 | 15 | /** 16 | * 17 | * @author ssanchez 18 | */ 19 | @Mapper(unmappedTargetPolicy = org.mapstruct.ReportingPolicy.IGNORE) 20 | public abstract class ProcessedFileMapper { 21 | 22 | /** 23 | * 24 | * @param processedFileEntity 25 | * @return 26 | */ 27 | @Mappings({ 28 | @Mapping(expression = "java(createWebhdfsUrlForFilename(processedFileEntity))", target = "url") 29 | }) 30 | @Named("entityToDTO") 31 | public abstract ProcessedFileDTO entityToDTO(ProcessedFileEntity processedFileEntity); 32 | 33 | /** 34 | * 35 | * @param processedFileEntityList 36 | * @return 37 | */ 38 | @IterableMapping(qualifiedByName = "entityToDTO") 39 | public abstract List entityToDTO(List processedFileEntityList); 40 | 41 | /** 42 | * 43 | * @param processedFileEntity 44 | * @return 45 | */ 46 | protected String createWebhdfsUrlForFilename(final ProcessedFileEntity processedFileEntity) { 47 | try { 48 | return String.format("http://namenode:9870/webhdfs/v1/uploads/%s?op=OPEN", URLEncoder.encode(processedFileEntity.getName().replace("_", "."), "UTF-8").replace("+", "%20")); 49 | } catch (UnsupportedEncodingException ex) { 50 | throw new RuntimeException(ex); 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /microservices/files_search/src/main/java/com/dreamsoftware/documentsearchengine/persistance/entity/ProcessedFileEntity.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.persistance.entity; 2 | 3 | import static com.dreamsoftware.documentsearchengine.persistance.entity.ProcessedFileEntity.INDEX_NAME; 4 | import java.util.Date; 5 | import lombok.AllArgsConstructor; 6 | import lombok.Data; 7 | import lombok.EqualsAndHashCode; 8 | import lombok.NoArgsConstructor; 9 | import org.springframework.data.annotation.Id; 10 | import org.springframework.data.elasticsearch.annotations.DateFormat; 11 | import org.springframework.data.elasticsearch.annotations.Document; 12 | import org.springframework.data.elasticsearch.annotations.Field; 13 | import org.springframework.data.elasticsearch.annotations.FieldType; 14 | 15 | /** 16 | * 17 | * @author ssanchez 18 | */ 19 | @Data 20 | @EqualsAndHashCode(callSuper = false) 21 | @AllArgsConstructor 22 | @NoArgsConstructor 23 | @Document(indexName = INDEX_NAME, createIndex = false) 24 | public class ProcessedFileEntity { 25 | 26 | public final static String INDEX_NAME = "processed_files"; 27 | 28 | /** 29 | * Document Id 30 | */ 31 | @Id 32 | private String id; 33 | 34 | /** 35 | * Document Version 36 | */ 37 | @Field(name = "@version") 38 | private Integer version; 39 | 40 | /** 41 | * File Created At 42 | */ 43 | @Field(name = "document.metadata.created", type = FieldType.Date, format = DateFormat.date_time_no_millis) 44 | private Date createdAt; 45 | 46 | /** 47 | * Processed At 48 | */ 49 | /*@Field(name = "document.document.processed_at", type = FieldType.Date, 50 | format = DateFormat.custom, pattern = "EEE MMM d HH:mm:ss z yyyy") 51 | private Date processedAt;*/ 52 | /** 53 | * File Language 54 | */ 55 | @Field(name = "document.metadata.language", type = FieldType.Text) 56 | private String language; 57 | 58 | /** 59 | * File Author 60 | */ 61 | @Field(name = "document.metadata.author", type = FieldType.Text) 62 | private String author; 63 | 64 | /** 65 | * File Creator 66 | */ 67 | @Field(name = "document.metadata.creator", type = FieldType.Text) 68 | private String creator; 69 | 70 | /** 71 | * File Producer 72 | */ 73 | @Field(name = "document.metadata.producer", type = FieldType.Text) 74 | private String producer; 75 | 76 | /** 77 | * File UUID 78 | */ 79 | @Field(name = "document.attrs.uuid", type = FieldType.Text) 80 | private String uuid; 81 | 82 | /** 83 | * File Name 84 | */ 85 | @Field(name = "document.attrs.filename", type = FieldType.Text) 86 | private String name; 87 | 88 | /** 89 | * File Permissions 90 | */ 91 | @Field(name = "document.attrs.file_permissions", type = FieldType.Text) 92 | private String permissions; 93 | 94 | /** 95 | * File Mime Type 96 | */ 97 | @Field(name = "document.attrs.mime_type", type = FieldType.Text) 98 | private String mimeType; 99 | 100 | /** 101 | * File Mime Extension 102 | */ 103 | @Field(name = "document.attrs.mime_extension", type = FieldType.Text) 104 | private String mimeExtension; 105 | 106 | /** 107 | * File Last Modified Time 108 | */ 109 | @Field(name = "document.attrs.file_lastModifiedTime", type = FieldType.Date, format = DateFormat.date_time_no_millis) 110 | private Date lastModifiedTime; 111 | 112 | } 113 | -------------------------------------------------------------------------------- /microservices/files_search/src/main/java/com/dreamsoftware/documentsearchengine/persistance/repository/FilesProcessedRepository.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.persistance.repository; 2 | 3 | import com.dreamsoftware.documentsearchengine.persistance.entity.ProcessedFileEntity; 4 | import org.springframework.data.domain.Page; 5 | import org.springframework.data.domain.Pageable; 6 | import org.springframework.data.elasticsearch.annotations.Query; 7 | import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; 8 | import org.springframework.stereotype.Repository; 9 | 10 | /** 11 | * 12 | * @author ssanchez 13 | */ 14 | @Repository 15 | public interface FilesProcessedRepository extends ElasticsearchRepository { 16 | 17 | /** 18 | * @param term 19 | * @param pageable 20 | * @return 21 | */ 22 | @Query("{\"match\": {\"document.document.content\": {\"query\": \"?0\"}}}") 23 | Page search(final String term, final Pageable pageable); 24 | } 25 | -------------------------------------------------------------------------------- /microservices/files_search/src/main/java/com/dreamsoftware/documentsearchengine/service/IFilesProcessedService.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.service; 2 | 3 | import com.dreamsoftware.documentsearchengine.web.dto.ProcessedFileDTO; 4 | import org.springframework.data.domain.Page; 5 | import org.springframework.data.domain.Pageable; 6 | 7 | /** 8 | * 9 | * @author ssanchez 10 | */ 11 | public interface IFilesProcessedService { 12 | 13 | /** 14 | * 15 | * @param searchText 16 | * @param page 17 | * @param size 18 | * @return 19 | */ 20 | Page search(final String searchText, final Integer page, final Integer size); 21 | 22 | /** 23 | * 24 | * @param searchText 25 | * @param pageable 26 | * @return 27 | */ 28 | Page search(final String searchText, final Pageable pageable); 29 | 30 | } 31 | -------------------------------------------------------------------------------- /microservices/files_search/src/main/java/com/dreamsoftware/documentsearchengine/service/impl/FilesProcessedServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.service.impl; 2 | 3 | import com.dreamsoftware.documentsearchengine.mapper.ProcessedFileMapper; 4 | import com.dreamsoftware.documentsearchengine.persistance.repository.FilesProcessedRepository; 5 | import com.dreamsoftware.documentsearchengine.service.IFilesProcessedService; 6 | import com.dreamsoftware.documentsearchengine.web.dto.ProcessedFileDTO; 7 | import lombok.RequiredArgsConstructor; 8 | import org.springframework.data.domain.Page; 9 | import org.springframework.data.domain.PageRequest; 10 | import org.springframework.data.domain.Pageable; 11 | import org.springframework.stereotype.Service; 12 | import org.springframework.util.Assert; 13 | 14 | /** 15 | * 16 | * @author ssanchez 17 | */ 18 | @Service("filesProcessedService") 19 | @RequiredArgsConstructor 20 | public class FilesProcessedServiceImpl implements IFilesProcessedService { 21 | 22 | private final ProcessedFileMapper processedFileMapper; 23 | private final FilesProcessedRepository filesProcessedRepository; 24 | 25 | /** 26 | * 27 | * @param searchText 28 | * @param page 29 | * @param size 30 | * @return 31 | */ 32 | @Override 33 | public Page search(final String searchText, final Integer page, final Integer size) { 34 | Assert.notNull(page, "Page can not be null"); 35 | Assert.notNull(size, "Size can not be null"); 36 | return search(searchText, PageRequest.of(page, size)); 37 | } 38 | 39 | /** 40 | * 41 | * @param searchText 42 | * @param pageable 43 | * @return 44 | */ 45 | @Override 46 | public Page search(final String searchText, final Pageable pageable) { 47 | Assert.notNull(pageable, "Pageable can not be null"); 48 | return filesProcessedRepository.search(searchText, pageable) 49 | .map(processedFileMapper::entityToDTO); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /microservices/files_search/src/main/java/com/dreamsoftware/documentsearchengine/web/controller/FilesSearchController.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.controller; 2 | 3 | import com.dreamsoftware.documentsearchengine.service.IFilesProcessedService; 4 | import com.dreamsoftware.documentsearchengine.web.controller.error.exception.NoFilesProcessedFoundException; 5 | import com.dreamsoftware.documentsearchengine.web.dto.ProcessedFileDTO; 6 | import com.dreamsoftware.documentsearchengine.web.core.APIResponse; 7 | import com.dreamsoftware.documentsearchengine.web.core.ErrorResponseDTO; 8 | import com.dreamsoftware.documentsearchengine.web.core.SupportController; 9 | import io.swagger.v3.oas.annotations.Operation; 10 | import io.swagger.v3.oas.annotations.media.Content; 11 | import io.swagger.v3.oas.annotations.media.Schema; 12 | import io.swagger.v3.oas.annotations.responses.ApiResponse; 13 | import io.swagger.v3.oas.annotations.responses.ApiResponses; 14 | import io.swagger.v3.oas.annotations.tags.Tag; 15 | import lombok.RequiredArgsConstructor; 16 | import org.springframework.data.domain.Page; 17 | import org.springframework.http.HttpStatus; 18 | import org.springframework.http.MediaType; 19 | import org.springframework.http.ResponseEntity; 20 | import org.springframework.security.access.prepost.PreAuthorize; 21 | import org.springframework.validation.annotation.Validated; 22 | import org.springframework.web.bind.annotation.RequestMapping; 23 | import org.springframework.web.bind.annotation.RequestMethod; 24 | import org.springframework.web.bind.annotation.RequestParam; 25 | import org.springframework.web.bind.annotation.RestController; 26 | 27 | @RestController 28 | @Validated 29 | @RequestMapping("/api/v1/search/") 30 | @Tag(name = "files_search", description = "/api/v1/search/ (Code Response interval -> 2XX)") 31 | @RequiredArgsConstructor 32 | public class FilesSearchController extends SupportController { 33 | 34 | /** 35 | * Files Processed Service 36 | */ 37 | private final IFilesProcessedService filesProcessedService; 38 | 39 | /** 40 | * 41 | * @param page 42 | * @param size 43 | * @param searchText 44 | * @return 45 | * @throws Throwable 46 | */ 47 | @Operation(summary = "SEARCH_FILES_PROCESSED - Search Files Processed", description = "Search Files Processed", tags = {"files_search"}) 48 | @ApiResponses(value = { 49 | @ApiResponse(responseCode = "200", description = "Search results", 50 | content = @Content( 51 | schema = @Schema(implementation = Page.class))), 52 | @ApiResponse(responseCode = "404", description = "No Files found", 53 | content = @Content( 54 | schema = @Schema(implementation = ErrorResponseDTO.class))) 55 | }) 56 | @PreAuthorize("hasAuthority('SCOPE_FILES_SEARCH')") 57 | @RequestMapping(value = "/", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) 58 | public ResponseEntity>> searchFilesProcessed( 59 | @RequestParam(name = "page", required = false, defaultValue = "0") final Integer page, 60 | @RequestParam(name = "size", required = false, defaultValue = "20") final Integer size, 61 | @RequestParam(name = "search", required = true) final String searchText) throws Throwable { 62 | 63 | final Page filesProcessedPage = filesProcessedService.search(searchText, page, size); 64 | 65 | if (!filesProcessedPage.hasContent()) { 66 | throw new NoFilesProcessedFoundException(); 67 | } 68 | 69 | return responseHelper.createAndSendResponse(FilesSearchResponseCodeEnum.SEARCH_RESULTS, 70 | HttpStatus.OK, filesProcessedPage); 71 | 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /microservices/files_search/src/main/java/com/dreamsoftware/documentsearchengine/web/controller/FilesSearchResponseCodeEnum.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.controller; 2 | 3 | import com.dreamsoftware.documentsearchengine.web.core.IResponseCodeTypes; 4 | 5 | /** 6 | * 7 | * @author ssanchez 8 | */ 9 | public enum FilesSearchResponseCodeEnum implements IResponseCodeTypes { 10 | 11 | SEARCH_RESULTS(200L), 12 | NO_FILES_PROCESSED_FOUND(201L); 13 | 14 | private final Long code; 15 | 16 | public static final String CATEGORY_NAME = "FILES_SEARCH_SERVICE"; 17 | 18 | private FilesSearchResponseCodeEnum(Long code) { 19 | this.code = code; 20 | } 21 | 22 | @Override 23 | public Long getResponseCode() { 24 | return code; 25 | } 26 | 27 | @Override 28 | public String getCategoryName() { 29 | return CATEGORY_NAME; 30 | } 31 | 32 | @Override 33 | public String getCodeName() { 34 | return name(); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /microservices/files_search/src/main/java/com/dreamsoftware/documentsearchengine/web/controller/error/FilesSearchErrorController.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.controller.error; 2 | 3 | import com.dreamsoftware.documentsearchengine.web.controller.FilesSearchResponseCodeEnum; 4 | import com.dreamsoftware.documentsearchengine.web.controller.error.exception.NoFilesProcessedFoundException; 5 | import com.dreamsoftware.documentsearchengine.web.core.APIResponse; 6 | import com.dreamsoftware.documentsearchengine.web.core.ErrorResponseDTO; 7 | import com.dreamsoftware.documentsearchengine.web.core.SupportController; 8 | import javax.servlet.http.HttpServletRequest; 9 | import org.springframework.core.Ordered; 10 | import org.springframework.core.annotation.Order; 11 | import org.springframework.http.HttpStatus; 12 | import org.springframework.http.ResponseEntity; 13 | import org.springframework.web.bind.annotation.ControllerAdvice; 14 | import org.springframework.web.bind.annotation.ExceptionHandler; 15 | import org.springframework.web.bind.annotation.ResponseBody; 16 | 17 | /** 18 | * 19 | * @author ssanchez 20 | */ 21 | @ControllerAdvice 22 | @Order(Ordered.HIGHEST_PRECEDENCE) 23 | public class FilesSearchErrorController extends SupportController { 24 | 25 | /** 26 | * 27 | * @param ex 28 | * @param request 29 | * @return 30 | */ 31 | @ExceptionHandler(NoFilesProcessedFoundException.class) 32 | @ResponseBody 33 | protected ResponseEntity> handleNoFilesProcessedFoundException(NoFilesProcessedFoundException ex, HttpServletRequest request) { 34 | return responseHelper.createAndSendErrorResponse(FilesSearchResponseCodeEnum.NO_FILES_PROCESSED_FOUND, 35 | HttpStatus.NOT_FOUND, "No Files Processed."); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /microservices/files_search/src/main/java/com/dreamsoftware/documentsearchengine/web/controller/error/exception/NoFilesProcessedFoundException.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.controller.error.exception; 2 | 3 | /** 4 | * 5 | * @author ssanchez 6 | */ 7 | public class NoFilesProcessedFoundException extends RuntimeException { 8 | } 9 | -------------------------------------------------------------------------------- /microservices/files_search/src/main/java/com/dreamsoftware/documentsearchengine/web/dto/ProcessedFileDTO.java: -------------------------------------------------------------------------------- 1 | package com.dreamsoftware.documentsearchengine.web.dto; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import java.util.Date; 5 | import lombok.AllArgsConstructor; 6 | import lombok.Data; 7 | import lombok.EqualsAndHashCode; 8 | import lombok.NoArgsConstructor; 9 | 10 | /** 11 | * 12 | * @author ssanchez 13 | */ 14 | @Data 15 | @EqualsAndHashCode(callSuper = false) 16 | @AllArgsConstructor 17 | @NoArgsConstructor 18 | public class ProcessedFileDTO { 19 | 20 | /** 21 | * File Id 22 | */ 23 | @JsonProperty("id") 24 | private String id; 25 | 26 | /** 27 | * Version 28 | */ 29 | @JsonProperty("version") 30 | private Integer version; 31 | 32 | /** 33 | * File UUID 34 | */ 35 | @JsonProperty("uuid") 36 | private String uuid; 37 | 38 | /** 39 | * File Name 40 | */ 41 | @JsonProperty("name") 42 | private String name; 43 | 44 | /** 45 | * File Permissions 46 | */ 47 | @JsonProperty("permissions") 48 | private String permissions; 49 | 50 | /** 51 | * File URL 52 | */ 53 | @JsonProperty("url") 54 | private String url; 55 | 56 | /** 57 | * File Mime Type 58 | */ 59 | @JsonProperty("mime_type") 60 | private String mimeType; 61 | 62 | /** 63 | * File Mime Extension 64 | */ 65 | @JsonProperty("mime_extension") 66 | private String mimeExtension; 67 | 68 | /** 69 | * File Created At 70 | */ 71 | @JsonProperty("created_at") 72 | private Date createdAt; 73 | 74 | /** 75 | * File Language 76 | */ 77 | @JsonProperty("language") 78 | private String language; 79 | 80 | /** 81 | * File Author 82 | */ 83 | @JsonProperty("author") 84 | private String author; 85 | 86 | /** 87 | * File Creator 88 | */ 89 | @JsonProperty("creator") 90 | private String creator; 91 | 92 | /** 93 | * File Producer 94 | */ 95 | @JsonProperty("producer") 96 | private String producer; 97 | 98 | /** 99 | * File Last Modified Time 100 | */ 101 | @JsonProperty("last_modified_time") 102 | private Date lastModifiedTime; 103 | 104 | } 105 | -------------------------------------------------------------------------------- /microservices/files_search/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | application: 3 | name: files-search-service 4 | security: 5 | oauth2: 6 | resourceserver: 7 | jwt: 8 | issuer-uri: http://keycloak:8080/auth/realms/document_search_engine 9 | cloud: 10 | consul: 11 | host: consul-server 12 | port: 8500 13 | discovery: 14 | port: 8080 15 | prefer-ip-address: true 16 | healthCheckPath: /actuator/health 17 | 18 | management: 19 | health: 20 | elasticsearch: 21 | enabled: false 22 | ## Spring Open API 23 | springdoc: 24 | swagger-ui: 25 | path: /swagger-ui.html 26 | api-docs: 27 | path: /v3/api-docs 28 | 29 | elasticsearch: 30 | host: elasticsearch:9200 31 | cert: 32 | file: elastic-certificates.p12 33 | password: ssanchez00 34 | user: 35 | name: elastic 36 | pass: ssanchez00 37 | 38 | logging.level: 39 | org.springframework.cloud.gateway: DEBUG 40 | org.springframework.security: DEBUG 41 | org.springframework.web.reactive.function.client: TRACE 42 | com.dreamsoftware.documentsearchengine: DEBUG 43 | 44 | server.port: 8080 -------------------------------------------------------------------------------- /microservices/files_search/src/main/resources/elastic-certificates.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/microservices/files_search/src/main/resources/elastic-certificates.p12 -------------------------------------------------------------------------------- /microservices/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 4.0.0 7 | com.dreamsoftware 8 | document_search_engine 9 | 0.0.1-SNAPSHOT 10 | pom 11 | Document Search Engine Microservices 12 | 13 | 14 | 8 15 | 1.3.1.Final 16 | Hoxton.SR8 17 | 18 | 19 | 20 | org.springframework.boot 21 | spring-boot-starter-parent 22 | 2.3.5.RELEASE 23 | 24 | 25 | 26 | 27 | files_api_gateway 28 | files_metadata 29 | files_management 30 | files_search 31 | files_commons 32 | files_notifications 33 | 34 | 35 | 36 | 37 | org.springframework.boot 38 | spring-boot-starter-actuator 39 | 40 | 41 | org.springframework.cloud 42 | spring-cloud-starter-consul-discovery 43 | 44 | 45 | org.projectlombok 46 | lombok 47 | provided 48 | 49 | 50 | org.mapstruct 51 | mapstruct 52 | ${org.mapstruct.version} 53 | 54 | 55 | 56 | 57 | 58 | 59 | org.springframework.cloud 60 | spring-cloud-dependencies 61 | ${spring-cloud.version} 62 | pom 63 | import 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | org.apache.maven.plugins 72 | maven-compiler-plugin 73 | 3.8.1 74 | 75 | 1.8 76 | 1.8 77 | 78 | 79 | org.mapstruct 80 | mapstruct-processor 81 | ${org.mapstruct.version} 82 | 83 | 84 | 85 | org.projectlombok 86 | lombok 87 | 1.18.12 88 | 89 | 90 | 91 | 92 | -Amapstruct.suppressGeneratorTimestamp=true 93 | 94 | 95 | -Amapstruct.suppressGeneratorVersionInfoComment=true 96 | 97 | 98 | -Amapstruct.defaultComponentModel=spring 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | -------------------------------------------------------------------------------- /nifi/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM apache/nifi:1.12.0 2 | COPY ./conf/core-site.xml /opt/nifi/conf/ 3 | COPY ./conf/hdfs-site.xml /opt/nifi/conf/ 4 | COPY ./templates/Ingest_Files_Pipeline_Template_v9.xml /opt/nifi/nifi-current/conf/templates 5 | COPY --chown=nifi:nifi flow.xml.gz /opt/nifi/nifi-current/conf/ 6 | EXPOSE 8080 7 | -------------------------------------------------------------------------------- /nifi/conf/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | hadoop.proxyuser.hue.hosts* 21 | fs.defaultFShdfs://namenode:9000 22 | hadoop.http.staticuser.userroot 23 | io.compression.codecsorg.apache.hadoop.io.compress.SnappyCodec 24 | hadoop.proxyuser.hue.groups* 25 | -------------------------------------------------------------------------------- /nifi/conf/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | dfs.namenode.datanode.registration.ip-hostname-checkfalse 22 | dfs.webhdfs.enabledtrue 23 | dfs.permissions.enabledfalse 24 | dfs.namenode.rpc-bind-host0.0.0.0 25 | dfs.namenode.servicerpc-bind-host0.0.0.0 26 | dfs.namenode.http-bind-host0.0.0.0 27 | dfs.namenode.https-bind-host0.0.0.0 28 | dfs.client.use.datanode.hostnametrue 29 | dfs.datanode.use.datanode.hostnametrue 30 | -------------------------------------------------------------------------------- /nifi/flow.xml.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sergio11/document_search_engine_architecture/50d43f03fe45ad0a22ea04f7fdf22817b71aa97f/nifi/flow.xml.gz -------------------------------------------------------------------------------- /notas.txt: -------------------------------------------------------------------------------- 1 | This tool assists you in the generation of X.509 certificates and certificate 2 | signing requests for use with SSL/TLS in the Elastic stack. 3 | 4 | The 'ca' mode generates a new 'certificate authority' 5 | This will create a new X.509 certificate and private key that can be used 6 | to sign certificate when running in 'cert' mode. 7 | 8 | Use the 'ca-dn' option if you wish to configure the 'distinguished name' 9 | of the certificate authority 10 | 11 | By default the 'ca' mode produces a single PKCS#12 output file which holds: 12 | * The CA certificate 13 | * The CA's private key 14 | 15 | If you elect to generate PEM format certificates (the -pem option), then the output will 16 | be a zip file containing individual files for the CA certificate and private key 17 | 18 | 19 | 20 | This tool assists you in the generation of X.509 certificates and certificate 21 | signing requests for use with SSL/TLS in the Elastic stack. 22 | 23 | The 'cert' mode generates X.509 certificate and private keys. 24 | * By default, this generates a single certificate and key for use 25 | on a single instance. 26 | * The '-multiple' option will prompt you to enter details for multiple 27 | instances and will generate a certificate and key for each one 28 | * The '-in' option allows for the certificate generation to be automated by describing 29 | the details of each instance in a YAML file 30 | 31 | * An instance is any piece of the Elastic Stack that requires an SSL certificate. 32 | Depending on your configuration, Elasticsearch, Logstash, Kibana, and Beats 33 | may all require a certificate and private key. 34 | * The minimum required value for each instance is a name. This can simply be the 35 | hostname, which will be used as the Common Name of the certificate. A full 36 | distinguished name may also be used. 37 | * A filename value may be required for each instance. This is necessary when the 38 | name would result in an invalid file or directory name. The name provided here 39 | is used as the directory name (within the zip) and the prefix for the key and 40 | certificate files. The filename is required if you are prompted and the name 41 | is not displayed in the prompt. 42 | * IP addresses and DNS names are optional. Multiple values can be specified as a 43 | comma separated string. If no IP addresses or DNS names are provided, you may 44 | disable hostname verification in your SSL configuration. 45 | 46 | * All certificates generated by this tool will be signed by a certificate authority (CA). 47 | * The tool can automatically generate a new CA for you, or you can provide your own with the 48 | -ca or -ca-cert command line options. 49 | 50 | By default the 'cert' mode produces a single PKCS#12 output file which holds: 51 | * The instance certificate 52 | * The private key for the instance certificate 53 | * The CA certificate 54 | 55 | If you specify any of the following options: 56 | * -pem (PEM formatted output) 57 | * -keep-ca-key (retain generated CA key) 58 | * -multiple (generate multiple certificates) 59 | * -in (generate certificates from an input file) 60 | then the output will be be a zip file containing individual certificate/key files 61 | 62 | 63 | Certificates written to /usr/share/elasticsearch/elastic-certificates.p12 64 | 65 | This file should be properly secured as it contains the private key for 66 | your instance. 67 | 68 | This file is a self contained file and can be copied and used 'as is' 69 | For each Elastic product that you wish to configure, you should copy 70 | this '.p12' file to the relevant configuration directory 71 | and then follow the SSL configuration instructions in the product guide. 72 | 73 | For client applications, you may only need to copy the CA certificate and 74 | configure the client to trust this certificate. 75 | 76 | 77 | 78 | curl -XPUT -u elastic '192.168.0.1:9200/_xpack/security/user/kibana/_password?pretty' -H 'Content-Type: application/json' -d' 79 | { 80 | "password": "NewKibanaPassword" 81 | } 82 | 83 | 84 | 85 | OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release. 86 | WARNING: An illegal reflective access operation has occurred 87 | WARNING: Illegal reflective access by org.jruby.ext.openssl.SecurityHelper (file:/usr/share/logstash/vendor/jruby/lib/ruby/stdlib/jopenssl.jar) to field java.security.MessageDigest.provider 88 | WARNING: Please consider reporting this to the maintainers of org.jruby.ext.openssl.SecurityHelper 89 | WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations 90 | WARNING: All illegal access operations will be denied in a future release 91 | 92 | 93 | 94 | docker-compose ps -q nifi 95 | 96 | 97 | docker cp 63046795432eeaf912179beac5f69f12787638d99b31aa96cf71c3badb73e803:/opt/nifi/nifi-current/conf/flow.xml.gz flow.xml.gz -------------------------------------------------------------------------------- /notas_logstash.txt: -------------------------------------------------------------------------------- 1 | Esto se debe a un control denominado validación de nombre de host, es decir, un control de que el CN ​​o una de las 2 | SAN incluidas en un certificado X509 coinciden con el nombre de host del host que utiliza ese certificado para TLS. 3 | 4 | 5 | 6 | xpack.security.transport.ssl.enabled: true 7 | xpack.security.transport.ssl.keystore.type: PKCS12 8 | xpack.security.transport.ssl.keystore.password: ssanchez00 9 | xpack.security.transport.ssl.verification_mode: certificate 10 | xpack.security.transport.ssl.keystore.path: elastic-certificates.p12 11 | xpack.security.transport.ssl.truststore.path: elastic-certificates.p12 12 | xpack.security.transport.ssl.truststore.type: PKCS12 13 | 14 | docker cp cc27174afdd8928a99815ed332277c0c9ddc5476f38630ed2a3dbae6daca6953:/usr/share/elasticsearch/certificates certici 15 | 16 | 17 | 18 | Creating logstash ... error 19 | 20 | ERROR: for logstash Cannot create container for service logstash: invalid mode: /usr/share/logstash/ 21 | 22 | ERROR: for logstash Cannot create container for service logstash: invalid mode: /usr/share/logstash/ 23 | ERROR: Encountered errors while bringing up the project. 24 | Step 1/4 : FROM docker.elastic.co/logstash/logstash:7.9.2 25 | 7.9.2: Pulling from logstash/logstash 26 | Digest: sha256:f2c9d6d0fcd63cdb2a44dea7e9af1ccd6cc388a8af409fa66865abcab64b2c61 27 | Status: Downloaded newer image for docker.elastic.co/logstash/logstash:7.9.2 28 | ---> 736bccdc74f4 29 | Step 2/4 : RUN logstash-plugin install logstash-input-mongodb 30 | ---> Running in 99a313fe7f05 -------------------------------------------------------------------------------- /notas_microservices.txt: -------------------------------------------------------------------------------- 1 | Apache Commons Net vs Spring Integration FTP Support 2 | 3 | En términos generales, en el mundo de la integración de aplicaciones empresariales (EAI), 4 | tenemos cuatro tipos de integración: sincronización de archivos, RPC, sincronización de bases de datos y mensajería. 5 | 6 | 7 | El servidor FTP Apache es una implementación súper escalable, liviana y completamente en Java del protocolo FTP 8 | 9 | Dota a las aplicaciones que lo incluyan de la capacidad de Consul de descubrimiento 10 | y registro de servicios, atacando al API HTTP del mismo. 11 | 12 | Para ello, por defecto, hace uso del endpoint /health, el cual disponibiliza *Spring Boot Actuator *de forma nativa 13 | y con solo incluir la dependencia tenemos health checking sin complicarnos demasiado: 14 | 15 | Ribbon 16 | 17 | La puerta de enlace API toma todas las llamadas a la API de los clientes y las enruta a los microservicios 18 | apropiados con el enrutamiento de solicitudes, la composición y la traducción del protocolo. 19 | 20 | https://www.learnnowlab.com/Spring-Cloud-Gateway-with-Spring-Security/ -------------------------------------------------------------------------------- /sftp/users.conf: -------------------------------------------------------------------------------- 1 | ssanchez:ssanchez00:1001:100 --------------------------------------------------------------------------------