├── .gitignore ├── LICENSE ├── Procfile ├── README.md ├── docker-compose.yaml ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── sjcdigital │ │ └── temis │ │ ├── TemisApplication.java │ │ ├── controller │ │ ├── AldermanController.java │ │ ├── LawsController.java │ │ ├── exceptions │ │ │ ├── ResourceNotFoundException.java │ │ │ └── VoteException.java │ │ └── util │ │ │ └── ResourceUtil.java │ │ ├── load │ │ ├── TemisCron.java │ │ └── TemisStarter.java │ │ ├── model │ │ ├── document │ │ │ ├── Alderman.java │ │ │ ├── AldermanSurrogate.java │ │ │ ├── Law.java │ │ │ ├── OrdinarySession.java │ │ │ └── Type.java │ │ ├── enums │ │ │ └── AldermanPresenceSheet.java │ │ ├── exceptions │ │ │ └── BotException.java │ │ ├── repositories │ │ │ ├── AldermanRepository.java │ │ │ ├── LawsRepository.java │ │ │ └── OrdinarySessionRepository.java │ │ └── service │ │ │ ├── bots │ │ │ ├── AbstractBot.java │ │ │ ├── Bot.java │ │ │ ├── BotService.java │ │ │ └── impl │ │ │ │ ├── AldermanPresenceBot.java │ │ │ │ ├── AldermenBot.java │ │ │ │ ├── CompositeBot.java │ │ │ │ └── LawsBot.java │ │ │ ├── camel │ │ │ ├── processor │ │ │ │ ├── AbstractProcessor.java │ │ │ │ └── impl │ │ │ │ │ ├── AldermanPresenceProcessor.java │ │ │ │ │ ├── AldermanProcessor.java │ │ │ │ │ └── LawsProcessor.java │ │ │ └── route │ │ │ │ ├── AbstractRoute.java │ │ │ │ └── impl │ │ │ │ ├── AldermanPresenceRoute.java │ │ │ │ ├── AldermanRoute.java │ │ │ │ └── LawsRoute.java │ │ │ ├── machine_learn │ │ │ ├── ClassifyLaw.java │ │ │ └── Train.java │ │ │ ├── parsers │ │ │ ├── AbstractParser.java │ │ │ ├── impl │ │ │ │ ├── AldermanPresenceParser.java │ │ │ │ ├── AldermenParser.java │ │ │ │ └── LawsParser.java │ │ │ └── util │ │ │ │ └── AldermanParserUtil.java │ │ │ └── vote │ │ │ └── Vote.java │ │ └── util │ │ ├── RegexUtils.java │ │ ├── StringUtil.java │ │ └── TemisFileUtil.java ├── resources │ ├── application.yaml │ └── machine-learn │ │ ├── pt-leis.bin │ │ └── pt-leis.train └── webapp │ └── img │ └── vereadores │ ├── amelia_naomi.jpg │ ├── amélia_naomi.jpg │ ├── carlinhos_tiaca.jpg │ ├── dilermando_die.jpg │ ├── dilermando_dié.jpg │ ├── dr._roniel.jpg │ ├── dra._angela.jpg │ ├── dulce_rita.jpg │ ├── fernando_petiti_da_farmacia_comunitaria.jpg │ ├── fernando_petiti_da_farmácia_comunitária.jpg │ ├── juliana_fraga.jpg │ ├── juvenil_silverio.jpg │ ├── juvenil_silvério.jpg │ ├── luiz_mota.jpg │ ├── macedo_bastos.jpg │ ├── politico_sem_foto.png │ ├── prof._calasans_camargo.jpg │ ├── renata_paiva.jpg │ ├── robertinho_da_padaria.jpg │ ├── roberto_do_eleven.jpg │ ├── rogerio_cyborg.jpg │ ├── rogério_cyborg.jpg │ ├── shakespeare_carvalho.jpg │ ├── valdir_alvarenga.jpg │ ├── wagner_balieiro.jpg │ ├── walter_hayashi.jpg │ └── willis_goulart.jpg └── test └── java └── com └── sjcdigital └── temis └── model └── service └── parsers └── util └── AldermanParserUtilTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | 3 | # Package Files 4 | *.jar 5 | *.war 6 | *.ear 7 | 8 | # Maven Build 9 | target/ 10 | 11 | # Eclipse directories 12 | bin/ 13 | .settings 14 | .classpath 15 | .project 16 | .tern-project 17 | 18 | # Intellij directories 19 | .idea/ 20 | *.iml 21 | 22 | # Forge file 23 | .forge_settings 24 | 25 | # Mac File 26 | *.DS_Store 27 | 28 | *~ 29 | 30 | src/main/resources/data/html/* 31 | /data.zip -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: java -Dserver.port=$PORT -jar target/temis-server.jar -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TÊMIS-SERVER 2 | 3 | ![alt tag](http://files.deuseseherois.webnode.com.br/system_preview_detail_200000053-8ee8990dc5/T%C3%AAmis.jpg) 4 | 5 | ### Quem foi Têmis? 6 | 7 | Têmis era a deusa guardiã dos juramentos dos homens e da lei, sendo que era costumeiro invocá-la nos julgamentos perante os magistrados. 8 | 9 | Têmis empunha a balança, com que equilibra a razão com o julgamento, e/ou uma cornucópia. Seu nome significa "aquela que é posta, colocada". 10 | 11 | Fonte: http://deuseseherois.webnode.com.br/products/t%C3%AAmis/ 12 | 13 | ### O projeto 14 | 15 | Neste projeto, criamos um "chupa-cabra" e raspamos as Leis Ordinárias presentes na página [http://www.ceaam.net/sjc/legislacao/](http://www.ceaam.net/sjc/legislacao/). Com esses dados, expomos esses dados através de uma API rest. 16 | 17 | ### O que usamos? 18 | 19 | * Java 8; 20 | * Spring-boot; 21 | * Maven; 22 | * JPA; 23 | * MongoDB; 24 | * Camel; 25 | * JSoup 26 | 27 | ### URIs 28 | 29 | Api foi hospedada no Heroku, usando MLabs para armazenar os dados no Mongo. Todos serviços estão presentes em: http://temis-server.herokuapp.com/api e você pode explorar as APIS em http://temis-server.herokuapp.com/swagger-ui.html#/ 30 | 31 | contato: pedro-hos@outlook.com -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | mongo: 2 | container_name: mongodb-temis-api 3 | image: mongo 4 | ports: 5 | - 27017:27017 -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.sjcdigital.temis 7 | temis-server 8 | jar 9 | 10 | temis 11 | 12 | Serviço para raspar as leis de municipais de São José dos Campos. 13 | 14 | 15 | 16 | UTF-8 17 | UTF-8 18 | 1.8 19 | 1.4.0.RELEASE 20 | 1.8 21 | 1.8 22 | 3.3 23 | 24 | 25 | 26 | 27 | 28 | 29 | org.springframework.boot 30 | spring-boot-starter-web 31 | ${spring-boot.version} 32 | 33 | 34 | org.springframework.boot 35 | spring-boot-starter-tomcat 36 | 37 | 38 | 39 | 40 | 41 | org.springframework.boot 42 | spring-boot-starter-undertow 43 | ${spring-boot.version} 44 | 45 | 46 | 47 | org.springframework.boot 48 | spring-boot-starter-data-mongodb 49 | ${spring-boot.version} 50 | 51 | 52 | 53 | 54 | org.springframework.boot 55 | spring-boot-starter-hateoas 56 | ${spring-boot.version} 57 | 58 | 59 | 60 | 61 | com.fasterxml.jackson.datatype 62 | jackson-datatype-jsr310 63 | 2.8.1 64 | 65 | 66 | 67 | 68 | org.apache.camel 69 | camel-spring-boot 70 | 2.17.2 71 | 72 | 73 | 74 | 75 | org.apache.commons 76 | commons-lang3 77 | 3.4 78 | 79 | 80 | 81 | 82 | commons-io 83 | commons-io 84 | 2.5 85 | 86 | 87 | 88 | 89 | org.apache.poi 90 | poi 91 | 3.15 92 | 93 | 94 | 95 | 96 | org.jsoup 97 | jsoup 98 | 1.9.2 99 | 100 | 101 | 102 | io.springfox 103 | springfox-swagger2 104 | 2.4.0 105 | 106 | 107 | 108 | io.springfox 109 | springfox-swagger-ui 110 | 2.4.0 111 | 112 | 113 | 114 | 115 | org.apache.opennlp 116 | opennlp-tools 117 | 1.6.0 118 | 119 | 120 | 121 | 122 | 123 | 124 | org.springframework.boot 125 | spring-boot-starter-test 126 | ${spring-boot.version} 127 | test 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | org.springframework.boot 137 | spring-boot-maven-plugin 138 | ${spring-boot.version} 139 | 140 | 141 | 142 | repackage 143 | 144 | 145 | ${project.artifactId} 146 | 147 | 148 | 149 | 150 | 151 | 152 | org.apache.maven.plugins 153 | maven-jar-plugin 154 | 2.3.2 155 | 156 | ${project.artifactId} 157 | 158 | 159 | 160 | 161 | 162 | 163 | 1.0.0-ALPHA1 164 | 165 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/TemisApplication.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; 7 | import org.springframework.scheduling.annotation.EnableAsync; 8 | import org.springframework.scheduling.annotation.EnableScheduling; 9 | import org.springframework.web.servlet.config.annotation.CorsRegistry; 10 | import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; 11 | import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter; 12 | 13 | import springfox.documentation.builders.PathSelectors; 14 | import springfox.documentation.builders.RequestHandlerSelectors; 15 | import springfox.documentation.spi.DocumentationType; 16 | import springfox.documentation.spring.web.plugins.Docket; 17 | import springfox.documentation.swagger2.annotations.EnableSwagger2; 18 | 19 | @EnableAsync 20 | @EnableSwagger2 21 | @EnableScheduling 22 | @SpringBootApplication 23 | @EnableMongoRepositories("com.sjcdigital.temis.model.repositories") 24 | public class TemisApplication { 25 | 26 | public static void main(final String[] args) { 27 | SpringApplication.run(TemisApplication.class, args); 28 | } 29 | 30 | @Bean 31 | public WebMvcConfigurer corsConfigurer() { 32 | 33 | return new WebMvcConfigurerAdapter() { 34 | @Override 35 | public void addCorsMappings(final CorsRegistry registry) { 36 | registry.addMapping("/**") 37 | .allowedOrigins("*") 38 | .allowedMethods("POST", "PUT", "GET", "OPTIONS", "DELETE"); 39 | } 40 | }; 41 | } 42 | 43 | @Bean 44 | public Docket api() { 45 | return new Docket(DocumentationType.SWAGGER_2).select() 46 | .apis(RequestHandlerSelectors.any()) 47 | .paths(PathSelectors.any()) 48 | .build(); 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/controller/AldermanController.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.controller; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.data.domain.Page; 5 | import org.springframework.data.domain.Pageable; 6 | import org.springframework.data.web.PagedResourcesAssembler; 7 | import org.springframework.hateoas.EntityLinks; 8 | import org.springframework.hateoas.ExposesResourceFor; 9 | import org.springframework.hateoas.PagedResources; 10 | import org.springframework.hateoas.Resource; 11 | import org.springframework.hateoas.Resources; 12 | import org.springframework.web.bind.annotation.GetMapping; 13 | import org.springframework.web.bind.annotation.PathVariable; 14 | import org.springframework.web.bind.annotation.RequestMapping; 15 | import org.springframework.web.bind.annotation.RestController; 16 | 17 | import com.sjcdigital.temis.controller.exceptions.ResourceNotFoundException; 18 | import com.sjcdigital.temis.controller.util.ResourceUtil; 19 | import com.sjcdigital.temis.model.document.Alderman; 20 | import com.sjcdigital.temis.model.document.Law; 21 | import com.sjcdigital.temis.model.repositories.AldermanRepository; 22 | import com.sjcdigital.temis.model.repositories.LawsRepository; 23 | 24 | /** 25 | * @author pedro-hos 26 | */ 27 | 28 | @RestController 29 | @RequestMapping("/api/alderman") 30 | @ExposesResourceFor(Alderman.class) 31 | public class AldermanController { 32 | 33 | @Autowired 34 | private AldermanRepository aldermanRepository; 35 | 36 | @Autowired 37 | private LawsRepository lawRepository; 38 | 39 | @Autowired 40 | private EntityLinks entityLinks; 41 | 42 | /** 43 | * Get all Alderman 44 | * @return List Alderman 45 | */ 46 | @GetMapping 47 | public PagedResources> findAll(final Pageable pageable, final PagedResourcesAssembler assembler) { 48 | final PagedResources> pagedResources = assembler.toResource(aldermanRepository.findAll(pageable)); 49 | pagedResources.forEach(this :: createAldermanResource); 50 | return pagedResources; 51 | } 52 | 53 | 54 | /** 55 | * Get alderman by Name 56 | * @param name, alderman name 57 | * @return Alderman 58 | */ 59 | @GetMapping("/{name}") 60 | public Resource findByName(@PathVariable final String name) { 61 | Alderman alderman = aldermanRepository.findByName(name).orElseThrow(ResourceNotFoundException :: new); 62 | Resource resource = new Resource(alderman); 63 | createAldermanResource(resource); 64 | return resource; 65 | } 66 | 67 | /** 68 | * Get alderman law by alderman name 69 | * @param name 70 | * @return Alderman 71 | */ 72 | @GetMapping("/{name}/law") 73 | public Resources findLawByAlderman(@PathVariable final String name, final Pageable pageable, final PagedResourcesAssembler assembler) { 74 | Alderman alderman = aldermanRepository.findByName(name).orElseThrow(ResourceNotFoundException :: new); 75 | Page laws = lawRepository.findByAuthorId(alderman.getId(), pageable); 76 | return createAldermanLawResource(laws, name); 77 | } 78 | 79 | /** 80 | * build Alderman Law Resources 81 | * @param laws laws 82 | * @param name name 83 | * @return Resources of Law 84 | */ 85 | private Resources createAldermanLawResource(final Page laws, final String name) { 86 | final String endopoint = name + "/law"; 87 | return ResourceUtil.createResources(laws, endopoint, entityLinks, Alderman.class); 88 | } 89 | 90 | 91 | /** 92 | * build Alderman Resources 93 | * @param alderman 94 | * @param resource 95 | */ 96 | private void createAldermanResource(final Resource resource) { 97 | resource.add(entityLinks.linkFor(Alderman.class).slash(resource.getContent().getName()).withSelfRel()); 98 | resource.add(entityLinks.linkFor(Alderman.class).slash(resource.getContent().getName()).slash("/law").withRel("leis")); 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/controller/LawsController.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.controller; 2 | 3 | import java.math.BigInteger; 4 | 5 | import javax.servlet.http.HttpServletRequest; 6 | 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.data.domain.Pageable; 9 | import org.springframework.data.web.PagedResourcesAssembler; 10 | import org.springframework.hateoas.EntityLinks; 11 | import org.springframework.hateoas.ExposesResourceFor; 12 | import org.springframework.hateoas.PagedResources; 13 | import org.springframework.hateoas.Resource; 14 | import org.springframework.web.bind.annotation.GetMapping; 15 | import org.springframework.web.bind.annotation.PathVariable; 16 | import org.springframework.web.bind.annotation.PutMapping; 17 | import org.springframework.web.bind.annotation.RequestMapping; 18 | import org.springframework.web.bind.annotation.RequestParam; 19 | import org.springframework.web.bind.annotation.RestController; 20 | 21 | import com.sjcdigital.temis.controller.exceptions.ResourceNotFoundException; 22 | import com.sjcdigital.temis.model.document.Law; 23 | import com.sjcdigital.temis.model.repositories.LawsRepository; 24 | import com.sjcdigital.temis.model.service.vote.Vote; 25 | 26 | /** 27 | * @author pedro-hos 28 | */ 29 | 30 | @RestController 31 | @RequestMapping("/api/laws") 32 | @ExposesResourceFor(Law.class) 33 | public class LawsController { 34 | 35 | @Autowired 36 | private LawsRepository lawsRepository; 37 | 38 | @Autowired 39 | private Vote vote; 40 | 41 | @Autowired 42 | private EntityLinks entityLinks; 43 | 44 | /** 45 | * Find All Laws 46 | * @param pageable 47 | * @return Laws 48 | */ 49 | @GetMapping 50 | public PagedResources> findAllPageable(final Pageable pageable, final PagedResourcesAssembler assembler) { 51 | 52 | PagedResources> pagedResources = assembler.toResource(lawsRepository.findAllByOrderByCodeDesc(pageable) 53 | .orElseThrow(ResourceNotFoundException :: new)); 54 | pagedResources.forEach(this :: createVoteResource); 55 | 56 | return pagedResources; 57 | } 58 | 59 | /** 60 | * Find one Law by Code 61 | * @param pageable 62 | * @return Laws 63 | */ 64 | @GetMapping("/{code}") 65 | public Resource findByCode(@PathVariable String code) { 66 | Law law = lawsRepository.findByCode(code).orElseThrow(ResourceNotFoundException::new); 67 | Resource resource = new Resource(law); 68 | createVoteResource(resource); 69 | return resource; 70 | } 71 | 72 | /** 73 | * vote in law 74 | * @param code law code 75 | * @param rating law rating 76 | * @param request request 77 | * @return law updated 78 | */ 79 | @PutMapping("/{code}/vote") 80 | public Resource vote(@PathVariable String code, @RequestParam("rating") BigInteger rating, HttpServletRequest request) { 81 | Resource resource = new Resource(vote.addVote(code, rating, request.getRemoteAddr())); 82 | createVoteResource(resource); 83 | return resource; 84 | } 85 | 86 | private void createVoteResource(Resource resource) { 87 | resource.add(entityLinks.linkFor(Law.class).slash(resource.getContent().getCode()).withSelfRel()); 88 | resource.add(entityLinks.linkFor(Law.class).slash(resource.getContent().getCode()).slash("/vote").withRel("vote")); 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/controller/exceptions/ResourceNotFoundException.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.controller.exceptions; 2 | 3 | import org.springframework.http.HttpStatus; 4 | import org.springframework.web.bind.annotation.ResponseStatus; 5 | 6 | @ResponseStatus(value = HttpStatus.NOT_FOUND) 7 | public class ResourceNotFoundException extends RuntimeException { 8 | private static final long serialVersionUID = 1L; 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/controller/exceptions/VoteException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.sjcdigital.temis.controller.exceptions; 5 | 6 | import org.springframework.http.HttpStatus; 7 | import org.springframework.web.bind.annotation.ResponseStatus; 8 | 9 | /** 10 | * @author pedro-hos 11 | */ 12 | @ResponseStatus(value = HttpStatus.FORBIDDEN, reason = "Você já votou nessa Lei!") 13 | public class VoteException extends RuntimeException { 14 | private static final long serialVersionUID = 1L; 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/controller/util/ResourceUtil.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.controller.util; 2 | 3 | import java.util.Objects; 4 | import java.util.Optional; 5 | 6 | import org.springframework.data.domain.Page; 7 | import org.springframework.data.domain.Pageable; 8 | import org.springframework.hateoas.EntityLinks; 9 | import org.springframework.hateoas.Link; 10 | import org.springframework.hateoas.Resources; 11 | 12 | /** 13 | * @author pedro-hos 14 | * 15 | */ 16 | public class ResourceUtil { 17 | 18 | private static final String SIZE = "&size="; 19 | private static final String PAGE = "?page="; 20 | 21 | public static Resources createResources(Page page, String endopoint, EntityLinks links, Class clazz) { 22 | 23 | Resources resources = new Resources<>(page); 24 | 25 | Optional self = createLink(endopoint, page.getNumber(), page.getSize(), Link.REL_SELF, links, clazz); 26 | self.ifPresent(resources::add); 27 | 28 | Optional first = createLink(endopoint, 0, page.getSize(), Link.REL_FIRST, links, clazz); 29 | first.ifPresent(resources::add); 30 | 31 | Optional next = createLink(endopoint, page.nextPageable(), Link.REL_NEXT, links, clazz); 32 | next.ifPresent(resources::add); 33 | 34 | Optional privious = createLink(endopoint, page.previousPageable(), Link.REL_PREVIOUS, links, clazz); 35 | privious.ifPresent(resources::add); 36 | 37 | Optional last = createLink(endopoint, page.getTotalPages(), page.getSize(), Link.REL_LAST, links, clazz); 38 | last.ifPresent(resources::add); 39 | 40 | return resources; 41 | 42 | } 43 | 44 | protected static Optional createLink(String endpoint, int page, int size, String rel, EntityLinks links, Class clazz) { 45 | 46 | if (Objects.nonNull(page)) { 47 | String pageAndSize = PAGE + page + SIZE + size; 48 | return Optional.of(links.linkFor(clazz).slash(endpoint + pageAndSize).withRel(rel)); 49 | } 50 | 51 | return Optional.empty(); 52 | 53 | } 54 | 55 | protected static Optional createLink(String endpoint, Pageable pageable, String rel, EntityLinks links, Class clazz) { 56 | 57 | if (Objects.nonNull(pageable)) { 58 | String pageAndSize = PAGE + pageable.getPageNumber() + SIZE + pageable.getPageSize(); 59 | return Optional.of(links.linkFor(clazz).slash(endpoint + pageAndSize).withRel(rel)); 60 | } 61 | 62 | return Optional.empty(); 63 | 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/load/TemisCron.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.sjcdigital.temis.load; 5 | 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.scheduling.annotation.Scheduled; 8 | import org.springframework.stereotype.Component; 9 | 10 | import com.sjcdigital.temis.model.service.bots.BotService; 11 | 12 | /** 13 | * @author pedro-hos 14 | * Executa a cada mês, no dia 01, a busca por novos registros 15 | * 16 | */ 17 | @Component 18 | public class TemisCron { 19 | 20 | @Autowired 21 | private BotService service; 22 | 23 | @Scheduled(cron = "0 0 0 1 * ?") 24 | public void run() { 25 | service.run(); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/load/TemisStarter.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.load; 2 | 3 | import javax.annotation.PostConstruct; 4 | 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.stereotype.Component; 7 | 8 | import com.sjcdigital.temis.model.service.bots.BotService; 9 | 10 | /** 11 | * 12 | * @author pedro-hos Apenas um starter para fazer a carga assim que o servidor subir. 13 | */ 14 | @Component 15 | public class TemisStarter { 16 | 17 | @Autowired 18 | private BotService service; 19 | 20 | @PostConstruct 21 | public void run() { 22 | service.run(); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/document/Alderman.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.document; 2 | 3 | import java.math.BigInteger; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | import org.apache.commons.lang3.StringUtils; 8 | import org.apache.commons.lang3.text.WordUtils; 9 | import org.springframework.data.annotation.Id; 10 | import org.springframework.data.mongodb.core.index.Indexed; 11 | import org.springframework.data.mongodb.core.mapping.Document; 12 | 13 | @Document 14 | public class Alderman { 15 | 16 | @Id 17 | private String id; 18 | 19 | @Indexed(unique = true) 20 | private String name; 21 | 22 | private String politicalParty; 23 | private String info; 24 | private String email; 25 | private String legislature; 26 | private String workplace; 27 | private String photo; 28 | private String phone; 29 | private Boolean notFound = false; 30 | private BigInteger lawsCount = BigInteger.ZERO; 31 | 32 | public Alderman() {} 33 | 34 | public Alderman(final String name, final Boolean notFound, final String photo) { 35 | this.photo = photo; 36 | this.name = name; 37 | this.notFound = notFound; 38 | } 39 | 40 | public String getName() { 41 | return name; 42 | } 43 | 44 | public void setName(final String name) { 45 | this.name = name; 46 | } 47 | 48 | public String getPoliticalParty() { 49 | return politicalParty; 50 | } 51 | 52 | public void setPoliticalParty(final String politicalParty) { 53 | this.politicalParty = politicalParty; 54 | } 55 | 56 | public String getInfo() { 57 | return info; 58 | } 59 | 60 | public void setInfo(final String info) { 61 | this.info = info; 62 | } 63 | 64 | public String getEmail() { 65 | return email; 66 | } 67 | 68 | public void setEmail(final String email) { 69 | this.email = email; 70 | } 71 | 72 | public String getLegislature() { 73 | return legislature; 74 | } 75 | 76 | public void setLegislature(final String legislature) { 77 | this.legislature = legislature; 78 | } 79 | 80 | public String getWorkplace() { 81 | return workplace; 82 | } 83 | 84 | public void setWorkplace(final String workplace) { 85 | this.workplace = workplace; 86 | } 87 | 88 | public String getPhone() { 89 | return phone; 90 | } 91 | 92 | public void setPhone(final String phone) { 93 | this.phone = phone; 94 | } 95 | 96 | public String getPhoto() { 97 | return photo; 98 | } 99 | 100 | public void setPhoto(final String photo) { 101 | this.photo = photo; 102 | } 103 | 104 | public String getId() { 105 | return id; 106 | } 107 | 108 | public void setId(final String id) { 109 | this.id = id; 110 | } 111 | 112 | public Boolean getNotFound() { 113 | return notFound; 114 | } 115 | 116 | public void setNotFound(final Boolean notFound) { 117 | this.notFound = notFound; 118 | } 119 | 120 | public void plusLaw() { 121 | this.setLawsCount(getLawsCount().add(BigInteger.ONE)); 122 | } 123 | 124 | public BigInteger getLawsCount() { 125 | return lawsCount; 126 | } 127 | 128 | public void setLawsCount(BigInteger lawsCount) { 129 | this.lawsCount = lawsCount; 130 | } 131 | 132 | public static String normalizeName(final String name) { 133 | String newName = name.trim(); 134 | newName = newName.toLowerCase(); 135 | newName = StringUtils.normalizeSpace(newName); 136 | newName = WordUtils.capitalize(newName); 137 | newName = normalizeNameCharacters(newName); 138 | 139 | return newName; 140 | } 141 | 142 | private static String normalizeNameCharacters(final String name) { 143 | final Map specialChars = new HashMap<>(); 144 | specialChars.put("ª", "a"); 145 | specialChars.put("-", ""); 146 | 147 | String newName = name; 148 | for (Map.Entry e : specialChars.entrySet()) { 149 | newName = newName.replace(e.getKey(), e.getValue()); 150 | } 151 | 152 | return newName; 153 | } 154 | 155 | @Override 156 | public String toString() { 157 | return "{ " + "name: " + name + ", politicalParty: " + politicalParty + ", info:" + info + ", email: " + email 158 | + ", legislature: " + legislature + ", workplace: " + workplace + ", phone: " + phone + ", photo: " 159 | + photo + "}"; 160 | 161 | } 162 | 163 | } 164 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/document/AldermanSurrogate.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.document; 2 | 3 | /** 4 | * @author fabiohbarbosa 5 | */ 6 | public class AldermanSurrogate { 7 | 8 | private String justification; 9 | private String surrogate; 10 | private boolean surrogatePresent; 11 | 12 | public AldermanSurrogate() {} 13 | 14 | public AldermanSurrogate(final String justification, final String surrogate, final boolean surrogatePresent) { 15 | this.justification = justification; 16 | this.surrogate = surrogate; 17 | this.surrogatePresent = surrogatePresent; 18 | } 19 | 20 | public String getJustification() { 21 | return justification; 22 | } 23 | 24 | public void setJustification(final String justification) { 25 | this.justification = justification; 26 | } 27 | 28 | public String getSurrogate() { 29 | return surrogate; 30 | } 31 | 32 | public void setSurrogate(final String surrogate) { 33 | this.surrogate = surrogate; 34 | } 35 | 36 | public boolean isSurrogatePresent() { 37 | return surrogatePresent; 38 | } 39 | 40 | public void setSurrogatePresent(final boolean surrogatePresent) { 41 | this.surrogatePresent = surrogatePresent; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/document/Law.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.document; 2 | 3 | import java.math.BigInteger; 4 | import java.time.LocalDate; 5 | import java.util.Collection; 6 | 7 | import org.springframework.data.annotation.Id; 8 | import org.springframework.data.mongodb.core.index.Indexed; 9 | import org.springframework.data.mongodb.core.index.TextIndexed; 10 | import org.springframework.data.mongodb.core.mapping.DBRef; 11 | import org.springframework.data.mongodb.core.mapping.Document; 12 | import org.springframework.data.mongodb.core.mapping.Field; 13 | 14 | import com.fasterxml.jackson.annotation.JsonFormat; 15 | 16 | @Document 17 | public class Law { 18 | 19 | @Id 20 | private String id; 21 | 22 | @Indexed(unique = true) 23 | private String code; 24 | 25 | @DBRef 26 | private Collection author; 27 | 28 | @TextIndexed 29 | private String desc; 30 | 31 | private String summary; 32 | 33 | private String type; 34 | 35 | @Indexed 36 | @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "dd-MM-yyyy") 37 | private LocalDate date; 38 | 39 | private String title; 40 | 41 | @Field(value = "PLNumber") 42 | private String projectLawNumber; 43 | 44 | private BigInteger votesCount = BigInteger.ZERO; 45 | private BigInteger rating = BigInteger.ZERO; 46 | 47 | public void addVote() { 48 | this.votesCount = this.votesCount.add(BigInteger.ONE); 49 | } 50 | 51 | public void addRating(BigInteger rating) { 52 | this.rating = this.rating.add(rating).divide(votesCount); 53 | } 54 | 55 | public String getCode() { 56 | return code; 57 | } 58 | 59 | public void setCode(final String lawId) { 60 | code = lawId; 61 | } 62 | 63 | public Collection getAuthor() { 64 | return author; 65 | } 66 | 67 | public void setAuthor(final Collection author) { 68 | this.author = author; 69 | } 70 | 71 | public String getDesc() { 72 | return desc; 73 | } 74 | 75 | public void setDesc(final String description) { 76 | desc = description; 77 | } 78 | 79 | public LocalDate getDate() { 80 | return date; 81 | } 82 | 83 | public void setDate(final LocalDate date) { 84 | this.date = date; 85 | } 86 | 87 | public String getTitle() { 88 | return title; 89 | } 90 | 91 | public void setTitle(final String title) { 92 | this.title = title; 93 | } 94 | 95 | public String getProjectLawNumber() { 96 | return projectLawNumber; 97 | } 98 | 99 | public void setProjectLawNumber(final String projectLawNumber) { 100 | this.projectLawNumber = projectLawNumber; 101 | } 102 | 103 | public String getSummary() { 104 | return summary; 105 | } 106 | 107 | public void setSummary(final String summary) { 108 | this.summary = summary; 109 | } 110 | 111 | public BigInteger getVotesCount() { 112 | return votesCount; 113 | } 114 | 115 | public void setVotesCount(final BigInteger votesTotal) { 116 | this.votesCount = votesTotal; 117 | } 118 | 119 | public BigInteger getRating() { 120 | return rating; 121 | } 122 | 123 | public void setRating(final BigInteger rating) { 124 | this.rating = rating; 125 | } 126 | 127 | public String getType() { 128 | return type; 129 | } 130 | 131 | public void setType(String type) { 132 | this.type = type; 133 | } 134 | 135 | } 136 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/document/OrdinarySession.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.document; 2 | 3 | import java.time.LocalDate; 4 | 5 | import org.springframework.data.annotation.Id; 6 | import org.springframework.data.mongodb.core.mapping.DBRef; 7 | import org.springframework.data.mongodb.core.mapping.Document; 8 | 9 | import com.fasterxml.jackson.annotation.JsonFormat; 10 | 11 | /** 12 | * @author fabiohbarbosa 13 | */ 14 | @Document 15 | public class OrdinarySession { 16 | 17 | @Id 18 | private String id; 19 | 20 | private Integer session; 21 | 22 | @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "dd-MM-yyyy") 23 | private LocalDate date; 24 | 25 | @DBRef 26 | private Alderman alderman; 27 | 28 | private Boolean isPresent; 29 | private AldermanSurrogate surrogate; 30 | 31 | public OrdinarySession() {} 32 | 33 | public OrdinarySession( final Integer session, final LocalDate date, final Alderman alderman, 34 | final Boolean isPresent, final AldermanSurrogate surrogate) { 35 | 36 | this.session = session; 37 | this.date = date; 38 | this.alderman = alderman; 39 | this.isPresent = isPresent; 40 | this.surrogate = surrogate; 41 | } 42 | 43 | public String getId() { 44 | return id; 45 | } 46 | 47 | public void setId(final String id) { 48 | this.id = id; 49 | } 50 | 51 | public Integer getSession() { 52 | return session; 53 | } 54 | 55 | public void setSession(final Integer session) { 56 | this.session = session; 57 | } 58 | 59 | public LocalDate getDate() { 60 | return date; 61 | } 62 | 63 | public void setDate(final LocalDate date) { 64 | this.date = date; 65 | } 66 | 67 | public Alderman getAlderman() { 68 | return alderman; 69 | } 70 | 71 | public void setAlderman(final Alderman alderman) { 72 | this.alderman = alderman; 73 | } 74 | 75 | public Boolean getPresent() { 76 | return isPresent; 77 | } 78 | 79 | public void setPresent(final Boolean present) { 80 | isPresent = present; 81 | } 82 | 83 | public AldermanSurrogate getSurrogate() { 84 | return surrogate; 85 | } 86 | 87 | public void setSurrogate(final AldermanSurrogate surrogate) { 88 | this.surrogate = surrogate; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/document/Type.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.sjcdigital.temis.model.document; 5 | 6 | /** 7 | * @author pedro-hos 8 | * 9 | */ 10 | public enum Type { 11 | 12 | NOMEACAO("Nomeação"), 13 | UTILIDADE_PUBLICA("Declaração de Utilidade Pública"), 14 | SAUDE("Saúde"), 15 | ACESSIBILIDADE("Acessibilidade"), 16 | DATA_COMEMORATIVA("Data Comemorativa"), 17 | CAUSAS_ANIMAIS("Causas Animais"), 18 | REGISTRO_CANCELADO("Registro Cancelado"), 19 | OUTRO("Outro"), 20 | SEM_CLASSIFICACAO("Sem Classificação"); 21 | 22 | private String type; 23 | 24 | Type(String type) { 25 | this.type = type; 26 | } 27 | 28 | public String getType() { 29 | return this.type; 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/enums/AldermanPresenceSheet.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.enums; 2 | 3 | /** 4 | * @author fabiohbarbosa 5 | */ 6 | public enum AldermanPresenceSheet { 7 | FIRST_ROW(14), LAST_ROW(35), 8 | SESSION_ROW(10), SESSION_COLUMN(0), 9 | NAME_COLUMN(0), PRESENT_COLUMN(1), 10 | SURROGATE_JUSTIFICATION(2), SURROGATE_NAME_COLUMN(3), SURROGATE_PRESENT_COLUMN(4); 11 | 12 | public int NUM; 13 | 14 | AldermanPresenceSheet(int NUM) { 15 | this.NUM = NUM; 16 | } 17 | } -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/exceptions/BotException.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.exceptions; 2 | 3 | /** 4 | * 5 | * @author pedro-hos 6 | * 7 | */ 8 | public class BotException extends Exception { 9 | 10 | private static final long serialVersionUID = 1L; 11 | 12 | public BotException(final Throwable exec) { 13 | super("Error Unknown during to convert of the results.", exec); 14 | } 15 | 16 | public BotException() { 17 | super("Error Unknown during to convert of the results."); 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/repositories/AldermanRepository.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.repositories; 2 | 3 | import java.util.Optional; 4 | 5 | import org.springframework.data.mongodb.repository.MongoRepository; 6 | 7 | import com.sjcdigital.temis.model.document.Alderman; 8 | 9 | /** 10 | * @author pedro-hos 11 | */ 12 | public interface AldermanRepository extends MongoRepository { 13 | 14 | Optional findByName(final String name); 15 | Optional findByNameContainingIgnoreCase(final String name); 16 | 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/repositories/LawsRepository.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.repositories; 2 | 3 | import java.util.Optional; 4 | 5 | import org.springframework.data.domain.Page; 6 | import org.springframework.data.domain.Pageable; 7 | import org.springframework.data.mongodb.repository.Query; 8 | import org.springframework.data.repository.PagingAndSortingRepository; 9 | 10 | import com.sjcdigital.temis.model.document.Law; 11 | 12 | /** 13 | * @author pedro-hos 14 | */ 15 | public interface LawsRepository extends PagingAndSortingRepository { 16 | 17 | Optional findByCode(final String code); 18 | Optional findFirstByOrderByCodeDesc(); 19 | 20 | @Query("{'author' :{'$ref' : 'alderman' , '$id' : ?0}}") 21 | Page findByAuthorId(final String id, final Pageable page); 22 | 23 | Optional> findAllByOrderByCodeDesc(final Pageable page); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/repositories/OrdinarySessionRepository.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.repositories; 2 | 3 | import com.sjcdigital.temis.model.document.Alderman; 4 | import com.sjcdigital.temis.model.document.OrdinarySession; 5 | import org.springframework.data.mongodb.repository.MongoRepository; 6 | 7 | import java.time.LocalDate; 8 | 9 | /** 10 | * @author fabiohbarbosa 11 | */ 12 | public interface OrdinarySessionRepository extends MongoRepository { 13 | Integer countBySessionAndDateAndAlderman(int session, LocalDate date, Alderman alderman); 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/bots/AbstractBot.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.service.bots; 2 | 3 | import org.jsoup.Jsoup; 4 | import org.jsoup.nodes.Document; 5 | import org.springframework.scheduling.annotation.Async; 6 | import org.springframework.scheduling.annotation.AsyncResult; 7 | 8 | import java.io.IOException; 9 | import java.io.InputStream; 10 | import java.net.URL; 11 | import java.net.URLConnection; 12 | import java.util.concurrent.Future; 13 | 14 | /** 15 | * @author pedro-hos 16 | * 17 | */ 18 | public abstract class AbstractBot implements Bot { 19 | 20 | private static final String AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) " 21 | + "Chrome/33.0.1750.152 Safari/537.36"; 22 | 23 | @Async 24 | protected Future getPage(final String url) throws IOException { 25 | 26 | final URLConnection urlConnection = new URL(url).openConnection(); 27 | urlConnection.addRequestProperty("User-Agent", AGENT); 28 | 29 | final InputStream openStream = urlConnection.getInputStream(); 30 | final Document page = Jsoup.parse(openStream, "ISO-8859-9", url); 31 | 32 | return new AsyncResult(page); 33 | } 34 | 35 | protected abstract String getPath(); 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/bots/Bot.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.service.bots; 2 | 3 | import com.sjcdigital.temis.model.exceptions.BotException; 4 | 5 | /** 6 | * @author Rafael Peretta 7 | * 8 | * Interface responsável por definir a operação que será executado por um bot. 9 | * 10 | */ 11 | public interface Bot { 12 | 13 | void saveData() throws BotException; 14 | 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/bots/BotService.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.sjcdigital.temis.model.service.bots; 5 | 6 | import org.apache.commons.lang3.exception.ExceptionUtils; 7 | import org.apache.log4j.LogManager; 8 | import org.apache.log4j.Logger; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.scheduling.annotation.Async; 11 | import org.springframework.stereotype.Service; 12 | 13 | import com.sjcdigital.temis.model.exceptions.BotException; 14 | import com.sjcdigital.temis.model.service.bots.impl.CompositeBot; 15 | 16 | /** 17 | * @author pedro-hos 18 | */ 19 | 20 | @Service 21 | public class BotService { 22 | 23 | private final Logger LOGGER = LogManager.getLogger(this.getClass()); 24 | 25 | @Autowired 26 | private CompositeBot compositeBot; 27 | 28 | @Async 29 | public void run() { 30 | 31 | try { 32 | 33 | compositeBot.saveData(); 34 | 35 | } catch (final BotException e) { 36 | LOGGER.error(ExceptionUtils.getStackTrace(e)); 37 | } 38 | 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/bots/impl/AldermanPresenceBot.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.service.bots.impl; 2 | 3 | import java.io.IOException; 4 | import java.time.LocalDate; 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | import java.util.Optional; 8 | import java.util.concurrent.ExecutionException; 9 | import java.util.stream.Collectors; 10 | 11 | import org.apache.commons.lang3.exception.ExceptionUtils; 12 | import org.apache.log4j.LogManager; 13 | import org.apache.log4j.Logger; 14 | import org.jsoup.nodes.Document; 15 | import org.jsoup.select.Elements; 16 | import org.springframework.beans.factory.annotation.Autowired; 17 | import org.springframework.beans.factory.annotation.Value; 18 | import org.springframework.core.annotation.Order; 19 | import org.springframework.stereotype.Component; 20 | 21 | import com.sjcdigital.temis.model.exceptions.BotException; 22 | import com.sjcdigital.temis.model.service.bots.AbstractBot; 23 | import com.sjcdigital.temis.util.TemisFileUtil; 24 | 25 | /** 26 | * @author fabiohbarbosa 27 | */ 28 | @Component 29 | @Order(3) 30 | public class AldermanPresenceBot extends AbstractBot { 31 | 32 | private static final Logger LOGGER = LogManager.getLogger(AldermanPresenceBot.class); 33 | 34 | @Value("${url.alderman-presence}") 35 | private String aldermanPresenceUrl; 36 | 37 | @Value("${path.leis}") 38 | private String path; 39 | 40 | @Autowired 41 | private TemisFileUtil file; 42 | 43 | @Override 44 | public void saveData() throws BotException { 45 | getXLSFiles() 46 | .forEach((date, url) -> file.createXLSFile(getPath(), url, date, LocalDate.now().getYear())); 47 | } 48 | 49 | private Map getXLSFiles() throws BotException { 50 | 51 | final Map urlFiles = new HashMap<>(); 52 | 53 | try { 54 | 55 | final Document document = Optional.ofNullable(getPage(aldermanPresenceUrl).get()).orElseThrow(BotException::new); 56 | final Elements divsPresenca = document.getElementsByClass("presenca"); 57 | 58 | divsPresenca.stream() 59 | .collect(Collectors.toMap(e -> e.select("time").attr("datetime"), e -> e.select("a").attr("href"))) 60 | .forEach(urlFiles::put); 61 | 62 | } catch (InterruptedException | ExecutionException | IOException exception) { 63 | LOGGER.error(ExceptionUtils.getStackTrace(exception)); 64 | } 65 | 66 | return urlFiles; 67 | 68 | } 69 | 70 | @Override 71 | protected String getPath() { 72 | return path.concat("presencas/"); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/bots/impl/AldermenBot.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.service.bots.impl; 2 | 3 | import java.io.IOException; 4 | import java.time.LocalDate; 5 | import java.util.Collection; 6 | import java.util.HashSet; 7 | import java.util.Optional; 8 | import java.util.concurrent.ExecutionException; 9 | 10 | import org.apache.commons.lang3.exception.ExceptionUtils; 11 | import org.apache.log4j.LogManager; 12 | import org.apache.log4j.Logger; 13 | import org.jsoup.nodes.Document; 14 | import org.jsoup.nodes.Element; 15 | import org.jsoup.select.Elements; 16 | import org.springframework.beans.factory.annotation.Autowired; 17 | import org.springframework.beans.factory.annotation.Value; 18 | import org.springframework.core.annotation.Order; 19 | import org.springframework.stereotype.Component; 20 | 21 | import com.sjcdigital.temis.model.exceptions.BotException; 22 | import com.sjcdigital.temis.model.service.bots.AbstractBot; 23 | import com.sjcdigital.temis.util.TemisFileUtil; 24 | 25 | /** 26 | * 27 | * @author pedro-hos 28 | * 29 | */ 30 | 31 | @Component 32 | @Order(1) 33 | public class AldermenBot extends AbstractBot { 34 | 35 | private static final Logger LOGGER = LogManager.getLogger(AldermenBot.class); 36 | 37 | @Value("${url.aldermen}") 38 | private String aldermenUrl; 39 | 40 | @Value("${path.leis}") 41 | private String path; 42 | 43 | @Autowired 44 | private TemisFileUtil file; 45 | 46 | @Override 47 | public void saveData() throws BotException { 48 | 49 | final Collection allLinks = getAldermenLinks(); 50 | 51 | try { 52 | 53 | for (final String link : allLinks) { 54 | final Document document = Optional.ofNullable(getPage(link).get()).orElseThrow(BotException::new); 55 | file.createHTMLFile(getPath(), document.html(), getFileName(link), LocalDate.now().getYear()); 56 | } 57 | 58 | } catch (IOException | InterruptedException | ExecutionException exception) { 59 | LOGGER.error(ExceptionUtils.getStackTrace(exception)); 60 | } 61 | 62 | } 63 | 64 | private Collection getAldermenLinks() throws BotException { 65 | 66 | final Collection links = new HashSet<>(); 67 | 68 | try { 69 | 70 | final Document document = Optional.ofNullable(getPage(aldermenUrl).get()).orElseThrow(BotException::new); 71 | final Elements divsBack = document.getElementsByClass("back"); //
73 | 74 | for (final Element element : divsBack) { 75 | element.select("a").stream().map(l -> l.attr("href")).forEach(links::add); // 77 | } 78 | 79 | } catch (IOException | InterruptedException | ExecutionException exception) { 80 | LOGGER.error(ExceptionUtils.getStackTrace(exception)); 81 | } 82 | 83 | return links; 84 | } 85 | 86 | private String getFileName(final String link) { 87 | final String[] split = link.split("/"); 88 | return split[split.length - 1]; 89 | } 90 | 91 | @Override 92 | protected String getPath() { 93 | return path.concat("vereadores/"); 94 | } 95 | 96 | } 97 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/bots/impl/CompositeBot.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.service.bots.impl; 2 | 3 | import java.util.List; 4 | 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.stereotype.Component; 7 | 8 | import com.sjcdigital.temis.model.exceptions.BotException; 9 | import com.sjcdigital.temis.model.service.bots.Bot; 10 | 11 | /** 12 | * @author Rafael Peretta 13 | * 14 | * Classe utilizada para agrupar os bots e executá-los. 15 | */ 16 | @Component 17 | public class CompositeBot implements Bot { 18 | 19 | @Autowired 20 | private List bots; 21 | 22 | @Override 23 | public void saveData() throws BotException { 24 | 25 | for (Bot bot : bots) { 26 | bot.saveData(); 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/bots/impl/LawsBot.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.service.bots.impl; 2 | 3 | import java.io.FileNotFoundException; 4 | import java.io.IOException; 5 | import java.math.BigInteger; 6 | import java.time.LocalDate; 7 | import java.util.LinkedList; 8 | import java.util.List; 9 | import java.util.Optional; 10 | import java.util.concurrent.ExecutionException; 11 | 12 | import org.apache.commons.lang3.StringUtils; 13 | import org.apache.commons.lang3.exception.ExceptionUtils; 14 | import org.apache.log4j.LogManager; 15 | import org.apache.log4j.Logger; 16 | import org.springframework.beans.factory.annotation.Autowired; 17 | import org.springframework.beans.factory.annotation.Value; 18 | import org.springframework.core.annotation.Order; 19 | import org.springframework.stereotype.Component; 20 | 21 | import com.sjcdigital.temis.model.document.Law; 22 | import com.sjcdigital.temis.model.exceptions.BotException; 23 | import com.sjcdigital.temis.model.repositories.LawsRepository; 24 | import com.sjcdigital.temis.model.service.bots.AbstractBot; 25 | import com.sjcdigital.temis.util.TemisFileUtil; 26 | 27 | /** 28 | * 29 | * @author pedro-hos 30 | * 31 | * Classe responsável por pegar as páginas das leis em: http://www.ceaam.net/sjc/legislacao/index.php, 32 | * e salvá-las em uma pasta para realização de parse posteriormente. 33 | * 34 | */ 35 | @Component 36 | @Order(2) 37 | public class LawsBot extends AbstractBot { 38 | 39 | private static final Logger LOGGER = LogManager.getLogger(LawsBot.class); 40 | 41 | @Value("${url.laws}") 42 | private String lawsUrl; 43 | 44 | @Value("${path.leis}") 45 | private String path; 46 | 47 | @Value("${year.start.law.extract}") 48 | private int initialYear; 49 | 50 | @Value("${code.start.law.extract}") 51 | private String initialCode; 52 | 53 | @Autowired 54 | private TemisFileUtil file; 55 | 56 | @Autowired 57 | private LawsRepository lawsRepository; 58 | 59 | @Override 60 | public void saveData() throws BotException { 61 | 62 | final List allYears = getAllYears(); 63 | 64 | String code = getInitialCode(); 65 | String url = StringUtils.EMPTY; 66 | String body = StringUtils.EMPTY; 67 | 68 | boolean tryNextYear = false; 69 | 70 | final int currentYear = LocalDate.now().getYear(); 71 | int index = 0; 72 | int limitToTry = 10; 73 | Integer year = null; 74 | 75 | while (index != allYears.size()) { 76 | 77 | try { 78 | 79 | year = allYears.get(index); 80 | url = buildURL(year, code); 81 | body = getPage(url).get().html(); 82 | 83 | file.createHTMLFile(getPath(), body, code, year); 84 | code = buildLawCode(getNextLawCode(code)); 85 | tryNextYear = false; 86 | limitToTry = 10; 87 | 88 | } catch (InterruptedException | ExecutionException | IOException exception) { 89 | 90 | if (exception instanceof FileNotFoundException) { 91 | 92 | LOGGER.error("Error 404: " + url); 93 | 94 | if (year != currentYear && !tryNextYear) { 95 | index++; 96 | tryNextYear = true; 97 | 98 | } else { 99 | 100 | if (limitToTry == 0) { 101 | break; 102 | } 103 | 104 | if (year != currentYear) { 105 | index--; 106 | } 107 | 108 | limitToTry--; 109 | code = buildLawCode(getNextLawCode(code)); 110 | } 111 | 112 | } else { 113 | LOGGER.error(ExceptionUtils.getStackTrace(exception)); 114 | throw new BotException(exception); 115 | } 116 | 117 | } 118 | 119 | } 120 | 121 | } 122 | 123 | private String getInitialCode() { 124 | final Optional lastLaw = lawsRepository.findFirstByOrderByCodeDesc(); 125 | return lastLaw.isPresent() ? buildLawCode(getNextLawCode(lastLaw.get().getCode())) : initialCode; //L8865 is the last code of 2012 126 | } 127 | 128 | private List getAllYears() { 129 | 130 | final List years = new LinkedList<>(); 131 | final int currentYear = LocalDate.now().getYear(); 132 | 133 | if (lawsRepository.count() != 0) { 134 | years.add(currentYear); 135 | 136 | } else { 137 | 138 | while (initialYear <= currentYear) { 139 | years.add(initialYear); 140 | initialYear += 1; 141 | } 142 | 143 | } 144 | 145 | return years; 146 | 147 | } 148 | 149 | private String buildURL(final Integer year, final String code) { 150 | return lawsUrl.concat(year.toString()).concat("/").concat(code).concat(".htm"); 151 | } 152 | 153 | private BigInteger getNextLawCode(final String current) { 154 | final BigInteger nextLawCode = new BigInteger(current.replace("L", "")).add(BigInteger.ONE); 155 | return nextLawCode; 156 | } 157 | 158 | private String buildLawCode(final BigInteger code) { 159 | return "L" + StringUtils.leftPad(code.toString(), 4, "0"); 160 | } 161 | 162 | @Override 163 | protected String getPath() { 164 | return path.concat("leis/"); 165 | } 166 | 167 | } 168 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/camel/processor/AbstractProcessor.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.service.camel.processor; 2 | 3 | import com.sjcdigital.temis.model.service.parsers.AbstractParser; 4 | import org.apache.camel.Exchange; 5 | import org.apache.camel.Processor; 6 | 7 | import java.io.File; 8 | 9 | /** 10 | * @author fabiohbarbosa 11 | */ 12 | public abstract class AbstractProcessor implements Processor { 13 | 14 | @Override 15 | public void process(final Exchange exchange) throws Exception { 16 | final File file = exchange.getIn().getMandatoryBody(File.class); 17 | getParser().parse(file); 18 | } 19 | 20 | public abstract AbstractParser getParser(); 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/camel/processor/impl/AldermanPresenceProcessor.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.service.camel.processor.impl; 2 | 3 | import com.sjcdigital.temis.model.service.camel.processor.AbstractProcessor; 4 | import com.sjcdigital.temis.model.service.parsers.AbstractParser; 5 | import com.sjcdigital.temis.model.service.parsers.impl.AldermanPresenceParser; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.stereotype.Component; 8 | 9 | /** 10 | * @author fabiohbarbosa 11 | */ 12 | @Component 13 | public class AldermanPresenceProcessor extends AbstractProcessor { 14 | 15 | @Autowired 16 | private AldermanPresenceParser parser; 17 | 18 | @Override 19 | public AbstractParser getParser() { 20 | return parser; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/camel/processor/impl/AldermanProcessor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.sjcdigital.temis.model.service.camel.processor.impl; 5 | 6 | import com.sjcdigital.temis.model.service.camel.processor.AbstractProcessor; 7 | import com.sjcdigital.temis.model.service.parsers.AbstractParser; 8 | import com.sjcdigital.temis.model.service.parsers.impl.AldermenParser; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.stereotype.Component; 11 | 12 | /** 13 | * @author pedro-hos 14 | * 15 | */ 16 | @Component 17 | public class AldermanProcessor extends AbstractProcessor { 18 | 19 | @Autowired 20 | private AldermenParser parser; 21 | 22 | 23 | @Override 24 | public AbstractParser getParser() { 25 | return parser; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/camel/processor/impl/LawsProcessor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.sjcdigital.temis.model.service.camel.processor.impl; 5 | 6 | import com.sjcdigital.temis.model.service.camel.processor.AbstractProcessor; 7 | import com.sjcdigital.temis.model.service.parsers.AbstractParser; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.stereotype.Component; 10 | 11 | import com.sjcdigital.temis.model.service.parsers.impl.LawsParser; 12 | 13 | /** 14 | * @author pedro-hos 15 | */ 16 | 17 | @Component 18 | public class LawsProcessor extends AbstractProcessor { 19 | 20 | @Autowired 21 | private LawsParser parser; 22 | 23 | 24 | @Override 25 | public AbstractParser getParser() { 26 | return parser; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/camel/route/AbstractRoute.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.service.camel.route; 2 | 3 | import org.apache.camel.builder.RouteBuilder; 4 | import org.springframework.beans.factory.annotation.Value; 5 | 6 | /** 7 | * @author pedro-hos 8 | */ 9 | public abstract class AbstractRoute extends RouteBuilder { 10 | 11 | protected static final String FILE = "file://"; 12 | 13 | private static final String DELAY = "delay=30s"; 14 | private static final String RECURSIVE = "recursive=true"; 15 | private static final String DELETE = "delete=true"; 16 | private static final String AND = "&"; 17 | 18 | @Value("${path.leis}") 19 | protected String path; 20 | 21 | protected String options() { 22 | return "?".concat(DELAY).concat(AND).concat(RECURSIVE).concat(AND).concat(DELETE); 23 | } 24 | 25 | protected abstract String path(); 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/camel/route/impl/AldermanPresenceRoute.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.service.camel.route.impl; 2 | 3 | import com.sjcdigital.temis.model.service.camel.processor.impl.AldermanPresenceProcessor; 4 | import com.sjcdigital.temis.model.service.camel.route.AbstractRoute; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.stereotype.Component; 7 | 8 | /** 9 | * @author fabiohbarbosa 10 | */ 11 | @Component 12 | public class AldermanPresenceRoute extends AbstractRoute { 13 | 14 | @Autowired 15 | private AldermanPresenceProcessor processor; 16 | 17 | @Override 18 | public void configure() throws Exception { 19 | from(FILE + path() + options()).process(processor); 20 | } 21 | 22 | @Override 23 | protected String path() { 24 | return path.concat("presencas/"); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/camel/route/impl/AldermanRoute.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.service.camel.route.impl; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.stereotype.Component; 5 | 6 | import com.sjcdigital.temis.model.service.camel.processor.impl.AldermanProcessor; 7 | import com.sjcdigital.temis.model.service.camel.route.AbstractRoute; 8 | 9 | /** 10 | * @author pedro-hos 11 | */ 12 | 13 | @Component 14 | public class AldermanRoute extends AbstractRoute { 15 | 16 | @Autowired 17 | private AldermanProcessor processor; 18 | 19 | @Override 20 | public void configure() throws Exception { 21 | from(FILE + path() + options()).process(processor); 22 | } 23 | 24 | @Override 25 | protected String path() { 26 | return path + "vereadores/"; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/camel/route/impl/LawsRoute.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.service.camel.route.impl; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.stereotype.Component; 5 | 6 | import com.sjcdigital.temis.model.service.camel.processor.impl.LawsProcessor; 7 | import com.sjcdigital.temis.model.service.camel.route.AbstractRoute; 8 | 9 | /** 10 | * @author pedro-hos 11 | */ 12 | 13 | @Component 14 | public class LawsRoute extends AbstractRoute { 15 | 16 | @Autowired 17 | private LawsProcessor processor; 18 | 19 | @Override 20 | public void configure() throws Exception { 21 | from(FILE + path() + options()).process(processor); 22 | } 23 | 24 | @Override 25 | protected String path() { 26 | return path + "leis/"; 27 | } 28 | 29 | @Override 30 | protected String options() { 31 | return "?delay=30s&recursive=true&delete=true"; 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/machine_learn/ClassifyLaw.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.service.machine_learn; 2 | 3 | import java.io.FileInputStream; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.nio.file.Paths; 7 | import java.util.Objects; 8 | 9 | import org.apache.commons.lang3.exception.ExceptionUtils; 10 | import org.apache.log4j.LogManager; 11 | import org.apache.log4j.Logger; 12 | import org.springframework.beans.factory.annotation.Value; 13 | import org.springframework.stereotype.Service; 14 | 15 | import com.sjcdigital.temis.model.document.Type; 16 | 17 | import opennlp.tools.doccat.DoccatModel; 18 | import opennlp.tools.doccat.DocumentCategorizerME; 19 | 20 | /** 21 | * 22 | * @author pedro-hos 23 | * 24 | */ 25 | @Service 26 | public class ClassifyLaw { 27 | 28 | private static final Logger LOGGER = LogManager.getLogger(ClassifyLaw.class); 29 | 30 | @Value("${path.machine-learn.train}") 31 | private String train; 32 | 33 | @Value("${path.machine-learn.bin}") 34 | private String bin; 35 | 36 | public String classify(String summary) { 37 | 38 | InputStream inputStrean = null; 39 | 40 | try { 41 | 42 | inputStrean = new FileInputStream(Paths.get(bin).toFile()); 43 | DoccatModel doccatModel = new DoccatModel(inputStrean); 44 | DocumentCategorizerME myCategorizer = new DocumentCategorizerME(doccatModel); 45 | double[] outcomes = myCategorizer.categorize(summary); 46 | String category = myCategorizer.getBestCategory(outcomes); 47 | 48 | LOGGER.info(category); 49 | 50 | return Type.valueOf(category).getType(); 51 | 52 | } catch (IOException e) { 53 | LOGGER.error(ExceptionUtils.getStackTrace(e)); 54 | return Type.SEM_CLASSIFICACAO.getType(); 55 | 56 | } finally { 57 | if (Objects.nonNull(inputStrean)) { 58 | close(inputStrean); 59 | } 60 | } 61 | 62 | } 63 | 64 | private void close(InputStream is) { 65 | try { 66 | is.close(); 67 | } catch (IOException e) { 68 | LOGGER.error(ExceptionUtils.getStackTrace(e)); 69 | } 70 | 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/machine_learn/Train.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.service.machine_learn; 2 | 3 | import java.io.BufferedOutputStream; 4 | import java.io.FileOutputStream; 5 | import java.io.IOException; 6 | import java.io.OutputStream; 7 | import java.nio.file.Paths; 8 | import java.util.Objects; 9 | 10 | import javax.annotation.PostConstruct; 11 | 12 | import org.apache.commons.lang3.exception.ExceptionUtils; 13 | import org.apache.log4j.LogManager; 14 | import org.apache.log4j.Logger; 15 | import org.springframework.beans.factory.annotation.Value; 16 | import org.springframework.stereotype.Component; 17 | 18 | import opennlp.tools.doccat.DoccatModel; 19 | import opennlp.tools.doccat.DocumentCategorizerME; 20 | import opennlp.tools.doccat.DocumentSample; 21 | import opennlp.tools.doccat.DocumentSampleStream; 22 | import opennlp.tools.util.InputStreamFactory; 23 | import opennlp.tools.util.MarkableFileInputStreamFactory; 24 | import opennlp.tools.util.ObjectStream; 25 | import opennlp.tools.util.PlainTextByLineStream; 26 | 27 | /** 28 | * @author pedro-hos 29 | */ 30 | @Component 31 | public class Train { 32 | 33 | private static final Logger LOGGER = LogManager.getLogger(Train.class); 34 | 35 | @Value("${path.machine-learn.train}") 36 | private String train; 37 | 38 | @Value("${path.machine-learn.bin}") 39 | private String bin; 40 | 41 | @PostConstruct 42 | @SuppressWarnings("deprecation") 43 | public void run() { 44 | 45 | DoccatModel model = null; 46 | OutputStream modelOut = null; 47 | 48 | try { 49 | 50 | // Ensinando a máquina 51 | InputStreamFactory dataIn = new MarkableFileInputStreamFactory(Paths.get(train).toFile()); 52 | ObjectStream lineStream = new PlainTextByLineStream(dataIn, "UTF-8"); 53 | ObjectStream sampleStream = new DocumentSampleStream(lineStream); 54 | model = DocumentCategorizerME.train("pt", sampleStream); 55 | 56 | // Escrevendo arquivo que ela aprendeu 57 | modelOut = new BufferedOutputStream(new FileOutputStream(Paths.get(bin).toFile())); 58 | model.serialize(modelOut); 59 | 60 | } catch (IOException e) { 61 | LOGGER.error(ExceptionUtils.getStackTrace(e)); 62 | } finally { 63 | if (Objects.nonNull(modelOut)) { 64 | closeOutputStream(modelOut); 65 | } 66 | } 67 | 68 | } 69 | 70 | private static void closeOutputStream(OutputStream modelOut) { 71 | try { 72 | modelOut.close(); 73 | } catch (IOException e) { 74 | LOGGER.error(ExceptionUtils.getStackTrace(e)); 75 | } 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/parsers/AbstractParser.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.service.parsers; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.util.concurrent.Future; 6 | 7 | import org.jsoup.Jsoup; 8 | import org.jsoup.nodes.Document; 9 | import org.springframework.scheduling.annotation.Async; 10 | import org.springframework.scheduling.annotation.AsyncResult; 11 | 12 | /** 13 | * @author pedro-hos 14 | */ 15 | public abstract class AbstractParser { 16 | 17 | public abstract void parse(File file); 18 | 19 | @Async 20 | protected Future readFile(final File file) throws IOException { 21 | final Document document = Jsoup.parse(file, "UTF-8"); 22 | return new AsyncResult(document); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/parsers/impl/AldermanPresenceParser.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.service.parsers.impl; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.time.LocalDate; 6 | import java.util.Optional; 7 | 8 | import org.apache.commons.io.FileUtils; 9 | import org.apache.commons.io.FilenameUtils; 10 | import org.apache.commons.lang3.exception.ExceptionUtils; 11 | import org.apache.log4j.LogManager; 12 | import org.apache.log4j.Logger; 13 | import org.apache.poi.hssf.usermodel.HSSFWorkbook; 14 | import org.apache.poi.ss.usermodel.Row; 15 | import org.apache.poi.ss.usermodel.Sheet; 16 | import org.apache.poi.ss.usermodel.Workbook; 17 | import org.springframework.beans.factory.annotation.Autowired; 18 | import org.springframework.beans.factory.annotation.Value; 19 | import org.springframework.stereotype.Component; 20 | 21 | import com.sjcdigital.temis.model.document.Alderman; 22 | import com.sjcdigital.temis.model.document.AldermanSurrogate; 23 | import com.sjcdigital.temis.model.document.OrdinarySession; 24 | import com.sjcdigital.temis.model.enums.AldermanPresenceSheet; 25 | import com.sjcdigital.temis.model.repositories.AldermanRepository; 26 | import com.sjcdigital.temis.model.repositories.OrdinarySessionRepository; 27 | import com.sjcdigital.temis.model.service.parsers.AbstractParser; 28 | 29 | /** 30 | * @author fabiohbarbosa 31 | */ 32 | 33 | @Component 34 | public class AldermanPresenceParser extends AbstractParser { 35 | 36 | private static final Logger LOGGER = LogManager.getLogger(AldermanPresenceParser.class); 37 | 38 | @Value("${path.images}") 39 | private String pathImages; 40 | 41 | @Value("${url.context}") 42 | private String urlContext; 43 | 44 | @Value("${politico.sem_foto}") 45 | private String noPhoto; 46 | 47 | @Autowired 48 | private AldermanRepository aldermanRepository; 49 | 50 | @Autowired 51 | private OrdinarySessionRepository sessionRepository; 52 | 53 | @Override 54 | public void parse(final File file) { 55 | 56 | LOGGER.debug(String.format("Starting file %s parse", file.getName())); 57 | 58 | try { 59 | 60 | final Workbook wb = new HSSFWorkbook(FileUtils.openInputStream(file)); 61 | final Sheet sheet = wb.getSheetAt(0); 62 | 63 | final int session = parseSession(wb.getSheetAt(0).getRow(AldermanPresenceSheet.SESSION_ROW.NUM)); 64 | 65 | for (int i = AldermanPresenceSheet.FIRST_ROW.NUM; i < AldermanPresenceSheet.LAST_ROW.NUM; i++) { 66 | final Row row = sheet.getRow(i); 67 | 68 | final LocalDate date = parseDate(file); 69 | final Alderman alderman = parseAlderman(row); 70 | final boolean aldermanPresent = parsePresent(row); 71 | final AldermanSurrogate surrogate = parseSurrogate(aldermanPresent, row); 72 | 73 | save(session, date, alderman, aldermanPresent, surrogate); 74 | } 75 | 76 | wb.close(); 77 | 78 | } catch (IOException e) { 79 | LOGGER.error(ExceptionUtils.getStackTrace(e)); 80 | 81 | } 82 | 83 | } 84 | 85 | private AldermanSurrogate parseSurrogate(final boolean isPresent, final Row row) { 86 | if (isPresent) { 87 | return null; 88 | } 89 | 90 | final String justification = row.getCell(AldermanPresenceSheet.SURROGATE_JUSTIFICATION.NUM).getStringCellValue().trim(); 91 | final String surrogate = row.getCell(AldermanPresenceSheet.SURROGATE_NAME_COLUMN.NUM).getStringCellValue().trim(); 92 | final boolean surrogatePresent = row.getCell(AldermanPresenceSheet.SURROGATE_PRESENT_COLUMN.NUM).getStringCellValue().trim().equalsIgnoreCase("SIM"); 93 | 94 | return new AldermanSurrogate(justification, surrogate, surrogatePresent); 95 | } 96 | 97 | private void save(final int session, final LocalDate date, final Alderman alderman, final boolean isPresent, final AldermanSurrogate surrogate) { 98 | 99 | final Integer count = sessionRepository.countBySessionAndDateAndAlderman(session, date, alderman); 100 | 101 | if (count == 0) { 102 | sessionRepository.save(new OrdinarySession(session, date, alderman, isPresent, surrogate)); 103 | LOGGER.info(String.format("Save '%s' in session '%sº %s'", alderman.getName(), session, date)); 104 | return; 105 | } 106 | 107 | LOGGER.debug(String.format("%s already in session '%sº %s'", alderman.getName(), session, date)); 108 | } 109 | 110 | private boolean parsePresent(final Row row) { 111 | return row.getCell(AldermanPresenceSheet.PRESENT_COLUMN.NUM) 112 | .getStringCellValue() 113 | .trim() 114 | .equalsIgnoreCase("SIM"); 115 | } 116 | 117 | private int parseSession(final Row row) { 118 | return Integer.parseInt(row.getCell(AldermanPresenceSheet.SESSION_COLUMN.NUM) 119 | .getStringCellValue() 120 | .split("-")[1].trim() 121 | .split("ª")[0].trim()); 122 | } 123 | 124 | private LocalDate parseDate(final File file) { 125 | return LocalDate.parse(FilenameUtils.removeExtension(file.getName())); 126 | } 127 | 128 | private Alderman parseAlderman(final Row row) { 129 | 130 | final String name = Alderman.normalizeName(row.getCell(AldermanPresenceSheet.NAME_COLUMN.NUM).getStringCellValue()); 131 | final Optional optAlderman = aldermanRepository.findByNameContainingIgnoreCase(name); 132 | 133 | return optAlderman.orElseGet(() -> { 134 | LOGGER.warn(String.format("Not found Alderman %s", name)); 135 | return aldermanRepository.save(new Alderman(name, true, urlContext.concat(pathImages).concat(noPhoto))); 136 | }); 137 | 138 | } 139 | 140 | } 141 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/parsers/impl/AldermenParser.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.service.parsers.impl; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.util.Optional; 6 | import java.util.concurrent.ExecutionException; 7 | 8 | import org.apache.commons.lang3.exception.ExceptionUtils; 9 | import org.apache.log4j.LogManager; 10 | import org.apache.log4j.Logger; 11 | import org.jsoup.nodes.Document; 12 | import org.jsoup.nodes.Element; 13 | import org.jsoup.select.Elements; 14 | import org.springframework.beans.factory.annotation.Autowired; 15 | import org.springframework.beans.factory.annotation.Value; 16 | import org.springframework.stereotype.Component; 17 | 18 | import com.sjcdigital.temis.model.document.Alderman; 19 | import com.sjcdigital.temis.model.repositories.AldermanRepository; 20 | import com.sjcdigital.temis.model.service.parsers.AbstractParser; 21 | import com.sjcdigital.temis.util.StringUtil; 22 | import com.sjcdigital.temis.util.TemisFileUtil; 23 | 24 | /** 25 | * @author pedro-hos 26 | */ 27 | 28 | @Component 29 | public class AldermenParser extends AbstractParser { 30 | 31 | private static final Logger LOGGER = LogManager.getLogger(AldermenParser.class); 32 | 33 | @Value("${url.context}") 34 | private String urlContext; 35 | 36 | @Value("${path.webapp}") 37 | private String pathWebapp; 38 | 39 | @Value("${path.images}") 40 | private String pathImages; 41 | 42 | @Autowired 43 | private AldermanRepository aldermanRepository; 44 | 45 | @Autowired 46 | private TemisFileUtil fileUtil; 47 | 48 | @Override 49 | public void parse(final File file) { 50 | 51 | try { 52 | 53 | final Document document = readFile(file).get(); 54 | final Alderman alderman = extractAldermenInfo(document.select("div.row.info")); //
55 | saveOrUpdate(alderman); 56 | 57 | } catch (InterruptedException | ExecutionException | IOException e) { 58 | LOGGER.error(ExceptionUtils.getStackTrace(e)); 59 | } 60 | 61 | } 62 | 63 | protected Alderman extractAldermenInfo(final Elements elements) { 64 | 65 | final Alderman alderman = new Alderman(); 66 | 67 | for (final Element element : elements) { 68 | 69 | final Elements elementsInfo = element.select("div.col-sm-7.texto"); 70 | 71 | String politicianName = elementsInfo.select("h3").text().trim(); 72 | alderman.setName(politicianName); 73 | alderman.setEmail(getElementValue(elementsInfo, "E-mail").nextElementSibling().text().trim()); 74 | alderman.setInfo(getElementValue(elementsInfo, "Dados Pessoais").nextElementSibling().text().trim()); 75 | alderman.setLegislature(getElementValue(elementsInfo, "Legislatura").nextSibling().toString().trim()); 76 | alderman.setPhone(getElementValue(elementsInfo, "Telefone").nextSibling().toString().trim()); 77 | alderman.setPoliticalParty(extractedPoliticalParty(elementsInfo).trim()); 78 | alderman.setWorkplace(getElementValue(elementsInfo, "Local de Trabalho").nextSibling().toString().trim()); 79 | alderman.setPhoto(createPhoto(element.getElementsByClass("img-responsive").attr("src").trim(), politicianName)); 80 | 81 | } 82 | 83 | return alderman; 84 | 85 | } 86 | 87 | protected String createPhoto(String url, String politicianName) { 88 | politicianName = StringUtil.unaccent(politicianName.toLowerCase().replaceAll(" ", "_")); 89 | String fullImagePath = pathImages.concat(politicianName).concat(".jpg"); 90 | fileUtil.savePhoto(url, pathWebapp.concat(fullImagePath)); 91 | return urlContext.concat(fullImagePath); 92 | } 93 | 94 | protected String extractedPoliticalParty(final Elements elements) { 95 | return elements.select("h3").first().nextElementSibling().text().replaceAll("Partido: ", ""); 96 | } 97 | 98 | protected Element getElementValue(final Elements elementsInfo, final String key) { 99 | return elementsInfo.select("h4:contains(" + key + ")").first(); 100 | } 101 | 102 | protected void saveOrUpdate(final Alderman aldermanToSave) { 103 | 104 | final Optional alderman = aldermanRepository.findByName(aldermanToSave.getName()); 105 | 106 | if (alderman.isPresent()) { 107 | aldermanToSave.setId(alderman.get().getId()); 108 | aldermanToSave.setLawsCount(alderman.get().getLawsCount()); 109 | aldermanRepository.save(aldermanToSave); 110 | 111 | } else { 112 | aldermanRepository.save(aldermanToSave); 113 | } 114 | 115 | } 116 | 117 | } 118 | -------------------------------------------------------------------------------- /src/main/java/com/sjcdigital/temis/model/service/parsers/impl/LawsParser.java: -------------------------------------------------------------------------------- 1 | package com.sjcdigital.temis.model.service.parsers.impl; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.time.LocalDate; 6 | import java.time.format.DateTimeFormatter; 7 | import java.time.format.DateTimeParseException; 8 | import java.util.Locale; 9 | import java.util.Objects; 10 | import java.util.Optional; 11 | import java.util.concurrent.ExecutionException; 12 | import java.util.regex.Matcher; 13 | 14 | import org.apache.commons.lang3.exception.ExceptionUtils; 15 | import org.apache.log4j.LogManager; 16 | import org.apache.log4j.Logger; 17 | import org.jsoup.nodes.Document; 18 | import org.jsoup.nodes.Element; 19 | import org.springframework.beans.factory.annotation.Autowired; 20 | import org.springframework.stereotype.Component; 21 | 22 | import com.sjcdigital.temis.model.document.Law; 23 | import com.sjcdigital.temis.model.repositories.LawsRepository; 24 | import com.sjcdigital.temis.model.service.machine_learn.ClassifyLaw; 25 | import com.sjcdigital.temis.model.service.parsers.AbstractParser; 26 | import com.sjcdigital.temis.model.service.parsers.util.AldermanParserUtil; 27 | import com.sjcdigital.temis.util.RegexUtils; 28 | 29 | /** 30 | * @author pedro-hos 31 | */ 32 | 33 | @Component 34 | public class LawsParser extends AbstractParser { 35 | 36 | private static final Logger LOGGER = LogManager.getLogger(LawsParser.class); 37 | 38 | @Autowired 39 | private LawsRepository lawsRepository; 40 | 41 | @Autowired 42 | private ClassifyLaw classifyLaw; 43 | 44 | @Autowired 45 | private AldermanParserUtil aldermanParserUtil; 46 | 47 | @Override 48 | public void parse(final File file) { 49 | 50 | try { 51 | 52 | final Document document = readFile(file).get(); 53 | 54 | final Law law = new Law(); 55 | final Optional title = buildTitle(document.title().trim()); 56 | 57 | String summary = buildSummary(document.head().select("script").toString()).orElse(null); 58 | 59 | law.setSummary(summary); 60 | law.setType(Objects.nonNull(summary) ? classifyLaw.classify(summary) : null); 61 | law.setTitle(title.orElse(null)); 62 | law.setDate(buildDate(title.orElse("")).orElse(LocalDate.now())); 63 | 64 | cleanDocument(document); 65 | 66 | final Element body = document.body(); 67 | law.setAuthor(aldermanParserUtil.buildAuthor(Optional.ofNullable(body.getElementsByClass("RegPub").first().text()).orElse(""))); 68 | law.setDesc(body.html().trim()); 69 | law.setCode(extractedCode(file).orElse(null)); 70 | law.setProjectLawNumber(buildProjectLawNumber(body).orElse(null)); 71 | 72 | saveLaw(law); 73 | 74 | } catch (InterruptedException | ExecutionException | IOException e) { 75 | LOGGER.error(ExceptionUtils.getStackTrace(e)); 76 | } 77 | 78 | } 79 | 80 | private void cleanDocument(final Document document) { 81 | document.select("script").remove(); 82 | document.select("a[href]").remove(); 83 | } 84 | 85 | private Optional buildTitle(String title) { 86 | 87 | final Matcher matcher = RegexUtils.getMatcher("lei\\s*municipal\\s*nº?\\s*\\d+\\.?\\d+,?\\s*(de)?\\s*\\d{1,2}/\\d{1,2}/\\d{2,4}", title); 88 | 89 | if (matcher.find()) { 90 | return Optional.of(matcher.group(0)); 91 | } 92 | 93 | return Optional.empty(); 94 | } 95 | 96 | private Optional buildSummary(String script) { 97 | 98 | final Matcher matcher = RegexUtils.getMatcher("Xtesta\\((.+)\\)", script); 99 | 100 | if (matcher.find()) { 101 | return Optional.of(matcher.group(1).split("\",\"")[1].replaceAll("