├── .gitignore ├── .gitlab-ci.yml ├── .idea └── runConfigurations │ └── Local_Environment.xml ├── CHANGELOG.md ├── Dockerfile ├── LICENSE.txt ├── README.md ├── build.gradle ├── docker-compose.yml ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── settings.gradle └── src ├── main ├── kotlin │ └── me │ │ └── ruslanys │ │ └── ifunny │ │ ├── Application.kt │ │ ├── channel │ │ ├── BastardidentroChannel.kt │ │ ├── BestiChannel.kt │ │ ├── Channel.kt │ │ ├── DebesteChannel.kt │ │ ├── FunpotChannel.kt │ │ ├── FuoriditestaImagesChannel.kt │ │ ├── FuoriditestaVideoChannel.kt │ │ ├── LachschonChannel.kt │ │ ├── MemeInfo.kt │ │ ├── OrschlurchChannel.kt │ │ ├── Page.kt │ │ ├── RigolotesChannel.kt │ │ └── YatahongaChannel.kt │ │ ├── config │ │ ├── AwsS3Config.kt │ │ ├── EventChannelConfig.kt │ │ ├── EventListenersConfig.kt │ │ ├── ExecutorsConfig.kt │ │ ├── RedisConfig.kt │ │ └── WebClientConfig.kt │ │ ├── controller │ │ ├── FeedController.kt │ │ ├── GlobalExceptionHandler.kt │ │ ├── MemeController.kt │ │ ├── SwaggerUiController.kt │ │ ├── api │ │ │ └── FeedApiController.kt │ │ ├── dto │ │ │ ├── ErrorDto.kt │ │ │ ├── MemeDto.kt │ │ │ ├── PageRequest.kt │ │ │ └── PageResponse.kt │ │ └── validation │ │ │ ├── SortConstraint.kt │ │ │ └── SortValidator.kt │ │ ├── domain │ │ ├── Language.kt │ │ ├── Meme.kt │ │ └── S3File.kt │ │ ├── exception │ │ └── NotFoundException.kt │ │ ├── grab │ │ ├── Coordinator.kt │ │ ├── MemeIndexer.kt │ │ ├── PageIndexer.kt │ │ ├── ResourceDownloader.kt │ │ ├── SuspendedEventListener.kt │ │ └── event │ │ │ ├── GrabEvent.kt │ │ │ ├── MemeIndexRequest.kt │ │ │ ├── PageIndexEvent.kt │ │ │ ├── PageIndexRequest.kt │ │ │ ├── PageIndexSuccessful.kt │ │ │ └── ResourceDownloadRequest.kt │ │ ├── property │ │ ├── AwsS3Properties.kt │ │ └── GrabProperties.kt │ │ ├── repository │ │ ├── MemeRepository.kt │ │ ├── MongoIndexCreator.kt │ │ └── PageRepository.kt │ │ └── service │ │ ├── DefaultMemeService.kt │ │ └── Services.kt └── resources │ ├── application.properties │ ├── banner.png │ ├── banner.txt │ ├── static │ ├── contract.yaml │ ├── favicon.ico │ ├── img │ │ └── logo.png │ └── swagger-ui │ │ ├── favicon-16x16.png │ │ ├── favicon-32x32.png │ │ ├── index.html │ │ ├── oauth2-redirect.html │ │ ├── swagger-ui-bundle.js │ │ ├── swagger-ui-bundle.js.map │ │ ├── swagger-ui-standalone-preset.js │ │ ├── swagger-ui-standalone-preset.js.map │ │ ├── swagger-ui.css │ │ ├── swagger-ui.css.map │ │ ├── swagger-ui.js │ │ └── swagger-ui.js.map │ └── templates │ ├── _layout.ftlh │ ├── feed.ftlh │ └── meme.ftlh └── test ├── kotlin ├── me │ └── ruslanys │ │ └── ifunny │ │ ├── ApplicationTests.kt │ │ ├── base │ │ ├── ControllerTests.kt │ │ ├── MongoRepositoryTests.kt │ │ ├── RedisRepositoryTests.kt │ │ └── ServiceTests.kt │ │ ├── channel │ │ ├── BastardidentroChannelTests.kt │ │ ├── BestiChannelTests.kt │ │ ├── DebesteChannelTests.kt │ │ ├── FunpotChannelTests.kt │ │ ├── FuoriditestaImagesChannelTests.kt │ │ ├── FuoriditestaVideoChannelTests.kt │ │ ├── LachschonChannelTests.kt │ │ ├── OrschlurchChannelTests.kt │ │ ├── RigolotesChannelTests.kt │ │ └── YatahongaChannelTests.kt │ │ ├── controller │ │ ├── FeedControllerTests.kt │ │ ├── MemeControllerTests.kt │ │ ├── api │ │ │ └── FeedApiControllerTests.kt │ │ └── dto │ │ │ └── PageRequestTests.kt │ │ ├── grab │ │ ├── CoordinatorTests.kt │ │ ├── MemeIndexerTests.kt │ │ ├── PageIndexerTests.kt │ │ └── ResourceDownloaderTests.kt │ │ ├── repository │ │ ├── MemeRepositoryTests.kt │ │ └── PageRepositoryTests.kt │ │ ├── service │ │ └── DefaultMemeServiceTests.kt │ │ └── util │ │ ├── DummyData.kt │ │ ├── IOUtils.kt │ │ └── MockClient.kt └── phash │ ├── AverageHashTests.kt │ └── CompoundHashTests.kt └── resources ├── me └── ruslanys │ └── ifunny │ ├── channel │ ├── bastardidentro │ │ ├── meme_picture.html │ │ ├── page.html │ │ ├── page_last.html │ │ └── page_without_header.html │ ├── besti │ │ ├── meme_picture.html │ │ ├── meme_video.html │ │ ├── page.html │ │ └── page_last.html │ ├── debeste │ │ ├── meme_gif_Date.html │ │ ├── meme_picture_Author.html │ │ ├── meme_picture_AuthorDate.html │ │ ├── meme_video_Comments.html │ │ ├── page.html │ │ └── page_last.html │ ├── funpot │ │ ├── meme_gif.html │ │ ├── meme_picture.html │ │ ├── meme_video.html │ │ ├── meme_video_Direct.html │ │ ├── page.html │ │ └── page_last.html │ ├── fuoriditesta-images │ │ ├── meme_picture.html │ │ ├── page.html │ │ └── page_last.html │ ├── fuoriditesta-video │ │ ├── frame.html │ │ ├── meme_video.html │ │ ├── page.html │ │ └── page_last.html │ ├── lachschon │ │ ├── meme.html │ │ ├── page.html │ │ └── page_last.html │ ├── orschlurch │ │ ├── meme_video.html │ │ ├── page.html │ │ └── page_last.html │ ├── rigolotes │ │ ├── meme_gif.html │ │ ├── meme_picture.html │ │ ├── meme_video.html │ │ ├── page.html │ │ ├── page_broken_upvotes.html │ │ ├── page_last.html │ │ ├── page_with_video.html │ │ └── page_with_youtube.html │ └── yatahonga │ │ ├── meme_picture.html │ │ ├── meme_video.html │ │ ├── page.html │ │ └── page_last.html │ ├── controller │ └── api │ │ ├── feed_getById.json │ │ └── feed_getPage.json │ └── grab │ ├── debeste_meme.html │ ├── debeste_page.html │ └── picture.jpg └── phash ├── dataset-1 ├── 1.jpg ├── 2.jpg ├── 3.jpg ├── 4.jpg └── 5.jpg ├── dataset-2 ├── debeste.jpg ├── funpot.jpg └── funpot_smaller.jpg ├── dataset-3 ├── debeste.jpg ├── debeste_smaller.jpg └── funpot.jpg ├── dataset-4 ├── debeste.gif └── funpot.gif └── dataset-5 ├── 1.jpg ├── 2.jpg ├── 3.jpg ├── 4.jpg └── 5.jpg /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/* 2 | !.idea/runConfigurations/ 3 | build/ 4 | .gradle 5 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | stages: 2 | - build 3 | - package 4 | - deploy 5 | 6 | 7 | variables: 8 | DOCKER_DRIVER: overlay2 9 | DOCKER_TLS_CERTDIR: "" 10 | IMAGE_NAME: registry.gitlab.com/$CI_PROJECT_PATH 11 | 12 | 13 | ##################### 14 | # Building 15 | ##################### 16 | build-jar: 17 | stage: build 18 | image: openjdk:11-oracle 19 | variables: 20 | MONGO_INITDB_ROOT_USERNAME: ifunny 21 | MONGO_INITDB_ROOT_PASSWORD: ifunny 22 | MONGODB_HOST: mongo 23 | REDIS_HOST: redis 24 | REDIS_PORT: 6379 25 | services: 26 | - name: mongo 27 | alias: $MONGODB_HOST 28 | - name: redis 29 | alias: $REDIS_HOST 30 | before_script: 31 | - export GRADLE_USER_HOME=`pwd`/.gradle 32 | script: 33 | - ./gradlew build 34 | after_script: 35 | - cat build/reports/jacoco/test/html/index.html 36 | coverage: '/Total.*?([0-9]{1,3})%/' 37 | artifacts: 38 | paths: 39 | - build/reports/ 40 | - build/libs/*.jar 41 | expire_in: 1 week 42 | when: always 43 | cache: 44 | paths: 45 | - .gradle/wrapper 46 | - .gradle/caches 47 | 48 | 49 | 50 | ########################### 51 | # Packaging 52 | ########################### 53 | package-docker: 54 | image: docker:stable 55 | stage: package 56 | services: 57 | - docker:stable-dind 58 | before_script: 59 | - if [[ ${CI_COMMIT_REF_NAME} == master ]]; then export IMAGE_TAG=latest; else export IMAGE_TAG=${CI_COMMIT_REF_NAME}; fi; 60 | script: 61 | - docker build -t $IMAGE_NAME:$IMAGE_TAG . 62 | - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com 63 | - docker push $IMAGE_NAME:$IMAGE_TAG 64 | only: 65 | - master 66 | - tags 67 | 68 | 69 | ##################### 70 | # Deployment 71 | ##################### 72 | .prepare-key-script: &prepare_key 73 | - apk add --no-cache openssh-client 74 | - eval $(ssh-agent -s) 75 | - echo "$SSH_KEY" | tr -d '\r' | ssh-add - > /dev/null 76 | - mkdir -p ~/.ssh 77 | - chmod 700 ~/.ssh 78 | - ssh-keyscan $SSH_HOST >> ~/.ssh/known_hosts 79 | - chmod 644 ~/.ssh/known_hosts 80 | 81 | .deploy-script: &deploy 82 | - if [[ ${CI_COMMIT_REF_NAME} == master ]]; then export IMAGE_TAG=latest; else export IMAGE_TAG=${CI_COMMIT_REF_NAME}; fi; 83 | - ssh $SSH_USER@$SSH_HOST "docker stop $CONTAINER_NAME || true" 84 | - ssh $SSH_USER@$SSH_HOST "docker rm $CONTAINER_NAME || true" 85 | - ssh $SSH_USER@$SSH_HOST "docker rmi $IMAGE_NAME:$IMAGE_TAG || true" 86 | - ssh $SSH_USER@$SSH_HOST "docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com" 87 | - | 88 | ssh $SSH_USER@$SSH_HOST " 89 | docker run -d --name $CONTAINER_NAME --restart always \ 90 | -p $CONTAINER_PORT:8080 \ 91 | -e MONGODB_HOST=$MONGODB_HOST \ 92 | -e MONGODB_DATABASE=$MONGODB_DATABASE \ 93 | -e MONGODB_USERNAME=$MONGODB_USERNAME \ 94 | -e MONGODB_PASSWORD=$MONGODB_PASSWORD \ 95 | -e REDIS_HOST=$REDIS_HOST \ 96 | -e SPRING_REDIS_PASSWORD=$REDIS_PASSWORD \ 97 | -e AWS_S3_ACCESS_KEY=$AWS_S3_ACCESS_KEY \ 98 | -e AWS_S3_SECRET_KEY=$AWS_S3_SECRET_KEY \ 99 | -e AWS_S3_REGION=$AWS_S3_REGION \ 100 | -e AWS_S3_BUCKET=$AWS_S3_BUCKET \ 101 | $IMAGE_NAME:$IMAGE_TAG 102 | " 103 | 104 | deploy-production: 105 | image: docker:latest 106 | stage: deploy 107 | dependencies: [] 108 | variables: 109 | CONTAINER_NAME: ifunny 110 | CONTAINER_PORT: 4386 111 | MONGODB_HOST: $PRODUCTION_MONGODB_HOST 112 | MONGODB_DATABASE: $PRODUCTION_MONGODB_DATABASE 113 | MONGODB_USERNAME: $PRODUCTION_MONGODB_USERNAME 114 | MONGODB_PASSWORD: $PRODUCTION_MONGODB_PASSWORD 115 | REDIS_HOST: $PRODUCTION_REDIS_HOST 116 | REDIS_PASSWORD: $PRODUCTION_REDIS_PASSWORD 117 | AWS_S3_ACCESS_KEY: $PRODUCTION_AWS_S3_ACCESS_KEY 118 | AWS_S3_SECRET_KEY: $PRODUCTION_AWS_S3_SECRET_KEY 119 | AWS_S3_REGION: $PRODUCTION_AWS_S3_REGION 120 | AWS_S3_BUCKET: $PRODUCTION_AWS_S3_BUCKET 121 | before_script: *prepare_key 122 | script: *deploy 123 | environment: 124 | name: production 125 | url: https://ifunny.ruslanys.me 126 | only: 127 | - tags 128 | -------------------------------------------------------------------------------- /.idea/runConfigurations/Local_Environment.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | ## [Unreleased] 9 | 10 | ## [2.4.1] - 2020-02-27 11 | ### Security 12 | - Spring Boot upgrade to 2.2.5 13 | 14 | ## [2.4.0] - 2020-02-22 15 | ### Added 16 | - Fuoriditesta images channel 17 | - Fuoriditesta video channel 18 | 19 | ### Changed 20 | - All methods of Channel are suspendable 21 | 22 | ## [2.3.1] - 2020-02-21 23 | ### Fixed 24 | - BastardidentroChannel: Parse a page with meme without header 25 | 26 | ## [2.3.0] - 2020-02-21 27 | ### Added 28 | - Bastardidentro.it channel 29 | 30 | ## [2.2.0] - 2020-02-21 31 | ### Added 32 | - Besti channel 33 | 34 | ## [2.1.1] - 2020-02-21 35 | ### Fixed 36 | - Debeste channel changed its markup for adv boxes and broke pages parsing 37 | 38 | ## [2.1.0] - 2020-02-19 39 | ### Added 40 | - Orschlurch channel 41 | 42 | ### Changed 43 | - Event channel size extracted into properties variable (`grab.channel-size`) 44 | 45 | ## [2.0.3] - 2020-01-28 46 | ### Changed 47 | - Dockerfile `CMD` replaced with `ENTRYPOINT` 48 | - Dockerfile shell form replaced with exec form 49 | 50 | ## [2.0.2] - 2020-01-22 51 | ### Fixed 52 | - Rigolotes channel changed its markup for rating (votes) and broke pages parsing 53 | 54 | ## [2.0.1] - 2020-01-22 55 | ### Fixed 56 | - Rigolotes channel changed its markup for publish date and broke pages parsing 57 | 58 | ## [2.0.0] - 2020-01-19 59 | ### Changed 60 | - Blocking IO changed with NIO 61 | 62 | ### Added 63 | - Reactive Approach 64 | - Coroutines 65 | 66 | ## [1.0.0] - 2020-01-11 67 | ### Added 68 | - Feed API Endpoint 69 | - Open API 3 Specification (including Swagger UI) 70 | - Frontend: Feed Page 71 | - Frontend: Meme individual page 72 | - Crawler itself 73 | - Memes deduplication 74 | - S3 Integration 75 | - Yatahonga.com channel 76 | - Rigolotes.fr channel 77 | - Lachschon.de channel 78 | - Funpot.net channel 79 | - Debeste.de channel 80 | - Prometheus metrics 81 | - GitLab Pipeline: CD 82 | 83 | ### Fixed 84 | - Tag v0.0.1 reference fix 85 | 86 | ## [0.0.1] - 2019-12-26 87 | ### Added 88 | - Application Skeleton 89 | - GitLab CI: Build job 90 | - GitLab CI: Package Docker image job 91 | 92 | [unreleased]: https://gitlab.com/ruslanys/ifunny/compare/v2.4.1...master 93 | [2.4.1]: https://gitlab.com/ruslanys/ifunny/compare/v2.4.0...v2.4.1 94 | [2.4.0]: https://gitlab.com/ruslanys/ifunny/compare/v2.3.1...v2.4.0 95 | [2.3.1]: https://gitlab.com/ruslanys/ifunny/compare/v2.3.0...v2.3.1 96 | [2.3.0]: https://gitlab.com/ruslanys/ifunny/compare/v2.2.0...v2.3.0 97 | [2.2.0]: https://gitlab.com/ruslanys/ifunny/compare/v2.1.1...v2.2.0 98 | [2.1.1]: https://gitlab.com/ruslanys/ifunny/compare/v2.1.0...v2.1.1 99 | [2.1.0]: https://gitlab.com/ruslanys/ifunny/compare/v2.0.3...v2.1.0 100 | [2.0.3]: https://gitlab.com/ruslanys/ifunny/compare/v2.0.2...v2.0.3 101 | [2.0.2]: https://gitlab.com/ruslanys/ifunny/compare/v2.0.1...v2.0.2 102 | [2.0.1]: https://gitlab.com/ruslanys/ifunny/compare/v2.0.0...v2.0.1 103 | [2.0.0]: https://gitlab.com/ruslanys/ifunny/compare/v1.0.0...v2.0.0 104 | [1.0.0]: https://gitlab.com/ruslanys/ifunny/compare/v0.0.1...v1.0.0 105 | [0.0.1]: https://gitlab.com/ruslanys/ifunny/-/tags/v0.0.1 106 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:11-jre-slim 2 | EXPOSE 8080 3 | 4 | WORKDIR root/ 5 | ARG JAR_FILE=build/libs/ifunny-*.jar 6 | ADD ${JAR_FILE} ./application.jar 7 | 8 | ENTRYPOINT ["java", "-server", "-Xms2G", "-Xmx2G", "-XX:MaxMetaspaceSize=256M", "-XX:MaxDirectMemorySize=1G",\ 9 | "-XX:+HeapDumpOnOutOfMemoryError", "-XX:HeapDumpPath=dump.hprof", "-Djava.security.egd=/dev/zrandom",\ 10 | "-jar", "/root/application.jar"] 11 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'jacoco' 3 | id 'idea' 4 | id 'org.jetbrains.kotlin.jvm' version '1.3.61' 5 | id 'org.jetbrains.kotlin.kapt' version '1.3.61' 6 | id 'org.jetbrains.kotlin.plugin.spring' version '1.3.61' 7 | id 'org.springframework.boot' version '2.2.5.RELEASE' 8 | id 'io.zensoft.versioning' version '1.1.0' // This plugin made by me and my previous team :) 9 | } 10 | 11 | apply plugin: 'io.spring.dependency-management' 12 | 13 | group = 'me.ruslanys' 14 | 15 | sourceCompatibility = JavaVersion.VERSION_1_8 16 | 17 | repositories { 18 | mavenCentral() 19 | jcenter() 20 | } 21 | 22 | ext { 23 | jsoupVersion = '1.12.1' 24 | awsSdkVersion = '2.10.47' 25 | } 26 | 27 | dependencies { 28 | // Kotlin 29 | implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") 30 | implementation("org.jetbrains.kotlin:kotlin-reflect") 31 | implementation('com.fasterxml.jackson.module:jackson-module-kotlin') 32 | implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core") 33 | implementation("org.jetbrains.kotlinx:kotlinx-coroutines-reactor") 34 | implementation("org.jetbrains.kotlinx:kotlinx-coroutines-jdk8") 35 | 36 | 37 | // Spring 38 | implementation("org.springframework.boot:spring-boot-starter-webflux") 39 | implementation('org.springframework.boot:spring-boot-starter-freemarker') 40 | implementation('org.springframework.boot:spring-boot-starter-actuator') 41 | implementation("org.springframework.boot:spring-boot-starter-data-mongodb-reactive") 42 | implementation("org.springframework.boot:spring-boot-starter-data-redis-reactive") 43 | 44 | 45 | // Tools 46 | implementation("org.jsoup:jsoup:$jsoupVersion") 47 | implementation('io.micrometer:micrometer-registry-prometheus') 48 | implementation("software.amazon.awssdk:s3:$awsSdkVersion") 49 | implementation("software.amazon.awssdk:netty-nio-client:$awsSdkVersion") 50 | implementation('org.apache.tika:tika-core:1.23') 51 | implementation('com.github.kilianB:JImageHash:3.0.0') 52 | 53 | 54 | // DevTools 55 | runtimeOnly('org.springframework.boot:spring-boot-devtools') 56 | kapt('org.springframework.boot:spring-boot-configuration-processor') 57 | 58 | 59 | // Tests 60 | testImplementation('org.springframework.boot:spring-boot-starter-test') { 61 | exclude group: 'org.junit.vintage', module: 'junit-vintage-engine' 62 | } 63 | testImplementation("io.projectreactor:reactor-test") 64 | testImplementation("com.nhaarman.mockitokotlin2:mockito-kotlin:2.2.0") 65 | } 66 | 67 | sourceSets { 68 | main.kotlin.srcDirs += 'src/main/kotlin' 69 | test.kotlin.srcDirs += 'src/test/kotlin' 70 | } 71 | 72 | // Jar 73 | bootJar { 74 | manifest { 75 | attributes("Implementation-Version": archiveVersion) 76 | } 77 | } 78 | 79 | // Kotlin 80 | compileKotlin { 81 | kotlinOptions { 82 | freeCompilerArgs = ["-Xjsr305=strict"] 83 | jvmTarget = "1.8" 84 | } 85 | } 86 | compileTestKotlin { 87 | kotlinOptions { 88 | freeCompilerArgs = ["-Xjsr305=strict"] 89 | jvmTarget = "1.8" 90 | } 91 | } 92 | 93 | // Tests 94 | test { 95 | useJUnitPlatform() 96 | 97 | if (project.hasProperty('maxParallelForks')) { 98 | maxParallelForks = project.maxParallelForks as int 99 | } 100 | if (project.hasProperty('forkEvery')) { 101 | forkEvery = project.forkEvery as int 102 | } 103 | } 104 | jacoco { 105 | toolVersion = "0.8.5" 106 | } 107 | jacocoTestReport { 108 | reports { 109 | xml.enabled = true 110 | html.enabled = true 111 | } 112 | 113 | afterEvaluate { 114 | classDirectories.from = files(classDirectories.files.collect { 115 | fileTree(dir: it, exclude: [ 116 | 'me/ruslanys/ifunny/*Application*', 117 | 'me/ruslanys/ifunny/config/**', 118 | 'me/ruslanys/ifunny/property/**', 119 | 'me/ruslanys/ifunny/domain/**', 120 | 'me/ruslanys/ifunny/grab/event/**' 121 | ]) 122 | }) 123 | } 124 | } 125 | check.dependsOn jacocoTestReport 126 | 127 | // IDEA 128 | idea { 129 | module { 130 | def kaptMain = file('build/generated/source/kapt/main') 131 | sourceDirs += kaptMain 132 | generatedSourceDirs += kaptMain 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.1' 2 | 3 | services: 4 | 5 | mongo: 6 | image: mongo 7 | restart: always 8 | ports: 9 | - 27017:27017 10 | environment: 11 | MONGO_INITDB_ROOT_USERNAME: ifunny 12 | MONGO_INITDB_ROOT_PASSWORD: ifunny 13 | 14 | redis: 15 | image: redis 16 | restart: always 17 | ports: 18 | - 6379:6379 19 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.2-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto init 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto init 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :init 68 | @rem Get command-line arguments, handling Windows variants 69 | 70 | if not "%OS%" == "Windows_NT" goto win9xME_args 71 | 72 | :win9xME_args 73 | @rem Slurp the command line arguments. 74 | set CMD_LINE_ARGS= 75 | set _SKIP=2 76 | 77 | :win9xME_args_slurp 78 | if "x%~1" == "x" goto execute 79 | 80 | set CMD_LINE_ARGS=%* 81 | 82 | :execute 83 | @rem Setup the command line 84 | 85 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 86 | 87 | @rem Execute Gradle 88 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 89 | 90 | :end 91 | @rem End local scope for the variables with windows NT shell 92 | if "%ERRORLEVEL%"=="0" goto mainEnd 93 | 94 | :fail 95 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 96 | rem the _cmd.exe /c_ return code! 97 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 98 | exit /b 1 99 | 100 | :mainEnd 101 | if "%OS%"=="Windows_NT" endlocal 102 | 103 | :omega 104 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = "ifunny" 2 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/Application.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny 2 | 3 | import org.springframework.boot.autoconfigure.SpringBootApplication 4 | import org.springframework.boot.context.properties.ConfigurationPropertiesScan 5 | import org.springframework.boot.runApplication 6 | 7 | @ConfigurationPropertiesScan("me.ruslanys.ifunny.property") 8 | @SpringBootApplication 9 | class Application 10 | 11 | fun main(args: Array) { 12 | runApplication(*args) 13 | } 14 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/channel/BastardidentroChannel.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | import me.ruslanys.ifunny.domain.Language 4 | import org.jsoup.Jsoup 5 | import org.jsoup.nodes.Element 6 | import org.springframework.stereotype.Component 7 | import java.net.URLDecoder 8 | import java.time.LocalDateTime 9 | import java.time.ZonedDateTime 10 | 11 | @Component 12 | class BastardidentroChannel : Channel(Language.ITALIAN, "https://www.bastardidentro.it") { 13 | 14 | override suspend fun pagePath(pageNumber: Int): String { 15 | return if (pageNumber == 1) { 16 | "$baseUrl/immagini-e-vignette-divertenti" 17 | } else { 18 | "$baseUrl/immagini-e-vignette-divertenti?page=${pageNumber - 1}" 19 | } 20 | } 21 | 22 | override suspend fun parsePage(pageNumber: Int, body: String): Page { 23 | val document = Jsoup.parse(body).apply { setBaseUri(baseUrl) } 24 | val boxes = document.select("#block-system-main .view-content > div.views-row-odd, #block-system-main .view-content > div.views-row-even") 25 | 26 | val list = arrayListOf() 27 | 28 | for (box in boxes) { 29 | val (url, title) = parseHeader(box) ?: continue 30 | 31 | // -- 32 | val info = MemeInfo(pageUrl = url, title = title) 33 | list.add(info) 34 | } 35 | 36 | return Page(pageNumber, isHasNext(document), list) 37 | } 38 | 39 | private fun parseHeader(box: Element): Pair? { 40 | val link = box.selectFirst(".views-field-title a") ?: return null 41 | val url = link.absUrl("href") 42 | val decodedUrl = URLDecoder.decode(url, Charsets.UTF_8) 43 | val title = link.text() 44 | 45 | return decodedUrl to title 46 | } 47 | 48 | private fun isHasNext(document: Element): Boolean { 49 | return document.select(".pagination > .next").isNotEmpty() 50 | } 51 | 52 | override suspend fun parseMeme(info: MemeInfo, body: String): MemeInfo { 53 | val document = Jsoup.parse(body).apply { setBaseUri(baseUrl) } 54 | 55 | val publishedDateTime = parsePublishDateTime(document) 56 | val resourceUrl = parsePictureUrl(document) 57 | 58 | return MemeInfo( 59 | pageUrl = info.pageUrl, 60 | title = info.title, 61 | publishDateTime = publishedDateTime, 62 | originUrl = resourceUrl 63 | ) 64 | } 65 | 66 | private fun parsePublishDateTime(document: Element): LocalDateTime { 67 | val meta = document.selectFirst("meta[property=article:published_time]").attr("content") 68 | return ZonedDateTime.parse(meta).toLocalDateTime() 69 | } 70 | 71 | private fun parsePictureUrl(document: Element): String { 72 | return document.selectFirst("#internal .img-responsive").absUrl("src") 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/channel/BestiChannel.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | import me.ruslanys.ifunny.domain.Language 4 | import org.jsoup.Jsoup 5 | import org.jsoup.nodes.Element 6 | import org.springframework.stereotype.Component 7 | import java.time.LocalDateTime 8 | import java.time.format.DateTimeFormatter 9 | import java.util.regex.Pattern 10 | 11 | @Component 12 | class BestiChannel : Channel(Language.ITALIAN, "https://besti.it") { 13 | 14 | override suspend fun pagePath(pageNumber: Int): String { 15 | return "$baseUrl/$pageNumber" 16 | } 17 | 18 | override suspend fun parsePage(pageNumber: Int, body: String): Page { 19 | val document = Jsoup.parse(body).apply { setBaseUri(baseUrl) } 20 | val boxes = document.select(".box") 21 | 22 | val list = arrayListOf() 23 | 24 | for (box in boxes) { 25 | val (url, title) = parseHeader(box) 26 | if (url == "#") { // skip adv box 27 | continue 28 | } 29 | 30 | // -- 31 | val rate = parseRate(box) 32 | if (rate < 0) { // skip negative 33 | continue 34 | } 35 | 36 | // -- 37 | val comments = parseComments(box) 38 | val author = parseAuthor(box) 39 | val publishDateTime = parsePublishDateTime(box) 40 | 41 | // -- 42 | val info = MemeInfo(url, null, title, publishDateTime, rate, comments, author) 43 | list.add(info) 44 | } 45 | 46 | return Page(pageNumber, isHasNext(document), list) 47 | } 48 | 49 | private fun parseHeader(box: Element): Pair { 50 | val link = box.selectFirst("h2 > a") 51 | val url = link.attr("href") 52 | val title = link.text() 53 | 54 | return url to title 55 | } 56 | 57 | private fun parseRate(box: Element): Int { 58 | val rateText = box.selectFirst(".rate").text() 59 | val rateMatcher = NUMBER_EXTRACTOR.matcher(rateText) 60 | return if (rateMatcher.find()) { 61 | rateMatcher.group(1).toInt() 62 | } else { 63 | throw IllegalStateException("Can't parse rate") 64 | } 65 | } 66 | 67 | private fun parseComments(box: Element): Int { 68 | val commentsText = box.select(".objectMeta > a").last().text() 69 | val commentsMatcher = NUMBER_EXTRACTOR.matcher(commentsText) 70 | return if (commentsMatcher.find()) { 71 | commentsMatcher.group(1).toInt() 72 | } else { 73 | throw IllegalStateException("Can't parse comments number") 74 | } 75 | } 76 | 77 | private fun parseAuthor(box: Element): String { 78 | return box.selectFirst(".objectMeta > a").text() 79 | } 80 | 81 | private fun parsePublishDateTime(box: Element): LocalDateTime { 82 | val metaText = box.selectFirst(".objectMeta").text() 83 | val dateTimeMatcher = DATE_EXTRACTOR.matcher(metaText) 84 | return if (dateTimeMatcher.find()) { 85 | val dateStr = dateTimeMatcher.group(1) 86 | LocalDateTime.parse(dateStr, DATE_TIME_FORMATTER) 87 | } else { 88 | throw IllegalStateException("Can't parse publishDateTime") 89 | } 90 | } 91 | 92 | private fun isHasNext(document: Element): Boolean { 93 | return document.select("li.next").firstOrNull()?.hasClass("disabled") == false 94 | } 95 | 96 | override suspend fun parseMeme(info: MemeInfo, body: String): MemeInfo { 97 | val document = Jsoup.parse(body).apply { setBaseUri(baseUrl) } 98 | 99 | val pictureUrl = parsePictureUrl(document) 100 | val videoUrl = parseVideoUrl(document) 101 | 102 | val resourceUrl = pictureUrl ?: videoUrl 103 | 104 | return MemeInfo(info.pageUrl, resourceUrl, info.title, info.publishDateTime, info.likes, info.comments, info.author) 105 | } 106 | 107 | private fun parsePictureUrl(document: Element): String? { 108 | val image = document.selectFirst(".box-img > img") ?: return null 109 | return image.absUrl("src") 110 | } 111 | 112 | private fun parseVideoUrl(document: Element): String? { 113 | val video = document.selectFirst(".box-video > video > source") ?: return null 114 | return video.absUrl("src") 115 | } 116 | 117 | 118 | companion object { 119 | private val DATE_EXTRACTOR = Pattern.compile("Aggiunto: (.*) per") 120 | private val NUMBER_EXTRACTOR = Pattern.compile("\\(([\\d-+]+)\\)") 121 | 122 | private val DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") 123 | } 124 | 125 | } 126 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/channel/Channel.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | import me.ruslanys.ifunny.domain.Language 4 | 5 | /** 6 | * Channel itself is a source of memes. 7 | * To add a new channel a developer should extend this class and declare the implementation as a Spring [Bean][org.springframework.context.annotation.Bean]. 8 | * 9 | * Each channel should implement three declared methods: [pagePath], [parsePage], [parseMeme]. 10 | * 11 | * Architecture based on the assumption, that each channel returns memes in a pageable format (divided by pages aka Pagination). 12 | * 13 | * In that case grab process is the following: 14 | * 1. Getting a page URL for a specific page number by [pagePath]. 15 | * 1. Downloading the page content and parse it by [parsePage]. As a result, a channel should return a list of memes. 16 | * All fields are optional, except `pageUrl` field as each website is very specific. 17 | * 1. Per each meme individual page (`pageUrl` parameter) downloading page content and parse it by [parseMeme] returning all possible to fetch information. 18 | */ 19 | abstract class Channel(val language: Language, val baseUrl: String) { 20 | 21 | /** 22 | * The method returns page URL based on the page number. 23 | */ 24 | abstract suspend fun pagePath(pageNumber: Int): String 25 | 26 | /** 27 | * The method gets Page body and returns grabbed memes list. 28 | */ 29 | abstract suspend fun parsePage(pageNumber: Int, body: String): Page 30 | 31 | /** 32 | * The method gets meme individual page and returns extended information. 33 | */ 34 | abstract suspend fun parseMeme(info: MemeInfo, body: String): MemeInfo 35 | 36 | fun getName(): String = javaClass.simpleName 37 | 38 | override fun toString(): String { 39 | return getName() 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/channel/DebesteChannel.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | import me.ruslanys.ifunny.domain.Language 4 | import org.jsoup.Jsoup 5 | import org.jsoup.nodes.Element 6 | import org.springframework.stereotype.Component 7 | import java.time.LocalDateTime 8 | import java.time.format.DateTimeFormatter 9 | import java.util.regex.Pattern 10 | 11 | @Component 12 | class DebesteChannel : Channel(Language.GERMAN, "http://debeste.de") { 13 | 14 | override suspend fun pagePath(pageNumber: Int): String = "$baseUrl/$pageNumber" 15 | 16 | override suspend fun parsePage(pageNumber: Int, body: String): Page { 17 | val document = Jsoup.parse(body) 18 | val boxes = document.select(".box") 19 | 20 | val list = arrayListOf() 21 | 22 | for (box in boxes) { 23 | // Skip boxes without content 24 | if (box.select("h2").isEmpty()) { 25 | continue 26 | } 27 | 28 | // Header 29 | val (url, title) = parseHeader(box) 30 | 31 | if (url == "#") { // skip adv box 32 | continue 33 | } 34 | 35 | // Rate 36 | val rate = parseRate(box) 37 | if (rate != null && rate < 0) { // skip negative 38 | continue 39 | } 40 | 41 | // Comments 42 | val comments = parseComments(box) 43 | 44 | // -- 45 | val info = MemeInfo(pageUrl = url, title = title, likes = rate, comments = comments) 46 | list.add(info) 47 | } 48 | 49 | return Page(pageNumber, isHasNext(document), list) 50 | } 51 | 52 | private fun parseHeader(box: Element): Pair { 53 | val link = box.select("h2 > a") 54 | 55 | val memeUrl = link.attr("href") 56 | val title = link.text() 57 | 58 | return memeUrl to title 59 | } 60 | 61 | private fun parseRate(box: Element): Int? { 62 | val rateText = box.selectFirst(".rate").text() 63 | val rateMatcher = NUMBER_EXTRACTOR.matcher(rateText) 64 | return if (rateMatcher.find()) { 65 | rateMatcher.group(1).toInt() 66 | } else { 67 | null 68 | } 69 | } 70 | 71 | private fun parseComments(box: Element): Int? { 72 | val commentsText = box.selectFirst(".objectMeta > a").text() 73 | val commentsMatcher = NUMBER_EXTRACTOR.matcher(commentsText) 74 | return if (commentsMatcher.find()) { 75 | commentsMatcher.group(1).toInt() 76 | } else { 77 | null 78 | } 79 | } 80 | 81 | private fun isHasNext(document: Element): Boolean { 82 | return document.select("li.next").firstOrNull()?.hasClass("disabled") == false 83 | } 84 | 85 | override suspend fun parseMeme(info: MemeInfo, body: String): MemeInfo { 86 | val document = Jsoup.parse(body).also { it.setBaseUri(baseUrl) } 87 | val box = document.selectFirst(".box") 88 | 89 | // -- 90 | val pictureUrl = parsePictureUrl(box) 91 | val videoUrl = parseVideoUrl(box) 92 | val author = parseAuthor(box) 93 | val publishDateTime = parsePublishDateTime(box) 94 | 95 | // -- 96 | return MemeInfo( 97 | pageUrl = info.pageUrl, 98 | title = info.title, 99 | likes = info.likes, 100 | comments = info.comments, 101 | originUrl = pictureUrl ?: videoUrl, 102 | author = author, 103 | publishDateTime = publishDateTime 104 | ) 105 | } 106 | 107 | private fun parsePictureUrl(box: Element): String? { 108 | val image = box.selectFirst(".box-img > img") ?: return null 109 | return image.absUrl("src") 110 | } 111 | 112 | private fun parseVideoUrl(box: Element): String? { 113 | val video = box.selectFirst(".box-video > video > source") ?: return null 114 | return video.absUrl("src") 115 | } 116 | 117 | private fun parseAuthor(box: Element): String? { 118 | val metaText = box.selectFirst(".objectMeta").text() 119 | val authorMatcher = AUTHOR_EXTRACTOR.matcher(metaText) 120 | return if (authorMatcher.find()) { 121 | authorMatcher.group(1) 122 | } else { 123 | null 124 | } 125 | } 126 | 127 | private fun parsePublishDateTime(box: Element): LocalDateTime? { 128 | val metaText = box.selectFirst(".objectMeta").text() 129 | val dateTimeMatcher = DATE_EXTRACTOR.matcher(metaText) 130 | return if (dateTimeMatcher.find()) { 131 | val dateStr = dateTimeMatcher.group(1) 132 | LocalDateTime.parse(dateStr, DATE_TIME_FORMATTER) 133 | } else { 134 | null 135 | } 136 | } 137 | 138 | companion object { 139 | private val NUMBER_EXTRACTOR = Pattern.compile("\\(([\\d-+]+)\\)") 140 | private val AUTHOR_EXTRACTOR = Pattern.compile("von (.*) \\| Homepage") 141 | private val DATE_EXTRACTOR = Pattern.compile("hinzugefügt: (.*) Kommentar") 142 | private val DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") 143 | } 144 | 145 | } 146 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/channel/FunpotChannel.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | import me.ruslanys.ifunny.domain.Language 4 | import org.jsoup.Jsoup 5 | import org.jsoup.nodes.Element 6 | import org.springframework.stereotype.Component 7 | import java.time.LocalDateTime 8 | import java.time.format.DateTimeFormatter 9 | 10 | @Component 11 | class FunpotChannel : Channel(Language.GERMAN, "https://funpot.net") { 12 | 13 | override suspend fun pagePath(pageNumber: Int): String = "$baseUrl/entdecken/lustiges/$pageNumber/" 14 | 15 | override suspend fun parsePage(pageNumber: Int, body: String): Page { 16 | val document = Jsoup.parse(body) 17 | val boxes = document.getElementsByClass("contentline") 18 | 19 | val list = arrayListOf() 20 | 21 | for (box in boxes) { 22 | // Type 23 | val type = parseType(box) 24 | if (type == null || !type.startsWith("Bild") && !type.startsWith("Online-Video")) { 25 | continue // skip non-image and non-video content 26 | } 27 | 28 | // Header 29 | val (url, title) = parseHeader(box) 30 | 31 | // Likes 32 | val likes = parseLikes(box) 33 | 34 | // Author 35 | val author = parseAuthor(box) 36 | 37 | // Publish Date 38 | val publishDateTime = parsePublishDateTime(box) 39 | 40 | // -- 41 | val info = MemeInfo( 42 | pageUrl = url, 43 | title = title, 44 | likes = likes, 45 | author = author, 46 | publishDateTime = publishDateTime 47 | ) 48 | list.add(info) 49 | } 50 | 51 | return Page(pageNumber, isHasNext(document), list) 52 | } 53 | 54 | private fun parseType(box: Element): String? { 55 | return box.select(".look_datei > .kleine_schrift").firstOrNull()?.text() 56 | } 57 | 58 | private fun parseHeader(box: Element): Pair { 59 | val link = box.select(".look_datei > a").first() 60 | 61 | val memeUrl = link.attr("href") 62 | val title = link.text() 63 | 64 | return memeUrl to title 65 | } 66 | 67 | private fun parseLikes(box: Element): Int { 68 | return box.select("td.look_bewertung .kleine_schrift").firstOrNull()?.text()?.toInt() ?: 0 69 | } 70 | 71 | private fun parseAuthor(box: Element): String? { 72 | return box.select("td.look_nickname a").firstOrNull()?.text() 73 | } 74 | 75 | private fun parsePublishDateTime(box: Element): LocalDateTime { 76 | val dateText = box.select("td.look_freigabedatum").text() 77 | 78 | // Date unification 79 | val now = LocalDateTime.now() 80 | val yearString = now.year.toString().substring(2) 81 | 82 | val unifiedDateText = dateText 83 | .replace("heute", "${now.dayOfMonth}.${now.monthValue}.") 84 | .replace(".-", ".$yearString-") 85 | 86 | // -- 87 | return LocalDateTime.parse(unifiedDateText, DATE_EXTRACTOR) 88 | } 89 | 90 | private fun isHasNext(document: Element): Boolean { 91 | return document.select("a > img[src=https://funpot.net/includes/logos/pfeil_rechts.gif]").isNotEmpty() 92 | } 93 | 94 | override suspend fun parseMeme(info: MemeInfo, body: String): MemeInfo { 95 | val document = Jsoup.parse(body).also { it.setBaseUri(baseUrl) } 96 | val container = document.getElementById("content") 97 | 98 | // -- 99 | val resourceUrl = parseResourceUrl(container) 100 | 101 | // -- 102 | return MemeInfo( 103 | pageUrl = info.pageUrl, 104 | originUrl = resourceUrl, 105 | title = info.title, 106 | likes = info.likes, 107 | author = info.author, 108 | publishDateTime = info.publishDateTime 109 | ) 110 | } 111 | 112 | private fun parseResourceUrl(container: Element): String { 113 | return container.getElementById("Direktdownload")?.absUrl("href") 114 | ?: container.select("video > source").first().absUrl("src") 115 | } 116 | 117 | companion object { 118 | private val DATE_EXTRACTOR = DateTimeFormatter.ofPattern("d.M.yy'-'HH:mm") 119 | } 120 | 121 | } 122 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/channel/FuoriditestaImagesChannel.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | import me.ruslanys.ifunny.domain.Language 4 | import org.jsoup.Jsoup 5 | import org.jsoup.nodes.Element 6 | import org.springframework.stereotype.Component 7 | import java.time.LocalDate 8 | import java.time.format.DateTimeFormatter 9 | import java.util.* 10 | import java.util.regex.Pattern 11 | 12 | @Component 13 | class FuoriditestaImagesChannel : Channel(Language.ITALIAN, "http://www.fuoriditesta.it") { 14 | 15 | override suspend fun pagePath(pageNumber: Int): String { 16 | return "$baseUrl/immagini-divertenti/index-$pageNumber.php" 17 | } 18 | 19 | override suspend fun parsePage(pageNumber: Int, body: String): Page { 20 | val document = Jsoup.parse(body).apply { setBaseUri(baseUrl) } 21 | val boxes = document.select(".homeimmagini > .immaginidiv") 22 | 23 | val list = arrayListOf() 24 | 25 | for (box in boxes) { 26 | val (url, title) = parseHeader(box) 27 | val publishDateTime = parsePublishDate(box).atStartOfDay() 28 | 29 | // -- 30 | val info = MemeInfo(pageUrl = url, title = title, publishDateTime = publishDateTime) 31 | list.add(info) 32 | } 33 | 34 | return Page(pageNumber, isHasNext(document), list) 35 | } 36 | 37 | private fun parseHeader(box: Element): Pair { 38 | val link = box.selectFirst(".immaginititle > p > a") 39 | val url = link.absUrl("href") 40 | val title = link.text() 41 | 42 | return url to title 43 | } 44 | 45 | private fun parsePublishDate(box: Element): LocalDate { 46 | val metaText = box.select(".immaginititle").last().text() 47 | val dateMatcher = DATE_EXTRACTOR.matcher(metaText) 48 | 49 | return if (dateMatcher.find()) { 50 | val dateStr = dateMatcher.group(1).toLowerCase() 51 | LocalDate.parse(dateStr, DATE_FORMATTER) 52 | } else { 53 | throw IllegalStateException("Cannot parse publish date") 54 | } 55 | } 56 | 57 | private fun isHasNext(document: Element): Boolean { 58 | return document.select(".pagbloc").text().contains("»") 59 | } 60 | 61 | override suspend fun parseMeme(info: MemeInfo, body: String): MemeInfo { 62 | val document = Jsoup.parse(body).apply { setBaseUri(baseUrl) } 63 | 64 | val resourceUrl = parsePictureUrl(document) 65 | 66 | return MemeInfo( 67 | pageUrl = info.pageUrl, 68 | title = info.title, 69 | publishDateTime = info.publishDateTime, 70 | originUrl = resourceUrl 71 | ) 72 | } 73 | 74 | private fun parsePictureUrl(document: Element): String { 75 | return document.selectFirst(".container .imgcont img").absUrl("src") 76 | } 77 | 78 | companion object { 79 | private val DATE_EXTRACTOR = Pattern.compile("Inserita il (.*) su") 80 | private val DATE_FORMATTER = DateTimeFormatter.ofPattern("d MMMM yyyy", Locale.ITALIAN) 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/channel/FuoriditestaVideoChannel.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | import me.ruslanys.ifunny.domain.Language 4 | import org.jsoup.Jsoup 5 | import org.jsoup.nodes.Element 6 | import org.springframework.stereotype.Component 7 | import org.springframework.web.reactive.function.client.WebClient 8 | import org.springframework.web.reactive.function.client.awaitBody 9 | import java.time.LocalDate 10 | import java.time.format.DateTimeFormatter 11 | import java.util.* 12 | import java.util.regex.Pattern 13 | 14 | @Component 15 | class FuoriditestaVideoChannel(private val webClient: WebClient) : Channel(Language.ITALIAN, "http://www.fuoriditesta.it") { 16 | 17 | override suspend fun pagePath(pageNumber: Int): String { 18 | return "$baseUrl/video-divertenti/index-$pageNumber.php" 19 | } 20 | 21 | override suspend fun parsePage(pageNumber: Int, body: String): Page { 22 | val document = Jsoup.parse(body).apply { setBaseUri(baseUrl) } 23 | val boxes = document.select(".homevideo > .videodiv") 24 | 25 | val list = arrayListOf() 26 | 27 | for (box in boxes) { 28 | val (url, title) = parseHeader(box) ?: continue 29 | 30 | // -- 31 | val info = MemeInfo(pageUrl = url, title = title) 32 | list.add(info) 33 | } 34 | 35 | return Page(pageNumber, isHasNext(document), list) 36 | } 37 | 38 | private fun parseHeader(box: Element): Pair? { 39 | val link = box.selectFirst(".videotitle > p > a") ?: return null 40 | val url = link.absUrl("href") 41 | val title = link.text() 42 | 43 | return url to title 44 | } 45 | 46 | private fun isHasNext(document: Element): Boolean { 47 | return document.select(".pagbloc").text().contains("»") 48 | } 49 | 50 | override suspend fun parseMeme(info: MemeInfo, body: String): MemeInfo { 51 | val document = Jsoup.parse(body).apply { setBaseUri(baseUrl) } 52 | 53 | val publishDateTime = parsePublishDate(document).atStartOfDay() 54 | val resourceUrl = parseVideoUrl(document) 55 | 56 | return MemeInfo( 57 | pageUrl = info.pageUrl, 58 | title = info.title, 59 | publishDateTime = publishDateTime, 60 | originUrl = resourceUrl 61 | ) 62 | } 63 | 64 | private fun parsePublishDate(box: Element): LocalDate { 65 | val metaText = box.select(".content .related").text() 66 | val dateMatcher = DATE_EXTRACTOR.matcher(metaText) 67 | 68 | return if (dateMatcher.find()) { 69 | val dateStr = dateMatcher.group(1).toLowerCase() 70 | LocalDate.parse(dateStr, DATE_FORMATTER) 71 | } else { 72 | throw IllegalStateException("Cannot parse publish date") 73 | } 74 | } 75 | 76 | private suspend fun parseVideoUrl(document: Element): String { 77 | val frameUrl = document.selectFirst(".container iframe").absUrl("src") 78 | val frameBody = webClient.get().uri(frameUrl).retrieve().awaitBody() 79 | val frameDocument = Jsoup.parse(frameBody) 80 | return frameDocument.selectFirst("#my-video > source").attr("src") 81 | } 82 | 83 | companion object { 84 | private val DATE_EXTRACTOR = Pattern.compile("Video inserito il (.*) Condividi") 85 | private val DATE_FORMATTER = DateTimeFormatter.ofPattern("d MMMM yyyy", Locale.ITALIAN) 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/channel/LachschonChannel.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | import me.ruslanys.ifunny.domain.Language 4 | import org.jsoup.Jsoup 5 | import org.jsoup.nodes.Element 6 | import org.springframework.stereotype.Component 7 | import java.time.LocalDateTime 8 | import java.time.format.DateTimeFormatter 9 | 10 | @Component 11 | class LachschonChannel : Channel(Language.GERMAN, "https://www.lachschon.de") { 12 | 13 | override suspend fun pagePath(pageNumber: Int): String { 14 | return "$baseUrl/?set_gallery_type=image&page=$pageNumber" 15 | } 16 | 17 | override suspend fun parsePage(pageNumber: Int, body: String): Page { 18 | val document = Jsoup.parse(body).also { it.setBaseUri(baseUrl) } 19 | val boxes = document.select("#itemlist > li") 20 | 21 | val list = arrayListOf() 22 | 23 | for (box in boxes) { 24 | val (url, title) = parseHeader(box) 25 | 26 | val author = parseAuthor(box) 27 | val likes = parseLikes(box) 28 | val comments = parseComments(box) 29 | 30 | val info = MemeInfo( 31 | pageUrl = url, 32 | title = title, 33 | author = author, 34 | likes = likes, 35 | comments = comments 36 | ) 37 | list.add(info) 38 | } 39 | 40 | return Page(pageNumber, isHasNext(document), list) 41 | } 42 | 43 | private fun parseHeader(box: Element): Pair { 44 | val link = box.selectFirst("a.title") 45 | val memeUrl = link.absUrl("href") 46 | val title = link.text() 47 | 48 | return memeUrl to title 49 | } 50 | 51 | private fun parseAuthor(box: Element): String { 52 | val element = box.select("a.username").firstOrNull() ?: box.select("span.username_guest").first() 53 | return element.text() 54 | } 55 | 56 | private fun parseLikes(box: Element): Int { 57 | return box.selectFirst("a.favs").text().toInt() 58 | } 59 | 60 | private fun parseComments(box: Element): Int { 61 | return box.selectFirst("a.comments").text().toInt() 62 | } 63 | 64 | private fun isHasNext(document: Element): Boolean { 65 | return document.select("a.forward").isNotEmpty() 66 | } 67 | 68 | override suspend fun parseMeme(info: MemeInfo, body: String): MemeInfo { 69 | val document = Jsoup.parse(body).also { it.setBaseUri(baseUrl) } 70 | val box = document.getElementById("main_content") 71 | 72 | val resourceUrl = parseResourceUrl(box) 73 | val publishDateTime = parsePublishDateTime(box) 74 | 75 | return MemeInfo( 76 | pageUrl = info.pageUrl, 77 | originUrl = resourceUrl, 78 | title = info.title, 79 | publishDateTime = publishDateTime, 80 | likes = info.likes, 81 | comments = info.comments, 82 | author = info.author 83 | ) 84 | } 85 | 86 | private fun parsePublishDateTime(box: Element): LocalDateTime { 87 | val dateStr = box.select("span.created-at").text() 88 | return LocalDateTime.parse(dateStr, DATE_TIME_FORMATTER) 89 | } 90 | 91 | private fun parseResourceUrl(box: Element): String { 92 | return box.selectFirst("img.item_view").absUrl("src") 93 | } 94 | 95 | companion object { 96 | private val DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("dd.MM.yyyy',' HH:mm:ss") 97 | } 98 | 99 | } 100 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/channel/MemeInfo.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | import java.time.LocalDateTime 4 | 5 | data class MemeInfo( 6 | val pageUrl: String? = null, 7 | val originUrl: String? = null, 8 | val title: String? = null, 9 | val publishDateTime: LocalDateTime? = null, 10 | val likes: Int? = null, 11 | val comments: Int? = null, 12 | val author: String? = null 13 | ) { 14 | 15 | override fun equals(other: Any?): Boolean { 16 | if (this === other) return true 17 | if (javaClass != other?.javaClass) return false 18 | 19 | other as MemeInfo 20 | 21 | if (pageUrl != other.pageUrl) return false 22 | 23 | return true 24 | } 25 | 26 | override fun hashCode(): Int { 27 | return pageUrl?.hashCode() ?: 0 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/channel/OrschlurchChannel.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | import me.ruslanys.ifunny.domain.Language 4 | import org.jsoup.Jsoup 5 | import org.jsoup.nodes.Element 6 | import org.springframework.stereotype.Component 7 | import java.time.LocalDateTime 8 | import java.time.ZonedDateTime 9 | 10 | @Component 11 | class OrschlurchChannel : Channel(Language.GERMAN, "https://de.orschlurch.net") { 12 | 13 | override suspend fun pagePath(pageNumber: Int): String { 14 | return if (pageNumber == 1) { 15 | "$baseUrl/area/videos" 16 | } else { 17 | "$baseUrl/area/videos/seite/$pageNumber" 18 | } 19 | } 20 | 21 | override suspend fun parsePage(pageNumber: Int, body: String): Page { 22 | val document = Jsoup.parse(body).also { it.setBaseUri(baseUrl) } 23 | val boxes = document.getElementsByClass("portfolio-item") 24 | 25 | val list = arrayListOf() 26 | 27 | for (box in boxes) { 28 | val type = parseType(box) 29 | if (type != "Videos") { 30 | continue // skip not videos 31 | } 32 | 33 | val (url, title) = parseHeader(box) 34 | val likes = parseLikes(box) 35 | val comments = parseComments(box) 36 | 37 | // -- 38 | val info = MemeInfo(pageUrl = url, title = title, likes = likes, comments = comments) 39 | list.add(info) 40 | } 41 | 42 | return Page(pageNumber, isHasNext(document), list) 43 | } 44 | 45 | private fun parseType(box: Element): String { 46 | return box.select(".card-cat > a").text() 47 | } 48 | 49 | private fun parseHeader(box: Element): Pair { 50 | val link = box.selectFirst(".card-title > a") 51 | val url = link.absUrl("href") 52 | val title = link.text() 53 | 54 | return url to title 55 | } 56 | 57 | private fun parseLikes(box: Element): Int { 58 | return box.selectFirst(".card-meta .like").text().toInt() 59 | } 60 | 61 | private fun parseComments(box: Element): Int { 62 | return box.select(".card-meta span").last().text().toInt() 63 | } 64 | 65 | private fun isHasNext(document: Element): Boolean { 66 | return document.select(".pagination").text().contains("›") 67 | } 68 | 69 | override suspend fun parseMeme(info: MemeInfo, body: String): MemeInfo { 70 | val document = Jsoup.parse(body).also { it.setBaseUri(baseUrl) } 71 | 72 | // -- 73 | val publishDateTime = parsePublishDateTime(document) 74 | val author = parseAuthor(document) 75 | val resourceUrl = parseVideoUrl(document) 76 | 77 | // -- 78 | return MemeInfo(info.pageUrl, resourceUrl, info.title, publishDateTime, info.likes, info.comments, author) 79 | } 80 | 81 | private fun parsePublishDateTime(document: Element): LocalDateTime { 82 | val meta = document.selectFirst("meta[property=article:published_time]").attr("content") 83 | return ZonedDateTime.parse(meta).toLocalDateTime() 84 | } 85 | 86 | private fun parseAuthor(document: Element): String { 87 | return document.selectFirst(".uploader a").text() 88 | } 89 | 90 | private fun parseVideoUrl(document: Element): String { 91 | return document.selectFirst(".container > video > source").absUrl("src") 92 | } 93 | 94 | } 95 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/channel/Page.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | data class Page(val number: Int, val hasNext: Boolean, val memesInfo: List) 4 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/channel/RigolotesChannel.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | import kotlinx.coroutines.reactive.awaitFirstOrNull 4 | import kotlinx.coroutines.reactive.awaitSingle 5 | import me.ruslanys.ifunny.domain.Language 6 | import org.jsoup.Jsoup 7 | import org.jsoup.nodes.Element 8 | import org.springframework.data.redis.core.ReactiveRedisTemplate 9 | import org.springframework.data.redis.core.ReactiveValueOperations 10 | import org.springframework.stereotype.Component 11 | import org.springframework.web.reactive.function.client.WebClient 12 | import org.springframework.web.reactive.function.client.awaitBody 13 | import java.time.Duration 14 | import java.time.LocalDateTime 15 | import java.time.format.DateTimeFormatter 16 | 17 | @Component 18 | class RigolotesChannel( 19 | redisTemplate: ReactiveRedisTemplate, 20 | private val webClient: WebClient 21 | ) : Channel(Language.FRENCH, "https://rigolotes.fr") { 22 | 23 | private val valueOperations: ReactiveValueOperations = redisTemplate.opsForValue() 24 | 25 | override suspend fun pagePath(pageNumber: Int): String { 26 | val pages = getPagesNumber() ?: fetchPagesNumber() 27 | val current = pages - (pageNumber - 1) 28 | 29 | return "$baseUrl/page/$current" 30 | } 31 | 32 | private suspend fun getPagesNumber(): Int? { 33 | return valueOperations[pagesKey()].awaitFirstOrNull() as Int? 34 | } 35 | 36 | private suspend fun fetchPagesNumber(): Int { 37 | val body = webClient.get().uri(baseUrl).retrieve().awaitBody() 38 | val document = Jsoup.parse(body) 39 | val current = document.selectFirst("div.page-numbers > a.font-weight-bold").text() 40 | val pages = current.toInt() 41 | 42 | valueOperations.setIfAbsent(pagesKey(), pages, Duration.ofHours(1)).awaitSingle() 43 | 44 | return pages 45 | } 46 | 47 | private fun pagesKey() = "${getName()}:pages" 48 | 49 | override suspend fun parsePage(pageNumber: Int, body: String): Page { 50 | val document = Jsoup.parse(body).also { it.setBaseUri(baseUrl) } 51 | val boxes = document.select("div.articles-container > div.article-box") 52 | 53 | val list = arrayListOf() 54 | 55 | for (box in boxes) { 56 | if (box.select("div.video-container > iframe").isNotEmpty()) { 57 | continue // skip youtube videos 58 | } 59 | 60 | val (url, title) = parseHeader(box) 61 | 62 | val votes = parseVotes(box) 63 | if (votes < 0) { 64 | continue // skip negative 65 | } 66 | 67 | val publishDateTime = parsePublishDateTime(box) 68 | val author = parseAuthor(box) 69 | 70 | val info = MemeInfo( 71 | pageUrl = url, 72 | title = title, 73 | likes = votes, 74 | publishDateTime = publishDateTime, 75 | author = author 76 | ) 77 | list.add(info) 78 | } 79 | 80 | return Page(pageNumber, isHasNext(document), list) 81 | } 82 | 83 | private fun parseHeader(box: Element): Pair { 84 | val link = box.selectFirst("h2 > a") 85 | 86 | val memeUrl = link.absUrl("href") 87 | val title = link.text() 88 | 89 | return memeUrl to title 90 | } 91 | 92 | private fun parseVotes(box: Element): Int { 93 | return box.selectFirst("div.votes > span.upvotes").text() 94 | .replace("+", "") // remove plus sign 95 | .replace(" ", "") // remove spaces 96 | .toInt() 97 | } 98 | 99 | private fun parsePublishDateTime(box: Element): LocalDateTime { 100 | val dateStr = box.select("div.info > span").last().text() 101 | return LocalDateTime.parse(dateStr, DATE_TIME_FORMATTER) 102 | } 103 | 104 | private fun parseAuthor(box: Element): String { 105 | return box.select("div.info > a")[0].text() 106 | } 107 | 108 | private fun isHasNext(document: Element): Boolean { 109 | return document.select("div.pagination-menu div.next-button a").text().contains("Suivant") 110 | } 111 | 112 | override suspend fun parseMeme(info: MemeInfo, body: String): MemeInfo { 113 | val document = Jsoup.parse(body).also { it.setBaseUri(baseUrl) } 114 | val box = document.selectFirst(".article-box") 115 | 116 | val videoUrl = parseVideoUrl(box) 117 | val pictureUrl = parsePictureUrl(box) 118 | 119 | return MemeInfo( 120 | pageUrl = info.pageUrl, 121 | originUrl = videoUrl ?: pictureUrl, 122 | title = info.title, 123 | publishDateTime = info.publishDateTime, 124 | likes = info.likes, 125 | author = info.author 126 | ) 127 | } 128 | 129 | private fun parsePictureUrl(box: Element): String? { 130 | val image = box.selectFirst("div.center-block > a > img") ?: return null 131 | return image.absUrl("src") 132 | } 133 | 134 | private fun parseVideoUrl(box: Element): String? { 135 | val video = box.selectFirst("div.center-block > video > source") ?: return null 136 | return video.absUrl("src") 137 | } 138 | 139 | companion object { 140 | private val DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("dd-MM-yyyy HH:mm") 141 | } 142 | 143 | } 144 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/channel/YatahongaChannel.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties 4 | import com.fasterxml.jackson.databind.ObjectMapper 5 | import me.ruslanys.ifunny.domain.Language 6 | import org.jsoup.Jsoup 7 | import org.jsoup.nodes.Element 8 | import org.springframework.stereotype.Component 9 | import java.time.LocalDate 10 | import java.time.format.DateTimeFormatter 11 | 12 | @Component 13 | class YatahongaChannel(private val objectMapper: ObjectMapper) : Channel(Language.FRENCH, "https://www.yatahonga.com") { 14 | 15 | override suspend fun pagePath(pageNumber: Int): String { 16 | return if (pageNumber == 1) { 17 | "$baseUrl/nouveautes/" 18 | } else { 19 | "$baseUrl/nouveautes/p${pageNumber}/" 20 | } 21 | } 22 | 23 | override suspend fun parsePage(pageNumber: Int, body: String): Page { 24 | val document = Jsoup.parse(body) 25 | val articles = document.getElementsByTag("article") 26 | 27 | val list = arrayListOf() 28 | 29 | for (article in articles) { 30 | val (url, title) = parseHeader(article) ?: continue 31 | 32 | val points = parsePoints(article) 33 | if (points < 0) { 34 | continue // skip negative 35 | } 36 | 37 | val comments = parseComments(article) 38 | 39 | // -- 40 | val info = MemeInfo(pageUrl = url, title = title, likes = points, comments = comments) 41 | list.add(info) 42 | } 43 | 44 | return Page(pageNumber, isHasNext(document), list) 45 | } 46 | 47 | private fun parseHeader(article: Element): Pair? { 48 | val link = article.select("header a").firstOrNull() ?: return null 49 | val memeUrl = link.attr("href") 50 | val title = link.text() 51 | 52 | return memeUrl to title 53 | } 54 | 55 | private fun parsePoints(article: Element): Int { 56 | return article.select("p.post-meta > span.badge-item-love-count").text().toInt() 57 | } 58 | 59 | private fun parseComments(article: Element): Int { 60 | return article.select("p.post-meta > a.comment").text() 61 | .split(" ")[0] 62 | .toInt() 63 | } 64 | 65 | private fun isHasNext(document: Element): Boolean { 66 | return document.select("div.pagingbuttons > a.next").isNotEmpty() 67 | } 68 | 69 | override suspend fun parseMeme(info: MemeInfo, body: String): MemeInfo { 70 | val document = Jsoup.parse(body) 71 | val json = document.selectFirst("script[type=application/ld+json]").data() 72 | 73 | val post = objectMapper.readValue(json, BlogPost::class.java) 74 | 75 | val author = post.author 76 | val publishDateTime = LocalDate.parse(post.datePublished, DATE_EXTRACTOR).atStartOfDay() 77 | val resourceUrl = if (post.video != null) { 78 | post.video.url 79 | } else { 80 | post.image.url 81 | } 82 | 83 | return MemeInfo( 84 | pageUrl = info.pageUrl, 85 | originUrl = resourceUrl, 86 | title = info.title, 87 | publishDateTime = publishDateTime, 88 | likes = info.likes, 89 | comments = info.comments, 90 | author = author 91 | ) 92 | } 93 | 94 | 95 | @JsonIgnoreProperties(ignoreUnknown = true) 96 | data class BlogPost( 97 | val headline: String, 98 | val author: String, 99 | val datePublished: String, 100 | val dateModified: String, 101 | val image: Object, 102 | val video: Object? 103 | ) { 104 | @JsonIgnoreProperties(ignoreUnknown = true) 105 | data class Object(val url: String) 106 | } 107 | 108 | companion object { 109 | private val DATE_EXTRACTOR = DateTimeFormatter.ofPattern("yyyy-MM-dd") 110 | } 111 | 112 | } 113 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/config/AwsS3Config.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.config 2 | 3 | import kotlinx.coroutines.future.await 4 | import kotlinx.coroutines.runBlocking 5 | import me.ruslanys.ifunny.property.AwsS3Properties 6 | import org.slf4j.LoggerFactory 7 | import org.springframework.context.annotation.Bean 8 | import org.springframework.context.annotation.Configuration 9 | import software.amazon.awssdk.auth.credentials.AwsBasicCredentials 10 | import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider 11 | import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient 12 | import software.amazon.awssdk.regions.Region 13 | import software.amazon.awssdk.services.s3.S3AsyncClient 14 | import software.amazon.awssdk.services.s3.S3AsyncClientBuilder 15 | import software.amazon.awssdk.services.s3.model.BucketCannedACL 16 | import software.amazon.awssdk.services.s3.model.CreateBucketRequest 17 | import software.amazon.awssdk.services.s3.model.HeadBucketRequest 18 | import software.amazon.awssdk.services.s3.model.NoSuchBucketException 19 | import java.time.Duration 20 | import javax.annotation.PostConstruct 21 | 22 | @Configuration 23 | class AwsS3Config(private val properties: AwsS3Properties) { 24 | 25 | @Bean 26 | fun s3ClientBuilder(): S3AsyncClientBuilder = S3AsyncClient.builder() 27 | .httpClientBuilder(NettyNioAsyncHttpClient.builder() 28 | .writeTimeout(Duration.ofSeconds(10)) 29 | .readTimeout(Duration.ofSeconds(10)) 30 | .connectionMaxIdleTime(Duration.ofSeconds(10)) 31 | .connectionTimeout(Duration.ofSeconds(10)) 32 | .connectionAcquisitionTimeout(Duration.ofSeconds(10)) 33 | ) 34 | .serviceConfiguration { 35 | it.checksumValidationEnabled(false) 36 | it.chunkedEncodingEnabled(true) 37 | } 38 | .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(properties.accessKey, properties.secretKey))) 39 | .region(Region.of(properties.region)) 40 | 41 | 42 | @PostConstruct 43 | fun createBucket() = runBlocking { 44 | val client = s3ClientBuilder().build() 45 | try { 46 | val request = HeadBucketRequest.builder().bucket(properties.bucket).build() 47 | client.headBucket(request).await() 48 | } catch (e: NoSuchBucketException) { 49 | val request = CreateBucketRequest.builder() 50 | .acl(BucketCannedACL.PUBLIC_READ) 51 | .bucket(properties.bucket) 52 | .build() 53 | val response = client.createBucket(request).await() 54 | log.warn("A new S3 bucket created {}.", response.location()) 55 | } finally { 56 | client.close() 57 | } 58 | } 59 | 60 | 61 | companion object { 62 | private val log = LoggerFactory.getLogger(AwsS3Config::class.java) 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/config/EventChannelConfig.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.config 2 | 3 | import kotlinx.coroutines.channels.Channel 4 | import me.ruslanys.ifunny.grab.event.GrabEvent 5 | import me.ruslanys.ifunny.property.GrabProperties 6 | import org.springframework.context.annotation.Bean 7 | import org.springframework.context.annotation.Configuration 8 | 9 | @Configuration 10 | class EventChannelConfig(private val properties: GrabProperties) { 11 | 12 | @Bean 13 | fun eventChannel(): Channel { 14 | return Channel(properties.channelSize) 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/config/EventListenersConfig.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.config 2 | 3 | import kotlinx.coroutines.GlobalScope 4 | import kotlinx.coroutines.asCoroutineDispatcher 5 | import kotlinx.coroutines.channels.Channel 6 | import kotlinx.coroutines.launch 7 | import me.ruslanys.ifunny.Application 8 | import me.ruslanys.ifunny.grab.SuspendedEventListener 9 | import me.ruslanys.ifunny.grab.event.GrabEvent 10 | import org.slf4j.LoggerFactory 11 | import org.springframework.context.annotation.Configuration 12 | import java.util.concurrent.Executors 13 | import javax.annotation.PostConstruct 14 | import kotlin.math.max 15 | import kotlin.reflect.full.isSubtypeOf 16 | import kotlin.reflect.full.starProjectedType 17 | 18 | @Configuration 19 | class EventListenersConfig( 20 | listeners: List>, 21 | private val eventChannel: Channel 22 | ) { 23 | 24 | private val listeners = listeners.groupBy { listener -> 25 | listener::class.supertypes.first { it.isSubtypeOf(SuspendedEventListener::class.starProjectedType) }.arguments[0].type 26 | } 27 | 28 | @PostConstruct 29 | fun initCoroutines() = GlobalScope.launch { 30 | val threadsNumber = max(Runtime.getRuntime().availableProcessors(), 8) 31 | val coroutinesNumber = threadsNumber * 2 32 | 33 | val workerContext = Executors.newFixedThreadPool(threadsNumber).asCoroutineDispatcher() 34 | 35 | repeat(coroutinesNumber) { 36 | launch(workerContext) { 37 | handleEvents() 38 | } 39 | } 40 | } 41 | 42 | private suspend fun handleEvents() { 43 | for (event in eventChannel) { 44 | listeners[event::class.starProjectedType]?.forEach { 45 | try { 46 | @Suppress("UNCHECKED_CAST") 47 | (it as SuspendedEventListener).handleEvent(event) 48 | } catch (ex: Exception) { 49 | log.error("Unexpected exception occurred invoking listener: ${it::class}, with event: $event", ex) 50 | } 51 | } 52 | } 53 | } 54 | 55 | companion object { 56 | private val log = LoggerFactory.getLogger(Application::class.java) 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/config/ExecutorsConfig.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.config 2 | 3 | import me.ruslanys.ifunny.Application 4 | import org.slf4j.LoggerFactory 5 | import org.springframework.aop.interceptor.AsyncUncaughtExceptionHandler 6 | import org.springframework.boot.task.TaskExecutorBuilder 7 | import org.springframework.context.annotation.Bean 8 | import org.springframework.context.annotation.Configuration 9 | import org.springframework.scheduling.annotation.AsyncConfigurer 10 | import org.springframework.scheduling.annotation.EnableAsync 11 | import org.springframework.scheduling.annotation.EnableScheduling 12 | import java.util.concurrent.Executor 13 | 14 | @Configuration 15 | @EnableScheduling 16 | @EnableAsync 17 | class ExecutorsConfig(private val taskExecutorBuilder: TaskExecutorBuilder) : AsyncConfigurer { 18 | 19 | @Bean(name = ["asyncExecutor"]) 20 | override fun getAsyncExecutor(): Executor { 21 | return taskExecutorBuilder.build() 22 | } 23 | 24 | override fun getAsyncUncaughtExceptionHandler() = AsyncUncaughtExceptionHandler { ex, method, params -> 25 | if (!log.isErrorEnabled) { 26 | return@AsyncUncaughtExceptionHandler 27 | } 28 | 29 | log.error("Unexpected exception occurred invoking async method: $method, with params: ${params.toList()}", ex) 30 | } 31 | 32 | companion object { 33 | private val log = LoggerFactory.getLogger(Application::class.java) 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/config/RedisConfig.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.config 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper 4 | import org.springframework.context.annotation.Bean 5 | import org.springframework.context.annotation.Configuration 6 | import org.springframework.data.redis.connection.ReactiveRedisConnectionFactory 7 | import org.springframework.data.redis.core.ReactiveRedisTemplate 8 | import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer 9 | import org.springframework.data.redis.serializer.RedisSerializationContext 10 | 11 | 12 | @Configuration 13 | class RedisConfig { 14 | 15 | @Bean 16 | fun reactiveRedisTemplate(factory: ReactiveRedisConnectionFactory, objectMapper: ObjectMapper): ReactiveRedisTemplate { 17 | val serializer = GenericJackson2JsonRedisSerializer(objectMapper) 18 | val serializationContext = RedisSerializationContext.newSerializationContext(serializer).build() 19 | return ReactiveRedisTemplate(factory, serializationContext) 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/config/WebClientConfig.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.config 2 | 3 | import io.netty.handler.timeout.ReadTimeoutHandler 4 | import io.netty.handler.timeout.WriteTimeoutHandler 5 | import me.ruslanys.ifunny.property.GrabProperties 6 | import org.springframework.context.annotation.Bean 7 | import org.springframework.context.annotation.Configuration 8 | import org.springframework.http.client.reactive.ReactorClientHttpConnector 9 | import org.springframework.http.client.reactive.ReactorResourceFactory 10 | import org.springframework.web.reactive.function.client.WebClient 11 | import reactor.netty.resources.ConnectionProvider 12 | import reactor.netty.resources.LoopResources 13 | 14 | 15 | /** 16 | * https://github.com/spring-projects/spring-framework/blob/master/src/docs/asciidoc/web/webflux-webclient.adoc 17 | */ 18 | @Configuration 19 | class WebClientConfig(private val properties: GrabProperties) { 20 | 21 | @Bean 22 | fun resourceFactory() = ReactorResourceFactory().apply { 23 | isUseGlobalResources = false 24 | connectionProvider = ConnectionProvider.fixed("crawler", 100, 10_000) 25 | loopResources = LoopResources.create("crawler", 1, 4, true) 26 | } 27 | 28 | @Bean 29 | fun webClient(): WebClient = WebClient.builder() 30 | .codecs { 31 | it.defaultCodecs().maxInMemorySize(2 * 1024 * 1024) 32 | } 33 | .clientConnector(ReactorClientHttpConnector(resourceFactory()) { client -> 34 | client.tcpConfiguration { tcpClient -> 35 | tcpClient.doOnConnected { connection -> 36 | connection.addHandlerLast(ReadTimeoutHandler(10)) 37 | connection.addHandlerLast(WriteTimeoutHandler(10)) 38 | } 39 | } 40 | 41 | client.secure() 42 | client.keepAlive(false) 43 | client.followRedirect(true) 44 | }) 45 | .defaultHeader("User-Agent", properties.userAgent) 46 | .build() 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/controller/FeedController.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.controller 2 | 3 | import org.springframework.stereotype.Controller 4 | import org.springframework.web.bind.annotation.GetMapping 5 | 6 | @Controller 7 | class FeedController { 8 | 9 | @GetMapping("/", "/index", "/index.htm", "/index.html", "/feed", "/feed.html") 10 | fun showFeed(): String = "feed" 11 | 12 | } 13 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/controller/GlobalExceptionHandler.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.controller 2 | 3 | import me.ruslanys.ifunny.controller.dto.ErrorDto 4 | import me.ruslanys.ifunny.exception.NotFoundException 5 | import org.springframework.http.HttpStatus 6 | import org.springframework.http.MediaType 7 | import org.springframework.http.ResponseEntity 8 | import org.springframework.web.bind.annotation.ControllerAdvice 9 | import org.springframework.web.bind.annotation.ExceptionHandler 10 | import org.springframework.web.bind.annotation.ResponseBody 11 | import org.springframework.web.server.ServerWebExchange 12 | 13 | /** 14 | * FYI possible ways for Error Handling customization: 15 | * - Define a bean of type [org.springframework.boot.web.reactive.error.ErrorAttributes]. 16 | * - Define a bean of type [org.springframework.boot.autoconfigure.web.reactive.error.AbstractErrorWebExceptionHandler]. 17 | */ 18 | @ControllerAdvice 19 | @ResponseBody 20 | class GlobalExceptionHandler { 21 | 22 | @ExceptionHandler 23 | fun handleIllegalArgumentException(exchange: ServerWebExchange, exception: IllegalArgumentException): ResponseEntity { 24 | return handleException(HttpStatus.BAD_REQUEST, exchange, exception) 25 | } 26 | 27 | @ExceptionHandler 28 | fun handleNotFoundException(exchange: ServerWebExchange, exception: NotFoundException): ResponseEntity { 29 | return handleException(HttpStatus.NOT_FOUND, exchange, exception) 30 | } 31 | 32 | private fun handleException(status: HttpStatus, exchange: ServerWebExchange, exception: Exception): ResponseEntity { 33 | return if (MediaType.APPLICATION_JSON.isCompatibleWith(exchange.request.headers.accept.firstOrNull())) { 34 | ResponseEntity(ErrorDto(exchange.request.path.toString(), status, exception.message), HttpStatus.NOT_FOUND) 35 | } else { 36 | ResponseEntity(exception.message, status) 37 | } 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/controller/MemeController.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.controller 2 | 3 | import me.ruslanys.ifunny.controller.dto.MemeDto 4 | import me.ruslanys.ifunny.service.MemeService 5 | import org.springframework.stereotype.Controller 6 | import org.springframework.ui.Model 7 | import org.springframework.web.bind.annotation.GetMapping 8 | import org.springframework.web.bind.annotation.PathVariable 9 | 10 | @Controller 11 | class MemeController(private val memeService: MemeService) { 12 | 13 | @GetMapping("/meme/{id}") 14 | suspend fun showMeme(@PathVariable id: String, model: Model): String { 15 | val meme = memeService.getById(id) 16 | model.addAttribute("meme", MemeDto(meme)) 17 | return "meme" 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/controller/SwaggerUiController.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.controller 2 | 3 | import org.springframework.stereotype.Controller 4 | import org.springframework.web.bind.annotation.GetMapping 5 | 6 | @Controller 7 | class SwaggerUiController { 8 | 9 | @GetMapping("/swagger-ui", "/swagger-ui.html") 10 | fun swaggerUi(): String { 11 | return "redirect:/swagger-ui/index.html" 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/controller/api/FeedApiController.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.controller.api 2 | 3 | import me.ruslanys.ifunny.controller.dto.MemeDto 4 | import me.ruslanys.ifunny.controller.dto.PageRequest 5 | import me.ruslanys.ifunny.controller.dto.PageResponse 6 | import me.ruslanys.ifunny.domain.Language 7 | import me.ruslanys.ifunny.service.MemeService 8 | import org.springframework.data.domain.Sort 9 | import org.springframework.web.bind.annotation.GetMapping 10 | import org.springframework.web.bind.annotation.PathVariable 11 | import org.springframework.web.bind.annotation.RequestMapping 12 | import org.springframework.web.bind.annotation.RestController 13 | import javax.validation.Valid 14 | import javax.validation.constraints.NotNull 15 | 16 | @RestController 17 | @RequestMapping("/api/feed") 18 | class FeedApiController(private val memeService: MemeService) { 19 | 20 | @GetMapping 21 | suspend fun getPage(@Valid searchRequest: FeedSearchRequest, @Valid pageRequest: FeedPageRequest): PageResponse { 22 | val language = Language.findByCode(searchRequest.language!!) 23 | val page = memeService.getPage(language, pageRequest).map { MemeDto(it) } 24 | return PageResponse(page) 25 | } 26 | 27 | @GetMapping("/{id}") 28 | suspend fun getById(@PathVariable("id") id: String): MemeDto { 29 | val meme = memeService.getById(id) 30 | return MemeDto(meme) 31 | } 32 | 33 | 34 | data class FeedSearchRequest( 35 | @field:NotNull 36 | val language: String? = null 37 | ) 38 | 39 | class FeedPageRequest : PageRequest( 40 | sortBy = "publishDateTime", 41 | sortDirection = Sort.Direction.DESC, 42 | maySortBy = setOf("publishDateTime", "likes") 43 | ) 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/controller/dto/ErrorDto.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.controller.dto 2 | 3 | import org.springframework.http.HttpStatus 4 | import java.util.* 5 | 6 | data class ErrorDto( 7 | val path: String, 8 | val status: Int, 9 | val error: String, 10 | val message: String?, 11 | val timestamp: Date = Date() 12 | ) { 13 | constructor(path: String, httpStatus: HttpStatus, message: String?) : this( 14 | path, 15 | httpStatus.value(), 16 | httpStatus.reasonPhrase, 17 | message 18 | ) 19 | } 20 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/controller/dto/MemeDto.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.controller.dto 2 | 3 | import me.ruslanys.ifunny.domain.Meme 4 | import java.time.LocalDateTime 5 | 6 | data class MemeDto( 7 | val id: String, 8 | val language: String, 9 | val channelName: String, 10 | val title: String, 11 | val url: String, 12 | val contentType: String, 13 | val publishDateTime: LocalDateTime?, 14 | val author: String?, 15 | val likes: Int?, 16 | val comments: Int? 17 | ) { 18 | constructor(meme: Meme) : this( 19 | meme.id.toHexString(), 20 | meme.language, 21 | meme.channelName, 22 | meme.title, 23 | "https://${meme.file.bucket}.s3.${meme.file.region}.amazonaws.com/${meme.file.name}", 24 | meme.file.contentType, 25 | meme.publishDateTime, 26 | meme.author, 27 | meme.likes, 28 | meme.comments 29 | ) 30 | } 31 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/controller/dto/PageRequest.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.controller.dto 2 | 3 | import me.ruslanys.ifunny.controller.validation.SortConstraint 4 | import org.springframework.data.domain.Pageable 5 | import org.springframework.data.domain.Sort 6 | import javax.validation.constraints.Max 7 | import javax.validation.constraints.Min 8 | 9 | @Suppress("unused") 10 | @SortConstraint 11 | open class PageRequest( 12 | @field:Min(value = 0) private var offset: Long = 0, 13 | @field:Min(value = 1) @field:Max(100) private var limit: Int = 10, 14 | var sortBy: String = "id", 15 | var sortDirection: Sort.Direction = Sort.Direction.ASC, 16 | private val maySortBy: Set = setOf("id") 17 | ) : Pageable { 18 | 19 | override fun getPageNumber(): Int = (offset / limit + 1).toInt() 20 | 21 | override fun next(): Pageable = PageRequest(offset + limit, limit, sortBy, sortDirection, maySortBy) 22 | 23 | override fun getPageSize(): Int = limit 24 | 25 | fun setLimit(limit: Int) { 26 | this.limit = limit 27 | } 28 | 29 | override fun getOffset(): Long = offset 30 | 31 | fun setOffset(offset: Long) { 32 | this.offset = offset 33 | } 34 | 35 | override fun hasPrevious(): Boolean = offset > 0 36 | 37 | override fun getSort(): Sort = Sort.by(sortDirection, sortBy) 38 | 39 | override fun first(): Pageable = PageRequest(0, limit, sortBy, sortDirection, maySortBy) 40 | 41 | private fun previous(): PageRequest { 42 | return if (offset == 0L) this else { 43 | var newOffset = this.offset - limit 44 | if (newOffset < 0) newOffset = 0 45 | PageRequest(newOffset, limit, sortBy, sortDirection, maySortBy) 46 | } 47 | } 48 | 49 | override fun previousOrFirst(): Pageable = if (hasPrevious()) previous() else first() 50 | 51 | fun getMaySortBy(): Set = maySortBy 52 | 53 | override fun equals(other: Any?): Boolean { 54 | if (this === other) return true 55 | if (javaClass != other?.javaClass) return false 56 | 57 | other as PageRequest 58 | 59 | if (offset != other.offset) return false 60 | if (limit != other.limit) return false 61 | if (sortBy != other.sortBy) return false 62 | if (sortDirection != other.sortDirection) return false 63 | if (maySortBy != other.maySortBy) return false 64 | 65 | return true 66 | } 67 | 68 | override fun hashCode(): Int { 69 | var result = offset 70 | result = 31 * result + limit 71 | result = 31 * result + sortBy.hashCode() 72 | result = 31 * result + sortDirection.hashCode() 73 | result = 31 * result + maySortBy.hashCode() 74 | return result.toInt() 75 | } 76 | 77 | } 78 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/controller/dto/PageResponse.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.controller.dto 2 | 3 | import org.springframework.data.domain.Page 4 | 5 | data class PageResponse(val totalCount: Long, val list: List) { 6 | constructor(page: Page) : this(page.totalElements, page.content) 7 | constructor(list: List) : this(list.size.toLong(), list) 8 | } 9 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/controller/validation/SortConstraint.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.controller.validation 2 | 3 | import javax.validation.Constraint 4 | import javax.validation.Payload 5 | import kotlin.reflect.KClass 6 | 7 | @Target(AnnotationTarget.CLASS, AnnotationTarget.FIELD) 8 | @Retention(AnnotationRetention.RUNTIME) 9 | @Constraint(validatedBy = [(SortValidator::class)]) 10 | annotation class SortConstraint( 11 | val message: String = "Collection is not contains value", 12 | val groups: Array> = [], 13 | val payload: Array> = [] 14 | ) 15 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/controller/validation/SortValidator.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.controller.validation 2 | 3 | import me.ruslanys.ifunny.controller.dto.PageRequest 4 | import javax.validation.ConstraintValidator 5 | import javax.validation.ConstraintValidatorContext 6 | 7 | class SortValidator : ConstraintValidator { 8 | 9 | override fun isValid(pageRequest: PageRequest, context: ConstraintValidatorContext): Boolean = 10 | pageRequest.getMaySortBy().contains(pageRequest.sortBy) 11 | 12 | override fun initialize(annotation: SortConstraint) { 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/domain/Language.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.domain 2 | 3 | enum class Language(val code: String) { 4 | 5 | GERMAN("de"), 6 | FRENCH("fr"), 7 | SPANISH("es"), 8 | ITALIAN("it"), 9 | PORTUGUESE("pt"), 10 | RUSSIAN("ru"); 11 | 12 | companion object { 13 | 14 | fun findByCode(code: String): Language = values().firstOrNull { 15 | it.code == code 16 | } ?: throw IllegalArgumentException("Unsupported language code $code") 17 | 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/domain/Meme.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.domain 2 | 3 | import org.bson.types.ObjectId 4 | import org.springframework.data.mongodb.core.index.CompoundIndex 5 | import org.springframework.data.mongodb.core.index.CompoundIndexes 6 | import org.springframework.data.mongodb.core.index.Indexed 7 | import org.springframework.data.mongodb.core.mapping.Document 8 | import org.springframework.data.mongodb.core.mapping.MongoId 9 | import java.time.LocalDateTime 10 | 11 | @Document 12 | @CompoundIndexes( 13 | CompoundIndex(name = "language2fingerprint_idx", def = "{'language': 1, 'fingerprint': 1}"), 14 | CompoundIndex(name = "language2publishDateTime_idx", def = "{'language': 1, 'publishDateTime': -1}"), 15 | CompoundIndex(name = "language2likes_idx", def = "{'language': 1, 'likes': -1}") 16 | ) 17 | data class Meme( 18 | 19 | @Indexed 20 | val language: String, 21 | val channelName: String, 22 | 23 | @Indexed(unique = true) 24 | val pageUrl: String, 25 | val title: String, 26 | 27 | val originUrl: String, 28 | val file: S3File, 29 | 30 | val fingerprint: String? = null, 31 | 32 | val publishDateTime: LocalDateTime? = null, 33 | val author: String? = null, 34 | val likes: Int? = null, 35 | val comments: Int? = null, 36 | 37 | @MongoId 38 | val id: ObjectId = ObjectId() 39 | 40 | ) 41 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/domain/S3File.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.domain 2 | 3 | data class S3File( 4 | val region: String, 5 | val bucket: String, 6 | val name: String, 7 | val contentType: String, 8 | val checksum: String, 9 | val size: Long 10 | ) 11 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/exception/NotFoundException.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.exception 2 | 3 | class NotFoundException(message: String) : RuntimeException(message) 4 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/grab/Coordinator.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.grab 2 | 3 | import kotlinx.coroutines.GlobalScope 4 | import kotlinx.coroutines.channels.SendChannel 5 | import kotlinx.coroutines.launch 6 | import kotlinx.coroutines.reactive.awaitFirstOrNull 7 | import kotlinx.coroutines.reactive.awaitSingle 8 | import me.ruslanys.ifunny.channel.Channel 9 | import me.ruslanys.ifunny.grab.event.GrabEvent 10 | import me.ruslanys.ifunny.grab.event.PageIndexRequest 11 | import me.ruslanys.ifunny.grab.event.PageIndexSuccessful 12 | import me.ruslanys.ifunny.property.GrabProperties 13 | import me.ruslanys.ifunny.repository.PageRepository 14 | import org.slf4j.LoggerFactory 15 | import org.springframework.scheduling.annotation.Scheduled 16 | import org.springframework.stereotype.Component 17 | 18 | @Component 19 | class Coordinator( 20 | private val channels: List, 21 | private val eventChannel: SendChannel, 22 | private val pageRepository: PageRepository, 23 | private val grabProperties: GrabProperties 24 | ) : SuspendedEventListener { 25 | 26 | @Scheduled(initialDelay = 10_000, fixedDelay = 3600_000) 27 | fun scheduleGrabbing() = GlobalScope.launch { 28 | for (channel in channels) { 29 | nextPageIndexRequest(channel) 30 | } 31 | } 32 | 33 | override suspend fun handleEvent(event: PageIndexSuccessful) { 34 | log.info("Page #{} from {} has been processed", event.page.number, event.channel.getName()) 35 | 36 | val isChannelIndexed = pageRepository.getLast(event.channel).awaitFirstOrNull() != null 37 | 38 | if (!isChannelIndexed) { 39 | handleForNotIndexedChannel(event) 40 | } else { 41 | handleForIndexedChannel(event) 42 | } 43 | } 44 | 45 | /** 46 | * Next page indexation request. 47 | */ 48 | private suspend fun nextPageIndexRequest(channel: Channel) { 49 | val pageNumber = pageRepository.incCurrent(channel).awaitSingle() 50 | eventChannel.send(PageIndexRequest(channel, pageNumber)) 51 | } 52 | 53 | /** 54 | * Handle success page indexation event when the channel not fully indexed yet. 55 | */ 56 | private suspend fun handleForNotIndexedChannel(event: PageIndexSuccessful) { 57 | val channel = event.channel 58 | 59 | if (event.page.hasNext) { 60 | // Keep grabbing until the end 61 | nextPageIndexRequest(channel) 62 | } else { 63 | // The end reached 64 | pageRepository.setLast(channel, event.page.number, grabProperties.retention.fullIndex).awaitSingle() 65 | pageRepository.clearCurrent(channel).awaitSingle() 66 | 67 | log.info("{} fully indexed.", channel.getName()) 68 | } 69 | } 70 | 71 | /** 72 | * Handle success page indexation event when the channel already fully indexed. 73 | */ 74 | private suspend fun handleForIndexedChannel(event: PageIndexSuccessful) { 75 | val channel = event.channel 76 | 77 | if (event.page.hasNext && event.new > 0) { 78 | nextPageIndexRequest(channel) 79 | } else { 80 | pageRepository.clearCurrent(channel).awaitSingle() 81 | 82 | log.info("{} new pages from {} indexed.", event.page.number, event.channel.getName()) 83 | } 84 | } 85 | 86 | companion object { 87 | private val log = LoggerFactory.getLogger(Coordinator::class.java) 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/grab/MemeIndexer.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.grab 2 | 3 | import kotlinx.coroutines.channels.SendChannel 4 | import me.ruslanys.ifunny.grab.event.GrabEvent 5 | import me.ruslanys.ifunny.grab.event.MemeIndexRequest 6 | import me.ruslanys.ifunny.grab.event.ResourceDownloadRequest 7 | import org.slf4j.LoggerFactory 8 | import org.springframework.stereotype.Component 9 | import org.springframework.web.reactive.function.client.WebClient 10 | import org.springframework.web.reactive.function.client.awaitBody 11 | 12 | @Component 13 | class MemeIndexer( 14 | private val webClient: WebClient, 15 | private val eventChannel: SendChannel 16 | ) : SuspendedEventListener { 17 | 18 | /** 19 | * Indexation specific meme. 20 | */ 21 | override suspend fun handleEvent(event: MemeIndexRequest) { 22 | val channel = event.channel 23 | val baseInfo = event.info 24 | 25 | // Fetch and parse page 26 | val responseBody = webClient.get().uri(baseInfo.pageUrl!!).retrieve().awaitBody() 27 | val info = channel.parseMeme(baseInfo, responseBody) 28 | 29 | // Request resource download 30 | eventChannel.send(ResourceDownloadRequest(channel, info)) 31 | } 32 | 33 | companion object { 34 | private val log = LoggerFactory.getLogger(MemeIndexer::class.java) 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/grab/PageIndexer.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.grab 2 | 3 | import kotlinx.coroutines.channels.SendChannel 4 | import me.ruslanys.ifunny.channel.Channel 5 | import me.ruslanys.ifunny.channel.MemeInfo 6 | import me.ruslanys.ifunny.grab.event.GrabEvent 7 | import me.ruslanys.ifunny.grab.event.MemeIndexRequest 8 | import me.ruslanys.ifunny.grab.event.PageIndexRequest 9 | import me.ruslanys.ifunny.grab.event.PageIndexSuccessful 10 | import me.ruslanys.ifunny.service.MemeService 11 | import org.slf4j.LoggerFactory 12 | import org.springframework.stereotype.Component 13 | import org.springframework.web.reactive.function.client.WebClient 14 | import org.springframework.web.reactive.function.client.awaitBody 15 | 16 | @Component 17 | class PageIndexer( 18 | private val webClient: WebClient, 19 | private val eventChannel: SendChannel, 20 | private val memeService: MemeService 21 | ) : SuspendedEventListener { 22 | 23 | /** 24 | * Indexation specific page. 25 | */ 26 | override suspend fun handleEvent(event: PageIndexRequest) { 27 | val channel = event.channel 28 | val pageNumber = event.pageNumber 29 | 30 | // Fetch and parse Page 31 | val path = channel.pagePath(pageNumber) 32 | val responseBody = webClient.get().uri(path).retrieve().awaitBody() 33 | val page = channel.parsePage(pageNumber, responseBody) 34 | 35 | // -- 36 | val newMemes = memesIndexation(channel, page.memesInfo) 37 | 38 | // -- 39 | eventChannel.send(PageIndexSuccessful(channel, page, newMemes)) 40 | } 41 | 42 | /** 43 | * Memes indexation event. 44 | * 45 | * The method provides deduplication by filtering memes URLs with the existing in the DB list of URLs. 46 | */ 47 | private suspend fun memesIndexation(channel: Channel, memesInfo: List): Int { 48 | val pageUrls = memesInfo.mapNotNull { it.pageUrl } 49 | val existingUrls = memeService.findByPageUrls(pageUrls).map { it.pageUrl }.toSet() 50 | val newMemes = memesInfo.filter { !existingUrls.contains(it.pageUrl) }.toSet() 51 | 52 | // Request memes indexation 53 | for (memeInfo in newMemes) { 54 | eventChannel.send(MemeIndexRequest(channel, memeInfo)) 55 | } 56 | 57 | return newMemes.size 58 | } 59 | 60 | 61 | companion object { 62 | private val log = LoggerFactory.getLogger(PageIndexer::class.java) 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/grab/ResourceDownloader.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.grab 2 | 3 | import com.github.kilianB.hashAlgorithms.AverageHash 4 | import kotlinx.coroutines.flow.collect 5 | import kotlinx.coroutines.future.await 6 | import me.ruslanys.ifunny.channel.MemeInfo 7 | import me.ruslanys.ifunny.domain.S3File 8 | import me.ruslanys.ifunny.grab.event.ResourceDownloadRequest 9 | import me.ruslanys.ifunny.property.AwsS3Properties 10 | import me.ruslanys.ifunny.service.MemeService 11 | import org.apache.tika.Tika 12 | import org.slf4j.LoggerFactory 13 | import org.springframework.stereotype.Component 14 | import org.springframework.util.DigestUtils 15 | import org.springframework.web.reactive.function.client.WebClient 16 | import org.springframework.web.reactive.function.client.awaitExchange 17 | import org.springframework.web.reactive.function.client.bodyToFlow 18 | import software.amazon.awssdk.core.async.AsyncRequestBody 19 | import software.amazon.awssdk.services.s3.S3AsyncClientBuilder 20 | import software.amazon.awssdk.services.s3.model.ObjectCannedACL 21 | import software.amazon.awssdk.services.s3.model.PutObjectRequest 22 | import java.io.ByteArrayInputStream 23 | import java.io.ByteArrayOutputStream 24 | import java.util.* 25 | import javax.imageio.ImageIO 26 | 27 | @Component 28 | class ResourceDownloader( 29 | private val webClient: WebClient, 30 | private val memeService: MemeService, 31 | private val s3ClientBuilder: S3AsyncClientBuilder, 32 | private val s3Properties: AwsS3Properties 33 | ) : SuspendedEventListener { 34 | 35 | private val hasher = AverageHash(64) 36 | private val tika = Tika() 37 | 38 | /** 39 | * Downloading meme resource file and upload it to S3. 40 | */ 41 | override suspend fun handleEvent(event: ResourceDownloadRequest) { 42 | val channel = event.channel 43 | val memeInfo = event.info 44 | 45 | // Download Resource 46 | val (contentTypeHeader, responseBody) = downloadResource(memeInfo) 47 | 48 | // File 49 | val fileName = generateFilename(memeInfo) 50 | val contentType = contentTypeHeader ?: tika.detect(responseBody, fileName) 51 | val checksum = generateChecksum(responseBody) 52 | val fingerprint = generateFingerprint(contentType, responseBody) 53 | 54 | // Deduplication 55 | if (fingerprint != null && memeService.isExists(channel.language, fingerprint)) { 56 | log.info("Duplicate discovered. Fingerprint: [{}], URL: {}", fingerprint, memeInfo.pageUrl) 57 | return 58 | } 59 | 60 | // Upload to S3 61 | val file = uploadFile(fileName, contentType, checksum, responseBody) 62 | 63 | // Persist 64 | memeService.add(channel, memeInfo, file, fingerprint) 65 | } 66 | 67 | private suspend fun downloadResource(memeInfo: MemeInfo): Pair { 68 | val response = webClient.get().uri(memeInfo.originUrl!!).awaitExchange() 69 | val contentTypeHeader = response.headers().asHttpHeaders().getFirst("Content-Type") 70 | 71 | ByteArrayOutputStream().use { container -> 72 | response.bodyToFlow().collect { bytes -> 73 | container.writeBytes(bytes) 74 | } 75 | 76 | if (container.size() == 0) { 77 | throw IllegalStateException("Response body is empty.") 78 | } 79 | 80 | return contentTypeHeader to container.toByteArray() 81 | } 82 | } 83 | 84 | private fun generateFilename(memeInfo: MemeInfo): String { 85 | val fileExtension = memeInfo.originUrl!! 86 | .substringAfterLast(".") 87 | .substringBefore("?") 88 | return "${UUID.randomUUID()}.$fileExtension" 89 | } 90 | 91 | private fun generateChecksum(byteArray: ByteArray): String = DigestUtils.md5DigestAsHex(byteArray).toUpperCase() 92 | 93 | private fun generateFingerprint(contentType: String, byteArray: ByteArray): String? { 94 | return if (contentType.startsWith("image")) { 95 | val image = ImageIO.read(ByteArrayInputStream(byteArray)) 96 | hasher.hash(image).hashValue.toString(16).toUpperCase() 97 | } else { 98 | null 99 | } 100 | } 101 | 102 | private suspend fun uploadFile(fileName: String, contentType: String, checksum: String, byteArray: ByteArray): S3File { 103 | val request = PutObjectRequest.builder() 104 | .acl(ObjectCannedACL.PUBLIC_READ) 105 | .bucket(s3Properties.bucket!!) 106 | .key(fileName) 107 | .contentType(contentType) 108 | .contentLength(byteArray.size.toLong()) 109 | .build() 110 | 111 | val s3Client = s3ClientBuilder.build() 112 | s3Client.use { 113 | it.putObject(request, AsyncRequestBody.fromBytes(byteArray)).await() 114 | } 115 | 116 | return S3File(s3Properties.region!!, s3Properties.bucket!!, fileName, contentType, checksum, byteArray.size.toLong()) 117 | } 118 | 119 | companion object { 120 | private val log = LoggerFactory.getLogger(ResourceDownloader::class.java) 121 | } 122 | 123 | } 124 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/grab/SuspendedEventListener.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.grab 2 | 3 | interface SuspendedEventListener { 4 | 5 | suspend fun handleEvent(event: T) 6 | 7 | } 8 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/grab/event/GrabEvent.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.grab.event 2 | 3 | import me.ruslanys.ifunny.channel.Channel 4 | 5 | abstract class GrabEvent(val channel: Channel) 6 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/grab/event/MemeIndexRequest.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.grab.event 2 | 3 | import me.ruslanys.ifunny.channel.Channel 4 | import me.ruslanys.ifunny.channel.MemeInfo 5 | 6 | class MemeIndexRequest(channel: Channel, val info: MemeInfo) : GrabEvent(channel) { 7 | override fun toString(): String { 8 | return "MemeIndexRequest(channel=$channel, info=$info)" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/grab/event/PageIndexEvent.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.grab.event 2 | 3 | import me.ruslanys.ifunny.channel.Channel 4 | 5 | abstract class PageIndexEvent(channel: Channel) : GrabEvent(channel) 6 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/grab/event/PageIndexRequest.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.grab.event 2 | 3 | import me.ruslanys.ifunny.channel.Channel 4 | 5 | class PageIndexRequest(channel: Channel, val pageNumber: Int) : PageIndexEvent(channel) { 6 | override fun toString(): String { 7 | return "PageIndexRequest(channel=$channel, pageNumber=$pageNumber)" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/grab/event/PageIndexSuccessful.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.grab.event 2 | 3 | import me.ruslanys.ifunny.channel.Channel 4 | import me.ruslanys.ifunny.channel.Page 5 | 6 | class PageIndexSuccessful(channel: Channel, val page: Page, val new: Int) : PageIndexEvent(channel) { 7 | override fun toString(): String { 8 | return "PageIndexSuccessful(channel=$channel, page=$page, new=$new)" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/grab/event/ResourceDownloadRequest.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.grab.event 2 | 3 | import me.ruslanys.ifunny.channel.Channel 4 | import me.ruslanys.ifunny.channel.MemeInfo 5 | 6 | class ResourceDownloadRequest(channel: Channel, val info: MemeInfo) : GrabEvent(channel) { 7 | override fun toString(): String { 8 | return "ResourceDownloadRequest(channel=$channel, info=$info)" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/property/AwsS3Properties.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.property 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties 4 | import org.springframework.boot.context.properties.ConstructorBinding 5 | import org.springframework.validation.annotation.Validated 6 | import javax.validation.constraints.NotEmpty 7 | 8 | @ConstructorBinding 9 | @ConfigurationProperties(prefix = "aws.s3") 10 | @Validated 11 | data class AwsS3Properties( 12 | @field:NotEmpty 13 | val accessKey: String? = null, 14 | 15 | @field:NotEmpty 16 | val secretKey: String? = null, 17 | 18 | @field:NotEmpty 19 | val region: String? = null, 20 | 21 | @field:NotEmpty 22 | val bucket: String? = null 23 | ) 24 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/property/GrabProperties.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.property 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties 4 | import org.springframework.boot.context.properties.ConstructorBinding 5 | import java.time.Duration 6 | 7 | @ConstructorBinding 8 | @ConfigurationProperties(prefix = "grab") 9 | data class GrabProperties( 10 | val userAgent: String = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", 11 | val channelSize: Int = 512, 12 | val retention: Retention = Retention() 13 | ) { 14 | 15 | data class Retention( 16 | val fullIndex: Duration = Duration.ofDays(7) 17 | ) 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/repository/MemeRepository.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.repository 2 | 3 | import me.ruslanys.ifunny.domain.Meme 4 | import org.springframework.data.domain.Pageable 5 | import org.springframework.data.mongodb.repository.ReactiveMongoRepository 6 | import org.springframework.stereotype.Repository 7 | import reactor.core.publisher.Flux 8 | import reactor.core.publisher.Mono 9 | 10 | @Repository 11 | interface MemeRepository : ReactiveMongoRepository { 12 | 13 | fun findByLanguage(language: String, pageable: Pageable): Flux 14 | 15 | fun findByPageUrlIn(pageUrls: List): Flux 16 | 17 | fun existsByLanguageAndFingerprint(language: String, fingerprint: String): Mono 18 | 19 | fun countByLanguage(language: String): Mono 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/repository/MongoIndexCreator.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.repository 2 | 3 | import org.springframework.boot.context.event.ApplicationReadyEvent 4 | import org.springframework.context.event.EventListener 5 | import org.springframework.data.mongodb.core.MongoTemplate 6 | import org.springframework.data.mongodb.core.convert.MongoConverter 7 | import org.springframework.data.mongodb.core.index.MongoPersistentEntityIndexResolver 8 | import org.springframework.data.mongodb.core.mapping.Document 9 | import org.springframework.stereotype.Component 10 | 11 | @Component 12 | class MongoIndexCreator(private val mongoConverter: MongoConverter, private val mongoTemplate: MongoTemplate) { 13 | 14 | @EventListener(ApplicationReadyEvent::class) 15 | fun initIndicesAfterStartup() { 16 | val mappingContext = mongoConverter.mappingContext 17 | for (entity in mappingContext.persistentEntities) { 18 | val clazz = entity.type 19 | if (!clazz.isAnnotationPresent(Document::class.java)) { 20 | continue 21 | } 22 | 23 | val indexOps = mongoTemplate.indexOps(clazz) 24 | val resolver = MongoPersistentEntityIndexResolver(mappingContext) 25 | resolver.resolveIndexFor(clazz).forEach { indexOps.ensureIndex(it) } 26 | } 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/repository/PageRepository.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.repository 2 | 3 | import me.ruslanys.ifunny.channel.Channel 4 | import org.springframework.data.redis.core.ReactiveRedisTemplate 5 | import org.springframework.data.redis.core.ReactiveValueOperations 6 | import org.springframework.stereotype.Repository 7 | import reactor.core.publisher.Mono 8 | import java.time.Duration 9 | 10 | @Repository 11 | class PageRepository(private val redisTemplate: ReactiveRedisTemplate) { 12 | 13 | private val valueOperations: ReactiveValueOperations = redisTemplate.opsForValue() 14 | 15 | fun getCurrent(channel: Channel): Mono { 16 | val key = currentKey(channel) 17 | return valueOperations[key].defaultIfEmpty(1).map { it as Int } 18 | } 19 | 20 | fun incCurrent(channel: Channel): Mono { 21 | val key = currentKey(channel) 22 | return valueOperations.increment(key).map { it.toInt() } 23 | } 24 | 25 | fun getLast(channel: Channel): Mono { 26 | val key = lastKey(channel) 27 | return valueOperations[key].map { it as Int } 28 | } 29 | 30 | fun setLast(channel: Channel, pageNumber: Int, retention: Duration): Mono { 31 | val key = lastKey(channel) 32 | return valueOperations.setIfAbsent(key, pageNumber, retention) 33 | } 34 | 35 | fun clearCurrent(channel: Channel): Mono { 36 | val key = currentKey(channel) 37 | return redisTemplate.delete(key) 38 | } 39 | 40 | private fun currentKey(channel: Channel): String { 41 | return "$channel:$CURRENT_POSTFIX" 42 | } 43 | 44 | private fun lastKey(channel: Channel): String { 45 | return "$channel:$LAST_POSTFIX" 46 | } 47 | 48 | 49 | companion object { 50 | private const val CURRENT_POSTFIX = "current" 51 | private const val LAST_POSTFIX = "last" 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/service/DefaultMemeService.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.service 2 | 3 | import kotlinx.coroutines.async 4 | import kotlinx.coroutines.coroutineScope 5 | import kotlinx.coroutines.flow.toList 6 | import kotlinx.coroutines.reactive.asFlow 7 | import kotlinx.coroutines.reactive.awaitFirstOrElse 8 | import kotlinx.coroutines.reactive.awaitSingle 9 | import me.ruslanys.ifunny.channel.Channel 10 | import me.ruslanys.ifunny.channel.MemeInfo 11 | import me.ruslanys.ifunny.domain.Language 12 | import me.ruslanys.ifunny.domain.Meme 13 | import me.ruslanys.ifunny.domain.S3File 14 | import me.ruslanys.ifunny.exception.NotFoundException 15 | import me.ruslanys.ifunny.repository.MemeRepository 16 | import org.springframework.data.domain.Page 17 | import org.springframework.data.domain.PageImpl 18 | import org.springframework.data.domain.Pageable 19 | import org.springframework.stereotype.Service 20 | 21 | @Service 22 | class DefaultMemeService(private val memeRepository: MemeRepository) : MemeService { 23 | 24 | override suspend fun add(channel: Channel, info: MemeInfo, file: S3File, fingerprint: String?): Meme { 25 | info.pageUrl ?: throw IllegalArgumentException("A meme page URL can't be null!") 26 | info.title ?: throw IllegalArgumentException("A meme title can't be null!") 27 | info.originUrl ?: throw IllegalArgumentException("A meme origin URL can't be null!") 28 | 29 | val meme = Meme(channel.language.code, channel.getName(), info.pageUrl, info.title, info.originUrl, file, 30 | fingerprint, info.publishDateTime, info.author, info.likes, info.comments) 31 | return memeRepository.save(meme).awaitSingle() 32 | } 33 | 34 | override suspend fun isExists(language: Language, fingerprint: String): Boolean { 35 | return memeRepository.existsByLanguageAndFingerprint(language.code, fingerprint).awaitSingle() 36 | } 37 | 38 | override suspend fun findByPageUrls(urls: List): List { 39 | return memeRepository.findByPageUrlIn(urls).asFlow().toList() 40 | } 41 | 42 | override suspend fun getPage(language: Language, pageRequest: Pageable): Page = coroutineScope { 43 | val memes = async { 44 | memeRepository.findByLanguage(language.code, pageRequest).asFlow().toList() 45 | } 46 | val count = async { 47 | memeRepository.countByLanguage(language.code).awaitSingle() 48 | } 49 | 50 | PageImpl(memes.await(), pageRequest, count.await()) 51 | } 52 | 53 | override suspend fun getById(id: String): Meme = memeRepository.findById(id).awaitFirstOrElse { 54 | throw NotFoundException("Meme with ID $id not found.") 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/main/kotlin/me/ruslanys/ifunny/service/Services.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.service 2 | 3 | import me.ruslanys.ifunny.channel.Channel 4 | import me.ruslanys.ifunny.channel.MemeInfo 5 | import me.ruslanys.ifunny.domain.Language 6 | import me.ruslanys.ifunny.domain.Meme 7 | import me.ruslanys.ifunny.domain.S3File 8 | import org.springframework.data.domain.Page 9 | import org.springframework.data.domain.Pageable 10 | 11 | interface MemeService { 12 | 13 | suspend fun add(channel: Channel, info: MemeInfo, file: S3File, fingerprint: String?): Meme 14 | 15 | suspend fun isExists(language: Language, fingerprint: String): Boolean 16 | 17 | suspend fun findByPageUrls(urls: List): List 18 | 19 | suspend fun getPage(language: Language, pageRequest: Pageable): Page 20 | 21 | suspend fun getById(id: String): Meme 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | # SERVER 2 | server.error.include-stacktrace=never 3 | 4 | # FREEMARKER 5 | spring.freemarker.cache=true 6 | spring.freemarker.settings.auto_import=spring.ftl as spring, _layout.ftlh as layout 7 | 8 | # MANAGEMENT 9 | management.endpoints.web.exposure.include=health, info, prometheus, metrics 10 | 11 | # MONGODB 12 | spring.data.mongodb.host=${MONGODB_HOST:localhost} 13 | spring.data.mongodb.port=${MONGODB_PORT:27017} 14 | spring.data.mongodb.database=${MONGODB_DATABASE:ifunny} 15 | spring.data.mongodb.username=${MONGODB_USERNAME:ifunny} 16 | spring.data.mongodb.password=${MONGODB_PASSWORD:ifunny} 17 | spring.data.mongodb.authentication-database=${MONGODB_AUTH_DB:admin} 18 | spring.data.mongodb.auto-index-creation=false 19 | 20 | # REDIS 21 | spring.redis.host=${REDIS_HOST:localhost} 22 | spring.redis.port=${REDIS_PORT:6379} 23 | spring.redis.database=${REDIS_DATABASE:0} 24 | 25 | # AWS 26 | aws.s3.access-key=${AWS_S3_ACCESS_KEY} 27 | aws.s3.secret-key=${AWS_S3_SECRET_KEY} 28 | aws.s3.region=${AWS_S3_REGION} 29 | aws.s3.bucket=${AWS_S3_BUCKET} 30 | 31 | # LOGGING 32 | logging.level.FastPixel=off 33 | -------------------------------------------------------------------------------- /src/main/resources/banner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/main/resources/banner.png -------------------------------------------------------------------------------- /src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | ${AnsiBackground.YELLOW}${AnsiColor.BRIGHT_WHITE}${AnsiStyle.BOLD} :: iFunny${application.formatted-version} :: ${AnsiStyle.NORMAL} 2 | -------------------------------------------------------------------------------- /src/main/resources/static/contract.yaml: -------------------------------------------------------------------------------- 1 | openapi: 3.0.2 2 | info: 3 | title: iFunny API 4 | version: 1.1.0 5 | contact: 6 | name: Ruslan Molchanov 7 | email: me@ruslanys.me 8 | servers: 9 | - url: /api 10 | description: Local 11 | - url: https://ifunny.ruslanys.me/api 12 | description: Production 13 | paths: 14 | '/feed/{ID}': 15 | get: 16 | tags: 17 | - "Feed" 18 | summary: "Get Meme by its ID" 19 | operationId: "getById" 20 | parameters: 21 | - name: ID 22 | in: path 23 | required: true 24 | schema: 25 | type: string 26 | responses: 27 | 200: 28 | description: Successful operation 29 | content: 30 | 'application/json': 31 | schema: 32 | $ref: '#/components/schemas/Meme' 33 | 404: 34 | description: "Meme with the specified ID not found" 35 | content: 36 | 'text/plain' 37 | '/feed': 38 | get: 39 | tags: 40 | - "Feed" 41 | operationId: "getPage" 42 | parameters: 43 | - name: language 44 | in: query 45 | description: Language code 46 | required: true 47 | schema: 48 | type: string 49 | enum: 50 | - de 51 | - fr 52 | - es 53 | - it 54 | - pt 55 | - ru 56 | - name: offset 57 | in: query 58 | description: Pagination offset 59 | schema: 60 | type: integer 61 | format: int64 62 | minimum: 0 63 | default: 0 64 | - name: limit 65 | in: query 66 | description: Pagination limit 67 | schema: 68 | type: integer 69 | format: int32 70 | minimum: 1 71 | maximum: 200 72 | default: 10 73 | - name: sortBy 74 | in: query 75 | description: Sorting argument 76 | schema: 77 | type: string 78 | enum: 79 | - publishDateTime 80 | - likes 81 | default: publishDateTime 82 | - name: sortDirection 83 | description: Sorting direction 84 | in: query 85 | schema: 86 | type: string 87 | enum: 88 | - ASC 89 | - DESC 90 | default: DESC 91 | responses: 92 | 200: 93 | description: Successful operation 94 | content: 95 | 'application/json': 96 | schema: 97 | type: object 98 | properties: 99 | totalCount: 100 | type: integer 101 | format: int64 102 | list: 103 | type: array 104 | items: 105 | $ref: '#/components/schemas/Meme' 106 | 400: 107 | description: Arguments validation constraint 108 | content: 109 | 'text/plain' 110 | components: 111 | schemas: 112 | Meme: 113 | type: object 114 | properties: 115 | id: 116 | type: string 117 | language: 118 | type: string 119 | channelName: 120 | type: string 121 | title: 122 | type: string 123 | url: 124 | type: string 125 | contentType: 126 | type: string 127 | publishDateTime: 128 | type: string 129 | format: date-time 130 | author: 131 | type: string 132 | likes: 133 | type: integer 134 | format: int32 135 | comments: 136 | type: integer 137 | format: int32 138 | -------------------------------------------------------------------------------- /src/main/resources/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/main/resources/static/favicon.ico -------------------------------------------------------------------------------- /src/main/resources/static/img/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/main/resources/static/img/logo.png -------------------------------------------------------------------------------- /src/main/resources/static/swagger-ui/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/main/resources/static/swagger-ui/favicon-16x16.png -------------------------------------------------------------------------------- /src/main/resources/static/swagger-ui/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/main/resources/static/swagger-ui/favicon-32x32.png -------------------------------------------------------------------------------- /src/main/resources/static/swagger-ui/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Swagger UI 7 | 8 | 9 | 10 | 31 | 32 | 33 | 34 |
35 | 36 | 37 | 38 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /src/main/resources/static/swagger-ui/oauth2-redirect.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Swagger UI: OAuth2 Redirect 4 | 5 | 6 | 7 | 69 | -------------------------------------------------------------------------------- /src/main/resources/templates/_layout.ftlh: -------------------------------------------------------------------------------- 1 | <#assign SITE_NAME = "FunCode Challenge" /> 2 | 3 | <#macro head title='', image=''> 4 | <#assign full_title = title?has_content?then("${title} - ${SITE_NAME}", SITE_NAME) /> 5 | 6 | 7 | 8 | 9 | 10 | 11 | <@og full_title image/> 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | ${full_title} 22 | 23 | 24 | <#nested /> 25 | 26 | 27 | 28 | <#macro og title='', image=''> 29 | 30 | 31 | <#if title?has_content> 32 | 33 | 34 | 35 | <#if image?has_content> 36 | 37 | 38 | 39 | 40 | 41 | <#macro body> 42 | 43 | 44 | 45 |
46 |
47 |
48 |
49 |
50 |
FunCode
 51 |
52 |
53 |

FunCode

54 |

Java / Kotlin Challenge

55 |

Made with and focus on

56 |

Ruslan Molchanov

57 |
58 |
59 |
60 |
61 |
62 | 63 | 64 | <#nested /> 65 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /src/main/resources/templates/meme.ftlh: -------------------------------------------------------------------------------- 1 | 2 | 3 | <#if meme.contentType?starts_with("image")> 4 | <#assign OG_IMAGE_URL=meme.url /> 5 | <#else> 6 | <#assign OG_IMAGE_URL='' /> 7 | 8 | <@layout.head meme.title OG_IMAGE_URL> 9 | 14 | 15 | 16 | <@layout.body> 17 |
18 |
19 |
20 |
21 | 22 | 23 |
24 | 25 |
26 |
27 |
28 |

${meme.title}

29 |
30 | 31 |
32 | <#if meme.contentType?starts_with("image")> 33 |
34 | ${meme.title} 35 |
36 | <#else> 37 |
38 | 41 |
42 | 43 |
44 | 45 | <#if meme.author?? || meme.publishDateTime??> 46 |
47 |
48 | <#if meme.author??>

@${meme.author}

49 | <#if meme.publishDateTime??>${meme.publishDateTime} 50 |
51 |
52 | 53 | 54 | <#if (meme.likes?? && meme.likes != 0) || (meme.comments?? && meme.comments != 0)> 55 |
56 | <#if meme.likes?? && meme.likes != 0> 57 |

58 | 59 | ${meme.likes} Likes 60 |

61 | 62 | <#if meme.comments?? && meme.comments != 0> 63 |

64 | 65 | ${meme.comments} Comments 66 |

67 | 68 |
69 | 70 |
71 |
72 | 73 |
74 | 75 | Back to Feed 76 | 77 |
78 |
79 | 80 |
81 | 82 |
83 |
84 |
85 | 86 | 87 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/ApplicationTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny 2 | 3 | import org.junit.jupiter.api.Disabled 4 | import org.junit.jupiter.api.Test 5 | import org.springframework.boot.test.context.SpringBootTest 6 | 7 | /** 8 | * https://github.com/spring-projects/spring-boot/issues/19994 9 | * https://youtrack.jetbrains.com/issue/KT-34024 10 | */ 11 | @Disabled 12 | @SpringBootTest 13 | class ApplicationTests { 14 | 15 | @Test 16 | fun contextLoads() { 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/base/ControllerTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.base 2 | 3 | import org.springframework.beans.factory.annotation.Autowired 4 | import org.springframework.test.web.reactive.server.WebTestClient 5 | 6 | 7 | abstract class ControllerTests { 8 | 9 | @Autowired 10 | protected lateinit var webClient: WebTestClient 11 | 12 | 13 | } 14 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/base/MongoRepositoryTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.base 2 | 3 | import me.ruslanys.ifunny.repository.MongoIndexCreator 4 | import org.junit.jupiter.api.BeforeEach 5 | import org.springframework.beans.factory.annotation.Autowired 6 | import org.springframework.boot.autoconfigure.mongo.embedded.EmbeddedMongoAutoConfiguration 7 | import org.springframework.boot.test.autoconfigure.data.mongo.DataMongoTest 8 | import org.springframework.data.mongodb.core.MongoTemplate 9 | import org.springframework.data.mongodb.core.ReactiveMongoTemplate 10 | import org.springframework.data.mongodb.core.convert.MongoConverter 11 | 12 | @DataMongoTest(excludeAutoConfiguration = [EmbeddedMongoAutoConfiguration::class]) 13 | abstract class MongoRepositoryTests { 14 | 15 | // @formatter:off 16 | @Autowired protected lateinit var mongoTemplate: MongoTemplate 17 | @Autowired protected lateinit var reactiveMongoTemplate: ReactiveMongoTemplate 18 | @Autowired protected lateinit var mongoConverter: MongoConverter 19 | // @formatter:on 20 | 21 | 22 | @BeforeEach 23 | open fun setUp() { 24 | val indexCreator = MongoIndexCreator(mongoConverter, mongoTemplate) 25 | indexCreator.initIndicesAfterStartup() 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/base/RedisRepositoryTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.base 2 | 3 | import me.ruslanys.ifunny.config.RedisConfig 4 | import org.springframework.beans.factory.annotation.Autowired 5 | import org.springframework.boot.autoconfigure.jackson.JacksonAutoConfiguration 6 | import org.springframework.boot.test.autoconfigure.data.redis.DataRedisTest 7 | import org.springframework.context.annotation.Import 8 | import org.springframework.data.redis.core.ReactiveRedisTemplate 9 | 10 | @DataRedisTest 11 | @Import(RedisConfig::class, JacksonAutoConfiguration::class) 12 | abstract class RedisRepositoryTests { 13 | 14 | @Autowired 15 | protected lateinit var redisTemplate: ReactiveRedisTemplate 16 | 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/base/ServiceTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.base 2 | 3 | import org.junit.jupiter.api.extension.ExtendWith 4 | import org.mockito.junit.jupiter.MockitoExtension 5 | 6 | /** 7 | * This class doing the same as MockitoAnnotations.initMocks(this); 8 | */ 9 | @ExtendWith(MockitoExtension::class) 10 | abstract class ServiceTests 11 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/channel/BastardidentroChannelTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | import kotlinx.coroutines.runBlocking 4 | import me.ruslanys.ifunny.util.readResource 5 | import org.assertj.core.api.Assertions.assertThat 6 | import org.junit.jupiter.api.Test 7 | import java.time.LocalDateTime 8 | 9 | class BastardidentroChannelTests { 10 | 11 | private val channel = BastardidentroChannel() 12 | 13 | 14 | @Test 15 | fun firstPagePathTest() = runBlocking { 16 | val pagePath = channel.pagePath(1) 17 | assertThat(pagePath).isEqualTo("https://www.bastardidentro.it/immagini-e-vignette-divertenti") 18 | } 19 | 20 | @Test 21 | fun hundredPagePathTest() = runBlocking { 22 | val pagePath = channel.pagePath(100) 23 | assertThat(pagePath).isEqualTo("https://www.bastardidentro.it/immagini-e-vignette-divertenti?page=99") 24 | } 25 | 26 | @Test 27 | fun parseProperPageShouldReturnList() = runBlocking { 28 | val html = readResource("bastardidentro/page.html") 29 | 30 | // -- 31 | val page = channel.parsePage(6, html) 32 | val list = page.memesInfo 33 | 34 | // -- 35 | assertThat(page.hasNext).isTrue() 36 | assertThat(list).hasSize(10) 37 | assertThat(list).allMatch { it.pageUrl != null } 38 | assertThat(list).allMatch { it.title != null } 39 | } 40 | 41 | @Test 42 | fun parsePageWithMemeWithoutHeader() = runBlocking { 43 | val html = readResource("bastardidentro/page_without_header.html") 44 | 45 | // -- 46 | val page = channel.parsePage(761, html) 47 | val list = page.memesInfo 48 | 49 | // -- 50 | assertThat(page.hasNext).isTrue() 51 | assertThat(list).hasSize(9) 52 | assertThat(list).allMatch { it.pageUrl != null } 53 | assertThat(list).allMatch { it.title != null } 54 | } 55 | 56 | @Test 57 | fun parseLastPageShouldReturnHasNextFalse() = runBlocking { 58 | val html = readResource("bastardidentro/page_last.html") 59 | 60 | // -- 61 | val page = channel.parsePage(1421, html) 62 | 63 | // -- 64 | assertThat(page.hasNext).isFalse() 65 | } 66 | 67 | @Test 68 | fun parseInvalidPageShouldReturnEmptyList() = runBlocking { 69 | val page = channel.parsePage(1, "") 70 | assertThat(page.hasNext).isFalse() 71 | assertThat(page.memesInfo).isEmpty() 72 | } 73 | 74 | @Test 75 | fun parsePictureMeme() = runBlocking { 76 | val baseInfo = MemeInfo( 77 | pageUrl = "https://www.bastardidentro.it/immagini-e-vignette-divertenti/inconveniente-nel-lavare-i-piatti-511606", 78 | title = "Inconveniente nel lavare i piatti!" 79 | ) 80 | val html = readResource("bastardidentro/meme_picture.html") 81 | 82 | // -- 83 | val info = channel.parseMeme(baseInfo, html) 84 | 85 | // -- 86 | assertThat(info.pageUrl).isEqualTo(baseInfo.pageUrl) 87 | assertThat(info.title).isEqualTo(baseInfo.title) 88 | 89 | assertThat(info.originUrl).isEqualTo("https://www.bastardidentro.it/sites/default/files/styles/nullo/public/images/9/3/4/lavare-piatti.jpg?itok=hKaeHnDB") 90 | assertThat(info.publishDateTime).isEqualTo(LocalDateTime.of(2020, 2, 12, 19, 30)) 91 | } 92 | 93 | } 94 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/channel/BestiChannelTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | import kotlinx.coroutines.runBlocking 4 | import me.ruslanys.ifunny.util.readResource 5 | import org.assertj.core.api.Assertions.assertThat 6 | import org.junit.jupiter.api.Test 7 | import java.time.LocalDateTime 8 | 9 | class BestiChannelTests { 10 | 11 | private val channel = BestiChannel() 12 | 13 | 14 | @Test 15 | fun firstPagePathTest() = runBlocking { 16 | val pagePath = channel.pagePath(1) 17 | assertThat(pagePath).isEqualTo("https://besti.it/1") 18 | } 19 | 20 | @Test 21 | fun hundredPagePathTest() = runBlocking { 22 | val pagePath = channel.pagePath(100) 23 | assertThat(pagePath).isEqualTo("https://besti.it/100") 24 | } 25 | 26 | @Test 27 | fun parseProperPageShouldReturnList() = runBlocking { 28 | val html = readResource("besti/page.html") 29 | 30 | // -- 31 | val page = channel.parsePage(2, html) 32 | val list = page.memesInfo 33 | 34 | // -- 35 | assertThat(page.hasNext).isTrue() 36 | assertThat(list).hasSize(9) 37 | assertThat(list).allMatch { it.pageUrl != null } 38 | assertThat(list).allMatch { it.title != null } 39 | assertThat(list).allMatch { it.likes != null } 40 | assertThat(list).allMatch { it.comments != null } 41 | assertThat(list).allMatch { it.author != null } 42 | assertThat(list).allMatch { it.publishDateTime != null } 43 | } 44 | 45 | @Test 46 | fun parseLastPageShouldReturnHasNextFalse() = runBlocking { 47 | val html = readResource("besti/page_last.html") 48 | 49 | // -- 50 | val page = channel.parsePage(6590, html) 51 | 52 | // -- 53 | assertThat(page.hasNext).isFalse() 54 | } 55 | 56 | @Test 57 | fun parseInvalidPageShouldReturnEmptyList() = runBlocking { 58 | val page = channel.parsePage(1, "") 59 | assertThat(page.hasNext).isFalse() 60 | assertThat(page.memesInfo).isEmpty() 61 | } 62 | 63 | @Test 64 | fun parsePictureMeme() = runBlocking { 65 | val baseInfo = MemeInfo( 66 | "https://besti.it/74854/Io-alla-prova-costume-di-carnevale", 67 | null, 68 | "Io alla prova costume di carnevale..", 69 | LocalDateTime.of(2020, 2, 21, 5, 17, 44), 70 | 15, 71 | 0, 72 | "GiorgiaPi" 73 | ) 74 | val html = readResource("besti/meme_picture.html") 75 | 76 | // -- 77 | val info = channel.parseMeme(baseInfo, html) 78 | 79 | // -- 80 | assertThat(info.pageUrl).isEqualTo(baseInfo.pageUrl) 81 | assertThat(info.title).isEqualTo(baseInfo.title) 82 | assertThat(info.likes).isEqualTo(baseInfo.likes) 83 | assertThat(info.comments).isEqualTo(baseInfo.comments) 84 | assertThat(info.author).isEqualTo(baseInfo.author) 85 | assertThat(info.publishDateTime).isEqualTo(baseInfo.publishDateTime) 86 | 87 | assertThat(info.originUrl).isEqualTo("https://besti.it/upload/2519bab24510f843ab721f48c07a42992864.jpg") 88 | } 89 | 90 | @Test 91 | fun parseVideoMeme() = runBlocking { 92 | val baseInfo = MemeInfo( 93 | "https://besti.it/74846/succede", 94 | null, 95 | "succede", 96 | LocalDateTime.of(2020, 2, 21, 1, 17, 43), 97 | 6, 98 | 1, 99 | "AronF" 100 | ) 101 | val html = readResource("besti/meme_video.html") 102 | 103 | // -- 104 | val info = channel.parseMeme(baseInfo, html) 105 | 106 | // -- 107 | assertThat(info.pageUrl).isEqualTo(baseInfo.pageUrl) 108 | assertThat(info.title).isEqualTo(baseInfo.title) 109 | assertThat(info.likes).isEqualTo(baseInfo.likes) 110 | assertThat(info.comments).isEqualTo(baseInfo.comments) 111 | assertThat(info.author).isEqualTo(baseInfo.author) 112 | assertThat(info.publishDateTime).isEqualTo(baseInfo.publishDateTime) 113 | 114 | assertThat(info.originUrl).isEqualTo("https://besti.it/upload2/v/22b4793a8b08fb9ad822d5c4262b690d9104.mp4") 115 | } 116 | 117 | } 118 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/channel/FuoriditestaImagesChannelTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | import kotlinx.coroutines.runBlocking 4 | import me.ruslanys.ifunny.util.readResource 5 | import org.assertj.core.api.Assertions.assertThat 6 | import org.junit.jupiter.api.Test 7 | import java.time.LocalDateTime 8 | 9 | class FuoriditestaImagesChannelTests { 10 | 11 | private val channel = FuoriditestaImagesChannel() 12 | 13 | 14 | @Test 15 | fun firstPagePathTest() = runBlocking { 16 | val pagePath = channel.pagePath(1) 17 | assertThat(pagePath).isEqualTo("http://www.fuoriditesta.it/immagini-divertenti/index-1.php") 18 | } 19 | 20 | @Test 21 | fun hundredPagePathTest() = runBlocking { 22 | val pagePath = channel.pagePath(100) 23 | assertThat(pagePath).isEqualTo("http://www.fuoriditesta.it/immagini-divertenti/index-100.php") 24 | } 25 | 26 | @Test 27 | fun parseProperPageShouldReturnList() = runBlocking { 28 | val html = readResource("fuoriditesta-images/page.html") 29 | 30 | // -- 31 | val page = channel.parsePage(1, html) 32 | val list = page.memesInfo 33 | 34 | // -- 35 | assertThat(page.hasNext).isTrue() 36 | assertThat(list).hasSize(18) 37 | assertThat(list).allMatch { it.pageUrl != null } 38 | assertThat(list).allMatch { it.title != null } 39 | assertThat(list).allMatch { it.publishDateTime != null } 40 | } 41 | 42 | @Test 43 | fun parseLastPageShouldReturnHasNextFalse() = runBlocking { 44 | val html = readResource("fuoriditesta-images/page_last.html") 45 | 46 | // -- 47 | val page = channel.parsePage(216, html) 48 | 49 | // -- 50 | assertThat(page.hasNext).isFalse() 51 | } 52 | 53 | @Test 54 | fun parseInvalidPageShouldReturnEmptyList() = runBlocking { 55 | val page = channel.parsePage(1, "") 56 | assertThat(page.hasNext).isFalse() 57 | assertThat(page.memesInfo).isEmpty() 58 | } 59 | 60 | @Test 61 | fun parsePictureMeme() = runBlocking { 62 | val baseInfo = MemeInfo( 63 | pageUrl = "http://www.fuoriditesta.it/immagini-divertenti/calzini-di-trump.html", 64 | title = "Calzini di Trump", 65 | publishDateTime = LocalDateTime.of(2019, 8, 26, 0, 0) 66 | ) 67 | val html = readResource("fuoriditesta-images/meme_picture.html") 68 | 69 | // -- 70 | val info = channel.parseMeme(baseInfo, html) 71 | 72 | // -- 73 | assertThat(info.pageUrl).isEqualTo(baseInfo.pageUrl) 74 | assertThat(info.title).isEqualTo(baseInfo.title) 75 | assertThat(info.publishDateTime).isEqualTo(baseInfo.publishDateTime) 76 | 77 | assertThat(info.originUrl).isEqualTo("http://www.fuoriditesta.it/umorismo/immagini/divertenti/620x620xcalzini-di-trump.jpg.pagespeed.ic.44S2tn8xIE.jpg") 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/channel/FuoriditestaVideoChannelTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | import com.nhaarman.mockitokotlin2.mock 4 | import kotlinx.coroutines.runBlocking 5 | import me.ruslanys.ifunny.util.mockGet 6 | import me.ruslanys.ifunny.util.readResource 7 | import org.assertj.core.api.Assertions.assertThat 8 | import org.junit.jupiter.api.Test 9 | import org.springframework.web.reactive.function.client.WebClient 10 | import java.time.LocalDateTime 11 | 12 | class FuoriditestaVideoChannelTests { 13 | 14 | private val webClient: WebClient = mock() 15 | private val channel = FuoriditestaVideoChannel(webClient) 16 | 17 | 18 | @Test 19 | fun firstPagePathTest() = runBlocking { 20 | val pagePath = channel.pagePath(1) 21 | assertThat(pagePath).isEqualTo("http://www.fuoriditesta.it/video-divertenti/index-1.php") 22 | } 23 | 24 | @Test 25 | fun hundredPagePathTest() = runBlocking { 26 | val pagePath = channel.pagePath(100) 27 | assertThat(pagePath).isEqualTo("http://www.fuoriditesta.it/video-divertenti/index-100.php") 28 | } 29 | 30 | @Test 31 | fun parseProperPageShouldReturnList() = runBlocking { 32 | val html = readResource("fuoriditesta-video/page.html") 33 | 34 | // -- 35 | val page = channel.parsePage(1, html) 36 | val list = page.memesInfo 37 | 38 | // -- 39 | assertThat(page.hasNext).isTrue() 40 | assertThat(list).hasSize(15) 41 | assertThat(list).allMatch { it.pageUrl != null } 42 | assertThat(list).allMatch { it.title != null } 43 | } 44 | 45 | @Test 46 | fun parseLastPageShouldReturnHasNextFalse() = runBlocking { 47 | val html = readResource("fuoriditesta-video/page_last.html") 48 | 49 | // -- 50 | val page = channel.parsePage(221, html) 51 | 52 | // -- 53 | assertThat(page.hasNext).isFalse() 54 | } 55 | 56 | @Test 57 | fun parseInvalidPageShouldReturnEmptyList() = runBlocking { 58 | val page = channel.parsePage(1, "") 59 | assertThat(page.hasNext).isFalse() 60 | assertThat(page.memesInfo).isEmpty() 61 | } 62 | 63 | @Test 64 | fun parseVideoMeme() = runBlocking { 65 | val baseInfo = MemeInfo( 66 | pageUrl = "http://www.fuoriditesta.it/video-divertenti/cavallo-della-bambina-decide-di-giocare-nel-fango.html", 67 | title = "Il cavallo della bambina decide di giocare nel fango" 68 | ) 69 | val html = readResource("fuoriditesta-video/meme_video.html") 70 | val frameHtml = readResource("fuoriditesta-video/frame.html") 71 | 72 | mockGet(webClient, "http://www.fuoriditesta.it/video-divertenti/embed.php?id=3329", frameHtml) 73 | 74 | // -- 75 | val info = channel.parseMeme(baseInfo, html) 76 | 77 | // -- 78 | assertThat(info.pageUrl).isEqualTo(baseInfo.pageUrl) 79 | assertThat(info.title).isEqualTo(baseInfo.title) 80 | 81 | assertThat(info.publishDateTime).isEqualTo(LocalDateTime.of(2018, 5, 28, 0, 0)) 82 | assertThat(info.originUrl).isEqualTo("http://www.fuoriditesta.it/umorismo/files/cavallo-decide-di-giocare-nel-fango.mp4") 83 | } 84 | 85 | } 86 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/channel/LachschonChannelTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | import kotlinx.coroutines.runBlocking 4 | import me.ruslanys.ifunny.util.readResource 5 | import org.assertj.core.api.Assertions.assertThat 6 | import org.junit.jupiter.api.Test 7 | import java.time.LocalDateTime 8 | 9 | class LachschonChannelTests { 10 | 11 | private val channel = LachschonChannel() 12 | 13 | @Test 14 | fun firstPagePathTest() = runBlocking { 15 | val pagePath = channel.pagePath(1) 16 | assertThat(pagePath).isEqualTo("https://www.lachschon.de/?set_gallery_type=image&page=1") 17 | } 18 | 19 | @Test 20 | fun hundredPagePathTest() = runBlocking { 21 | val pagePath = channel.pagePath(100) 22 | assertThat(pagePath).isEqualTo("https://www.lachschon.de/?set_gallery_type=image&page=100") 23 | } 24 | 25 | @Test 26 | fun parseProperPageShouldReturnList() = runBlocking { 27 | val html = readResource("lachschon/page.html") 28 | 29 | // -- 30 | val page = channel.parsePage(1, html) 31 | val list = page.memesInfo 32 | 33 | // -- 34 | assertThat(page.hasNext).isTrue() 35 | assertThat(list).hasSize(10) 36 | assertThat(list).allMatch { it.pageUrl != null } 37 | assertThat(list).allMatch { it.title != null } 38 | assertThat(list).allMatch { it.author != null } 39 | assertThat(list).allMatch { it.likes != null } 40 | assertThat(list).allMatch { it.comments != null } 41 | } 42 | 43 | @Test 44 | fun parseLastPageShouldReturnHasNextFalse() = runBlocking { 45 | val html = readResource("lachschon/page_last.html") 46 | 47 | // -- 48 | val page = channel.parsePage(1, html) 49 | 50 | // -- 51 | assertThat(page.hasNext).isFalse() 52 | } 53 | 54 | @Test 55 | fun parseInvalidPageShouldReturnEmptyList() = runBlocking { 56 | val page = channel.parsePage(1, "") 57 | assertThat(page.hasNext).isFalse() 58 | assertThat(page.memesInfo).isEmpty() 59 | } 60 | 61 | @Test 62 | fun parsePictureMeme() = runBlocking { 63 | val baseInfo = MemeInfo( 64 | pageUrl = "https://www.lachschon.de/item/226337-Fastrichtig/", 65 | title = "Fast richtig", 66 | author = "panic", 67 | likes = 0, 68 | comments = 12 69 | ) 70 | val html = readResource("lachschon/meme.html") 71 | 72 | // -- 73 | val info = channel.parseMeme(baseInfo, html) 74 | 75 | // -- 76 | assertThat(info.pageUrl).isEqualTo(baseInfo.pageUrl) 77 | assertThat(info.title).isEqualTo(baseInfo.title) 78 | assertThat(info.author).isEqualTo(baseInfo.author) 79 | assertThat(info.likes).isEqualTo(baseInfo.likes) 80 | assertThat(info.comments).isEqualTo(baseInfo.comments) 81 | 82 | assertThat(info.originUrl).isEqualTo("https://img01.lachschon.de/images/226337_Fastrichtig_FuZCOZm.jpg") 83 | assertThat(info.publishDateTime).isEqualTo(LocalDateTime.of(2020, 1, 3, 12, 55, 8)) 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/channel/OrschlurchChannelTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | import kotlinx.coroutines.runBlocking 4 | import me.ruslanys.ifunny.util.readResource 5 | import org.assertj.core.api.Assertions.assertThat 6 | import org.junit.jupiter.api.Test 7 | import java.time.LocalDateTime 8 | 9 | class OrschlurchChannelTests { 10 | 11 | private val channel = OrschlurchChannel() 12 | 13 | 14 | @Test 15 | fun firstPagePathTest() = runBlocking { 16 | val pagePath = channel.pagePath(1) 17 | assertThat(pagePath).isEqualTo("https://de.orschlurch.net/area/videos") 18 | } 19 | 20 | @Test 21 | fun hundredPagePathTest() = runBlocking { 22 | val pagePath = channel.pagePath(100) 23 | assertThat(pagePath).isEqualTo("https://de.orschlurch.net/area/videos/seite/100") 24 | } 25 | 26 | @Test 27 | fun parseProperPageShouldReturnList() = runBlocking { 28 | val html = readResource("orschlurch/page.html") 29 | 30 | // -- 31 | val page = channel.parsePage(33, html) 32 | val list = page.memesInfo 33 | 34 | // -- 35 | assertThat(page.hasNext).isTrue() 36 | assertThat(list).hasSize(17) 37 | assertThat(list).allMatch { it.pageUrl != null } 38 | assertThat(list).allMatch { it.title != null } 39 | assertThat(list).allMatch { it.likes != null } 40 | assertThat(list).allMatch { it.comments != null } 41 | } 42 | 43 | @Test 44 | fun parseLastPageShouldReturnHasNextFalse() = runBlocking { 45 | val html = readResource("orschlurch/page_last.html") 46 | 47 | // -- 48 | val page = channel.parsePage(60, html) 49 | 50 | // -- 51 | assertThat(page.hasNext).isFalse() 52 | } 53 | 54 | @Test 55 | fun parseInvalidPageShouldReturnEmptyList() = runBlocking { 56 | val page = channel.parsePage(1, "") 57 | assertThat(page.hasNext).isFalse() 58 | assertThat(page.memesInfo).isEmpty() 59 | } 60 | 61 | @Test 62 | fun parseVideoMeme() = runBlocking { 63 | val baseInfo = MemeInfo( 64 | pageUrl = "https://de.orschlurch.net/post/der-etwas-andere-hochzeitskorso", 65 | title = "Der etwas andere Hochzeitskorso", 66 | likes = 1, 67 | comments = 0 68 | ) 69 | val html = readResource("orschlurch/meme_video.html") 70 | 71 | // -- 72 | val info = channel.parseMeme(baseInfo, html) 73 | 74 | // -- 75 | assertThat(info.pageUrl).isEqualTo(baseInfo.pageUrl) 76 | assertThat(info.title).isEqualTo(baseInfo.title) 77 | assertThat(info.likes).isEqualTo(baseInfo.likes) 78 | assertThat(info.comments).isEqualTo(baseInfo.comments) 79 | 80 | assertThat(info.originUrl).isEqualTo("https://static.orschlurch.net/videos/2090/667139c1f82508e0.mp4") 81 | assertThat(info.author).isEqualTo("Admin") 82 | assertThat(info.publishDateTime).isEqualTo(LocalDateTime.of(2019, 7, 9, 19, 17)) 83 | } 84 | 85 | } 86 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/channel/YatahongaChannelTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.channel 2 | 3 | import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper 4 | import kotlinx.coroutines.runBlocking 5 | import me.ruslanys.ifunny.util.readResource 6 | import org.assertj.core.api.Assertions.assertThat 7 | import org.junit.jupiter.api.Test 8 | import java.time.LocalDateTime 9 | 10 | class YatahongaChannelTests { 11 | 12 | private val channel = YatahongaChannel(jacksonObjectMapper()) 13 | 14 | @Test 15 | fun firstPagePathTest() = runBlocking { 16 | val pagePath = channel.pagePath(1) 17 | assertThat(pagePath).isEqualTo("https://www.yatahonga.com/nouveautes/") 18 | } 19 | 20 | @Test 21 | fun hundredPagePathTest() = runBlocking { 22 | val pagePath = channel.pagePath(100) 23 | assertThat(pagePath).isEqualTo("https://www.yatahonga.com/nouveautes/p100/") 24 | } 25 | 26 | @Test 27 | fun parseProperPageShouldReturnList() = runBlocking { 28 | val html = readResource("yatahonga/page.html") 29 | 30 | // -- 31 | val page = channel.parsePage(1, html) 32 | val list = page.memesInfo 33 | 34 | // -- 35 | assertThat(page.hasNext).isTrue() 36 | assertThat(list).hasSize(20) 37 | assertThat(list).allMatch { it.pageUrl != null } 38 | assertThat(list).allMatch { it.title != null } 39 | assertThat(list).allMatch { it.likes != 0 } 40 | assertThat(list).allMatch { it.comments != 0 } 41 | } 42 | 43 | @Test 44 | fun parseLastPageShouldReturnHasNextFalse() = runBlocking { 45 | val html = readResource("yatahonga/page_last.html") 46 | 47 | // -- 48 | val page = channel.parsePage(1, html) 49 | 50 | // -- 51 | assertThat(page.hasNext).isFalse() 52 | } 53 | 54 | @Test 55 | fun parseInvalidPageShouldReturnEmptyList() = runBlocking { 56 | val page = channel.parsePage(1, "") 57 | assertThat(page.hasNext).isFalse() 58 | assertThat(page.memesInfo).isEmpty() 59 | } 60 | 61 | @Test 62 | fun parsePictureMeme() = runBlocking { 63 | val baseInfo = MemeInfo( 64 | pageUrl = "https://www.yatahonga.com/actualites/768nq/", 65 | title = "La grève SNCF commence à se voir", 66 | likes = 223, 67 | comments = 8 68 | ) 69 | val html = readResource("yatahonga/meme_picture.html") 70 | 71 | // -- 72 | val info = channel.parseMeme(baseInfo, html) 73 | 74 | // -- 75 | assertThat(info.pageUrl).isEqualTo(baseInfo.pageUrl) 76 | assertThat(info.title).isEqualTo(baseInfo.title) 77 | assertThat(info.likes).isEqualTo(baseInfo.likes) 78 | assertThat(info.comments).isEqualTo(baseInfo.comments) 79 | 80 | assertThat(info.originUrl).isEqualTo("https://www.yatahonga.com/data/media/1/202010/5e13966a8c614.jpg") 81 | assertThat(info.author).isEqualTo("ROUGETNOIRS") 82 | assertThat(info.publishDateTime).isEqualTo(LocalDateTime.of(2020, 1, 6, 0, 0)) 83 | } 84 | 85 | @Test 86 | fun parseVideoMeme() = runBlocking { 87 | val baseInfo = MemeInfo( 88 | pageUrl = "https://www.yatahonga.com/gif/c78nq/", 89 | title = "Batman", 90 | likes = 4, 91 | comments = 2 92 | ) 93 | val html = readResource("yatahonga/meme_video.html") 94 | 95 | // -- 96 | val info = channel.parseMeme(baseInfo, html) 97 | 98 | // -- 99 | assertThat(info.pageUrl).isEqualTo(baseInfo.pageUrl) 100 | assertThat(info.title).isEqualTo(baseInfo.title) 101 | assertThat(info.likes).isEqualTo(baseInfo.likes) 102 | assertThat(info.comments).isEqualTo(baseInfo.comments) 103 | 104 | assertThat(info.originUrl).isEqualTo("https://www.yatahonga.com/data/media/11/202010/5e16bbcb751a3.mp4") 105 | assertThat(info.author).isEqualTo("sawubona94") 106 | assertThat(info.publishDateTime).isEqualTo(LocalDateTime.of(2020, 1, 9, 0, 0)) 107 | } 108 | 109 | } 110 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/controller/FeedControllerTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.controller 2 | 3 | import me.ruslanys.ifunny.base.ControllerTests 4 | import org.junit.jupiter.api.Test 5 | import org.springframework.boot.test.autoconfigure.web.reactive.WebFluxTest 6 | import org.springframework.http.HttpStatus 7 | import org.springframework.http.MediaType 8 | 9 | @WebFluxTest(FeedController::class) 10 | class FeedControllerTests : ControllerTests() { 11 | 12 | @Test 13 | fun indexHtmlShouldReturnFeedFrontend() { 14 | webClient.get() 15 | .uri("/index.html") 16 | .exchange() 17 | .expectStatus().isOk 18 | .expectHeader().contentTypeCompatibleWith(MediaType.TEXT_HTML) 19 | } 20 | 21 | @Test 22 | fun indexHtmShouldReturnFeedFrontend() { 23 | webClient.get() 24 | .uri("/index.htm") 25 | .exchange() 26 | .expectStatus().isOk 27 | .expectHeader().contentTypeCompatibleWith(MediaType.TEXT_HTML) 28 | } 29 | 30 | @Test 31 | fun indexShouldReturnFeedFrontend() { 32 | webClient.get() 33 | .uri("/index") 34 | .exchange() 35 | .expectStatus().isOk 36 | .expectHeader().contentTypeCompatibleWith(MediaType.TEXT_HTML) 37 | } 38 | 39 | @Test 40 | fun rootShouldReturnFeedFrontend() { 41 | webClient.get() 42 | .uri("/") 43 | .exchange() 44 | .expectStatus().isOk 45 | .expectHeader().contentTypeCompatibleWith(MediaType.TEXT_HTML) 46 | } 47 | 48 | @Test 49 | fun feedShouldReturnFeedFrontend() { 50 | webClient.get() 51 | .uri("/feed") 52 | .exchange() 53 | .expectStatus().isOk 54 | .expectHeader().contentTypeCompatibleWith(MediaType.TEXT_HTML) 55 | } 56 | 57 | @Test 58 | fun feedHtmlShouldReturnFeedFrontend() { 59 | webClient.get() 60 | .uri("/feed.html") 61 | .exchange() 62 | .expectStatus().isOk 63 | .expectHeader().contentTypeCompatibleWith(MediaType.TEXT_HTML) 64 | } 65 | 66 | @Test 67 | fun postRequestShouldNotBeAllowed() { 68 | webClient.post() 69 | .uri("/feed.html") 70 | .exchange() 71 | .expectStatus().isEqualTo(HttpStatus.METHOD_NOT_ALLOWED) 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/controller/MemeControllerTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.controller 2 | 3 | import com.nhaarman.mockitokotlin2.any 4 | import com.nhaarman.mockitokotlin2.given 5 | import kotlinx.coroutines.runBlocking 6 | import me.ruslanys.ifunny.base.ControllerTests 7 | import me.ruslanys.ifunny.exception.NotFoundException 8 | import me.ruslanys.ifunny.service.MemeService 9 | import me.ruslanys.ifunny.util.createDummyMeme 10 | import org.junit.jupiter.api.Test 11 | import org.springframework.boot.test.autoconfigure.web.reactive.WebFluxTest 12 | import org.springframework.boot.test.mock.mockito.MockBean 13 | import org.springframework.http.MediaType 14 | import org.springframework.test.web.reactive.server.expectBody 15 | 16 | @WebFluxTest(MemeController::class, GlobalExceptionHandler::class) 17 | class MemeControllerTests : ControllerTests() { 18 | 19 | @MockBean private lateinit var memeService: MemeService 20 | 21 | @Test 22 | fun showPageShouldReturn404StatusWhenMemeNotFound() = runBlocking { 23 | given(memeService.getById(any())).willThrow(NotFoundException("Not found")) 24 | 25 | webClient.get() 26 | .uri("/meme/112233") 27 | .exchange() 28 | .expectStatus().isNotFound 29 | .expectBody().isEqualTo("Not found") 30 | } 31 | 32 | @Test 33 | fun showPageShouldReturnModelAndView() = runBlocking { 34 | val meme = createDummyMeme() 35 | given(memeService.getById("123")).willReturn(meme) 36 | 37 | webClient.get() 38 | .uri("/meme/123") 39 | .exchange() 40 | .expectStatus().isOk 41 | .expectHeader().contentTypeCompatibleWith(MediaType.TEXT_HTML) 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/controller/api/FeedApiControllerTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.controller.api 2 | 3 | import com.nhaarman.mockitokotlin2.any 4 | import com.nhaarman.mockitokotlin2.given 5 | import kotlinx.coroutines.runBlocking 6 | import me.ruslanys.ifunny.base.ControllerTests 7 | import me.ruslanys.ifunny.controller.GlobalExceptionHandler 8 | import me.ruslanys.ifunny.domain.Language 9 | import me.ruslanys.ifunny.domain.Meme 10 | import me.ruslanys.ifunny.exception.NotFoundException 11 | import me.ruslanys.ifunny.service.MemeService 12 | import me.ruslanys.ifunny.util.createDummyFile 13 | import me.ruslanys.ifunny.util.createDummyMeme 14 | import me.ruslanys.ifunny.util.readResource 15 | import org.bson.types.ObjectId 16 | import org.junit.jupiter.api.Test 17 | import org.springframework.boot.test.autoconfigure.web.reactive.WebFluxTest 18 | import org.springframework.boot.test.mock.mockito.MockBean 19 | import org.springframework.data.domain.PageImpl 20 | import org.springframework.data.domain.Sort 21 | import org.springframework.http.HttpStatus 22 | import org.springframework.http.MediaType 23 | 24 | @WebFluxTest(FeedApiController::class, GlobalExceptionHandler::class) 25 | class FeedApiControllerTests : ControllerTests() { 26 | 27 | @MockBean 28 | private lateinit var memeService: MemeService 29 | 30 | 31 | @Test 32 | fun postMethodShouldNotBeAllowedToGetPage() { 33 | webClient.post() 34 | .uri("/api/feed") 35 | .exchange() 36 | .expectStatus().isEqualTo(HttpStatus.METHOD_NOT_ALLOWED) 37 | } 38 | 39 | @Test 40 | fun getPageWithoutSpecifiedLanguageShouldReturn400() { 41 | webClient.get() 42 | .uri("/api/feed") 43 | .exchange() 44 | .expectStatus().isBadRequest 45 | } 46 | 47 | @Test 48 | fun getPageWithWrongLanguageCodeShouldReturn400() { 49 | webClient.get() 50 | .uri("/api/feed?language=ch") 51 | .exchange() 52 | .expectStatus().isBadRequest 53 | } 54 | 55 | @Test 56 | fun getPageWithWrongOffsetShouldReturn400() { 57 | webClient.get() 58 | .uri("/api/feed?language=de&offset=-1") 59 | .exchange() 60 | .expectStatus().isBadRequest 61 | } 62 | 63 | @Test 64 | fun getPageWithWrongLimitShouldReturn400() { 65 | webClient.get() 66 | .uri("/api/feed?language=de&limit=101") 67 | .exchange() 68 | .expectStatus().isBadRequest 69 | } 70 | 71 | @Test 72 | fun getPageWithRestrictedSortingShouldReturn400() { 73 | webClient.get() 74 | .uri("/api/feed?language=de&sortBy=author") 75 | .exchange() 76 | .expectStatus().isBadRequest 77 | } 78 | 79 | @Test 80 | fun getPageWithProperParametersShouldReturnPageResponse() = runBlocking { 81 | val memes = listOf( 82 | createDummyMeme(title = "Title 1", file = createDummyFile(name = "1.jpg"), id = ObjectId("5e107cc18b12145d4624f23c")), 83 | createDummyMeme(title = "Title 2", file = createDummyFile(name = "2.jpg"), id = ObjectId("5e107cc18b12145d4624f23d")), 84 | createDummyMeme(title = "Title 3", file = createDummyFile(name = "3.jpg"), id = ObjectId("5e107cc18b12145d4624f23e")) 85 | ) 86 | val pageRequest = FeedApiController.FeedPageRequest().apply { 87 | offset = 10 88 | setLimit(5) 89 | sortBy = "likes" 90 | sortDirection = Sort.Direction.DESC 91 | } 92 | 93 | given(memeService.getPage(Language.GERMAN, pageRequest)).willReturn(PageImpl(memes)) 94 | 95 | // -- 96 | val json = readResource("feed_getPage.json") 97 | 98 | // -- 99 | webClient.get() 100 | .uri { 101 | it.path("/api/feed") 102 | .queryParam("language", "de") 103 | .queryParam("offset", "10") 104 | .queryParam("limit", "5") 105 | .queryParam("sortBy", "likes") 106 | .queryParam("sortDirection", "DESC") 107 | .build() 108 | } 109 | 110 | .exchange() 111 | .expectStatus().isOk 112 | .expectBody().json(json) 113 | } 114 | 115 | @Test 116 | fun getByIdShouldReturn404() = runBlocking { 117 | given(memeService.getById(any())).willThrow(NotFoundException("Not Found")) 118 | 119 | webClient.get() 120 | .uri("/api/feed/123") 121 | .accept(MediaType.APPLICATION_JSON) 122 | .exchange() 123 | .expectStatus().isNotFound 124 | .expectBody().json(""" 125 | { 126 | "status": 404, 127 | "error": "Not Found" 128 | } 129 | """.trimIndent()) 130 | } 131 | 132 | @Test 133 | fun getByIdShouldReturnMemeDto() = runBlocking { 134 | val meme = createDummyMeme(id = ObjectId("5e107cc18b12145d4624f23c")) 135 | given(memeService.getById("321")).willReturn(meme) 136 | 137 | val json = readResource("feed_getById.json") 138 | 139 | webClient.get() 140 | .uri("/api/feed/321") 141 | .exchange() 142 | .expectStatus().isOk 143 | .expectBody().json(json) 144 | } 145 | 146 | } 147 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/controller/dto/PageRequestTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.controller.dto 2 | 3 | import org.assertj.core.api.Assertions.assertThat 4 | import org.junit.jupiter.api.Test 5 | import org.springframework.data.domain.Sort 6 | 7 | class PageRequestTests { 8 | 9 | @Test 10 | fun pageNumberShouldStartsWithOne() { 11 | val pageRequest = PageRequest(0, 10) 12 | assertThat(pageRequest.pageNumber).isEqualTo(1) 13 | } 14 | 15 | @Test 16 | fun nextPageTest() { 17 | val first = PageRequest(0, 10) 18 | 19 | val second = first.next() 20 | assertThat(second.offset).isEqualTo(10) 21 | assertThat(second.pageSize).isEqualTo(10) 22 | 23 | val third = second.next() 24 | assertThat(third.offset).isEqualTo(20) 25 | assertThat(third.pageSize).isEqualTo(10) 26 | } 27 | 28 | @Test 29 | fun firstPageShouldNotHavePrevious() { 30 | val pageRequest = PageRequest(0, 10) 31 | assertThat(pageRequest.hasPrevious()).isFalse() 32 | } 33 | 34 | @Test 35 | fun secondPageShouldHavePrevios() { 36 | val secondPage = PageRequest(10, 10) 37 | 38 | assertThat(secondPage.hasPrevious()).isTrue() 39 | } 40 | 41 | @Test 42 | fun previousOrFirstShouldReturnPrevious() { 43 | val third = PageRequest(20, 10) 44 | val second = PageRequest(10, 10) 45 | 46 | assertThat(third.previousOrFirst()).isEqualTo(second) 47 | } 48 | 49 | @Test 50 | fun previousOrFirstShouldReturnFirst() { 51 | val second = PageRequest(10, 10) 52 | val first = PageRequest(0, 10) 53 | 54 | assertThat(second.previousOrFirst()).isEqualTo(first) 55 | } 56 | 57 | @Test 58 | fun previousOrFirstForTheFirstPageShouldReturnFirst() { 59 | val first = PageRequest(0, 10) 60 | 61 | assertThat(first.previousOrFirst()).isEqualTo(first) 62 | } 63 | 64 | @Test 65 | fun hashCodeShouldBeEqualsIfObjectsAreEquals() { 66 | val first = PageRequest(0, 10) 67 | 68 | assertThat(first.hashCode()).isEqualTo(first.previousOrFirst().hashCode()) 69 | } 70 | 71 | @Test 72 | fun hashCodeShouldBeDifferentIfObjectsAreDifferent() { 73 | val second = PageRequest(10, 10) 74 | 75 | assertThat(second.hashCode()).isNotEqualTo(second.previousOrFirst().hashCode()) 76 | } 77 | 78 | @Test 79 | fun getSortAscTest() { 80 | val request = PageRequest(0, 10, sortBy = "fieldName", sortDirection = Sort.Direction.ASC) 81 | assertThat(request.sort).isEqualTo(Sort.by(Sort.Direction.ASC, "fieldName")) 82 | } 83 | 84 | @Test 85 | fun getSortDescTest() { 86 | val request = PageRequest(0, 10, sortBy = "name", sortDirection = Sort.Direction.DESC) 87 | assertThat(request.sort).isEqualTo(Sort.by(Sort.Direction.DESC, "name")) 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/grab/CoordinatorTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.grab 2 | 3 | import com.nhaarman.mockitokotlin2.any 4 | import com.nhaarman.mockitokotlin2.never 5 | import com.nhaarman.mockitokotlin2.times 6 | import com.nhaarman.mockitokotlin2.verify 7 | import kotlinx.coroutines.channels.SendChannel 8 | import kotlinx.coroutines.runBlocking 9 | import me.ruslanys.ifunny.base.ServiceTests 10 | import me.ruslanys.ifunny.channel.DebesteChannel 11 | import me.ruslanys.ifunny.channel.FunpotChannel 12 | import me.ruslanys.ifunny.channel.Page 13 | import me.ruslanys.ifunny.grab.event.GrabEvent 14 | import me.ruslanys.ifunny.grab.event.PageIndexRequest 15 | import me.ruslanys.ifunny.grab.event.PageIndexSuccessful 16 | import me.ruslanys.ifunny.property.GrabProperties 17 | import me.ruslanys.ifunny.repository.PageRepository 18 | import org.junit.jupiter.api.BeforeEach 19 | import org.junit.jupiter.api.Test 20 | import org.mockito.BDDMockito.given 21 | import org.mockito.Mock 22 | import reactor.core.publisher.Mono 23 | 24 | 25 | class CoordinatorTests : ServiceTests() { 26 | 27 | // @formatter:off 28 | @Mock private lateinit var eventChannel: SendChannel 29 | @Mock private lateinit var pageRepository: PageRepository 30 | // @formatter:on 31 | 32 | private val channels = listOf(DebesteChannel(), FunpotChannel()) 33 | private val grabProperties = GrabProperties() 34 | 35 | private lateinit var coordinator: Coordinator 36 | 37 | @BeforeEach 38 | fun setUp() { 39 | coordinator = Coordinator(channels, eventChannel, pageRepository, grabProperties) 40 | } 41 | 42 | @Test 43 | fun scheduleGrabbingShouldStartWithTheFirstPagePerEachChannel() = runBlocking { 44 | given(pageRepository.incCurrent(any())).willReturn(Mono.just(1)) 45 | 46 | coordinator.scheduleGrabbing().join() 47 | verify(eventChannel, times(channels.size)).send(any()) 48 | } 49 | 50 | @Test 51 | fun whileChannelIsNotIndexedItShouldContinueParsing() = runBlocking { 52 | given(pageRepository.getLast(any())).willReturn(Mono.empty()) // not indexed 53 | given(pageRepository.incCurrent(any())).willReturn(Mono.just(5)) 54 | 55 | coordinator.handleEvent(PageIndexSuccessful(channels.first(), Page(1, true, listOf()), 2)) 56 | verify(eventChannel).send(any()) 57 | 58 | } 59 | 60 | @Test 61 | fun whenChannelIsNotIndexedAndItReachesTheLastPageItShouldFinalizeTheState() = runBlocking { 62 | given(pageRepository.getLast(any())).willReturn(Mono.empty()) // not indexed 63 | given(pageRepository.setLast(any(), any(), any())).willReturn(Mono.just(true)) // Reactive stub 64 | given(pageRepository.clearCurrent(any())).willReturn(Mono.just(1)) // Reactive stub 65 | 66 | // -- 67 | coordinator.handleEvent(PageIndexSuccessful(channels.first(), Page(100, false, listOf()), 2)) 68 | 69 | // -- 70 | verify(pageRepository).setLast(channels.first(), 100, grabProperties.retention.fullIndex) 71 | verify(pageRepository).clearCurrent(channels.first()) 72 | verify(eventChannel, never()).send(any()) 73 | } 74 | 75 | @Test 76 | fun fullyIndexedChannelShouldBeParsedUntilItHasNewMemes() = runBlocking { 77 | given(pageRepository.getLast(any())).willReturn(Mono.just(100)) // indexed 78 | given(pageRepository.incCurrent(any())).willReturn(Mono.just(5)) 79 | 80 | // -- 81 | val hasNext = true 82 | val new = 1 83 | 84 | coordinator.handleEvent(PageIndexSuccessful(channels.first(), Page(100, hasNext, listOf()), new)) 85 | 86 | // -- 87 | verify(eventChannel).send(any()) 88 | } 89 | 90 | @Test 91 | fun fullyIndexedChannelShouldCleanThePositionWhenItReachesTheOldPage() = runBlocking { 92 | given(pageRepository.getLast(any())).willReturn(Mono.just(100)) // indexed 93 | given(pageRepository.clearCurrent(any())).willReturn(Mono.just(1)) // Reactive stub 94 | 95 | // -- 96 | val hasNext = true 97 | val new = 0 98 | 99 | coordinator.handleEvent(PageIndexSuccessful(channels.first(), Page(100, hasNext, listOf()), new)) 100 | 101 | // -- 102 | verify(pageRepository).clearCurrent(channels.first()) 103 | verify(eventChannel, never()).send(any()) 104 | } 105 | 106 | @Test 107 | fun fullyIndexedChannelShouldCleanThePositionWhenItReachesTheLast() = runBlocking { 108 | given(pageRepository.getLast(any())).willReturn(Mono.just(100)) // indexed 109 | given(pageRepository.clearCurrent(any())).willReturn(Mono.just(1)) // Reactive stub 110 | 111 | // -- 112 | val hasNext = false 113 | val new = 1 114 | 115 | coordinator.handleEvent(PageIndexSuccessful(channels.first(), Page(100, hasNext, listOf()), new)) 116 | 117 | // -- 118 | verify(pageRepository).clearCurrent(channels.first()) 119 | verify(eventChannel, never()).send(any()) 120 | } 121 | 122 | } 123 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/grab/MemeIndexerTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.grab 2 | 3 | import com.nhaarman.mockitokotlin2.argumentCaptor 4 | import com.nhaarman.mockitokotlin2.mock 5 | import com.nhaarman.mockitokotlin2.times 6 | import com.nhaarman.mockitokotlin2.verify 7 | import kotlinx.coroutines.channels.SendChannel 8 | import kotlinx.coroutines.runBlocking 9 | import me.ruslanys.ifunny.channel.DebesteChannel 10 | import me.ruslanys.ifunny.channel.MemeInfo 11 | import me.ruslanys.ifunny.grab.event.GrabEvent 12 | import me.ruslanys.ifunny.grab.event.MemeIndexRequest 13 | import me.ruslanys.ifunny.grab.event.ResourceDownloadRequest 14 | import me.ruslanys.ifunny.util.mockGet 15 | import org.assertj.core.api.Assertions.assertThat 16 | import org.junit.jupiter.api.Test 17 | import org.springframework.core.io.ClassPathResource 18 | import org.springframework.web.reactive.function.client.WebClient 19 | 20 | 21 | class MemeIndexerTests { 22 | 23 | private val webClient: WebClient = mock() 24 | private val eventChannel: SendChannel = mock() 25 | 26 | private val memeIndexer = MemeIndexer(webClient, eventChannel) 27 | 28 | @Test 29 | fun memeIndexationShouldEmitDownloadRequest() = runBlocking { 30 | val channel = DebesteChannel() 31 | val memeInfo = MemeInfo(pageUrl = "http://debeste.de/meme-123") 32 | 33 | mockGet(webClient, memeInfo.pageUrl!!, ClassPathResource("debeste_meme.html", MemeIndexer::class.java)) 34 | 35 | // -- 36 | memeIndexer.handleEvent(MemeIndexRequest(channel, memeInfo)) 37 | 38 | // -- 39 | val eventCaptor = argumentCaptor() 40 | verify(eventChannel, times(1)).send(eventCaptor.capture()) 41 | 42 | assertThat(eventCaptor.firstValue.info.pageUrl).isEqualTo(memeInfo.pageUrl) 43 | assertThat(eventCaptor.firstValue.info.originUrl).isEqualTo("http://debeste.de/upload/e4b9c282887d58b5ecfcc7d02823d4e4.jpg") 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/grab/PageIndexerTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.grab 2 | 3 | import com.nhaarman.mockitokotlin2.any 4 | import com.nhaarman.mockitokotlin2.mock 5 | import com.nhaarman.mockitokotlin2.times 6 | import com.nhaarman.mockitokotlin2.verify 7 | import kotlinx.coroutines.channels.SendChannel 8 | import kotlinx.coroutines.runBlocking 9 | import me.ruslanys.ifunny.channel.DebesteChannel 10 | import me.ruslanys.ifunny.grab.event.GrabEvent 11 | import me.ruslanys.ifunny.grab.event.MemeIndexRequest 12 | import me.ruslanys.ifunny.grab.event.PageIndexRequest 13 | import me.ruslanys.ifunny.grab.event.PageIndexSuccessful 14 | import me.ruslanys.ifunny.service.MemeService 15 | import me.ruslanys.ifunny.util.createDummyMeme 16 | import me.ruslanys.ifunny.util.mockGet 17 | import org.junit.jupiter.api.Test 18 | import org.mockito.BDDMockito.given 19 | import org.springframework.core.io.ClassPathResource 20 | import org.springframework.web.reactive.function.client.WebClient 21 | 22 | 23 | class PageIndexerTests { 24 | 25 | private val webClient: WebClient = mock() 26 | private val eventChannel: SendChannel = mock() 27 | private val memeService: MemeService = mock() 28 | 29 | private val pageIndexer: PageIndexer = PageIndexer(webClient, eventChannel, memeService) 30 | 31 | @Test 32 | fun pageIndexationShouldEmitMemesIndexationEvents() = runBlocking { 33 | val channel = DebesteChannel() 34 | val pageNumber = 100 35 | 36 | given(memeService.findByPageUrls(any())).willReturn(listOf()) 37 | mockGet(webClient, channel.pagePath(pageNumber), ClassPathResource("debeste_page.html", PageIndexerTests::class.java)) 38 | 39 | // -- 40 | pageIndexer.handleEvent(PageIndexRequest(channel, pageNumber)) 41 | 42 | // -- 43 | verify(eventChannel, times(8)).send(any()) 44 | verify(eventChannel, times(1)).send(any()) 45 | } 46 | 47 | @Test 48 | fun pageIndexationShouldSkipProcessedMemes() = runBlocking { 49 | val channel = DebesteChannel() 50 | val pageNumber = 100 51 | 52 | val urls = listOf( 53 | "http://debeste.de/109036/Ich-sp-re-die-Macht-in-mir-K-nnte-allerdings-auch", 54 | "http://debeste.de/109026/Beste-Freunde", 55 | "http://debeste.de/109180/Hast-du-immer-noch-Lust,-so-ein-K-tzchen-zu-streicheln" 56 | ) 57 | val memes = urls.map { createDummyMeme(pageUrl = it) } 58 | 59 | given(memeService.findByPageUrls(any())).willReturn(memes) 60 | mockGet(webClient, channel.pagePath(pageNumber), ClassPathResource("debeste_page.html", PageIndexerTests::class.java)) 61 | 62 | // -- 63 | pageIndexer.handleEvent(PageIndexRequest(channel, pageNumber)) 64 | 65 | // -- 66 | verify(eventChannel, times(5)).send(any()) 67 | verify(eventChannel, times(1)).send(any()) 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/grab/ResourceDownloaderTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.grab 2 | 3 | import com.nhaarman.mockitokotlin2.any 4 | import com.nhaarman.mockitokotlin2.argumentCaptor 5 | import com.nhaarman.mockitokotlin2.mock 6 | import kotlinx.coroutines.runBlocking 7 | import me.ruslanys.ifunny.channel.Channel 8 | import me.ruslanys.ifunny.channel.DebesteChannel 9 | import me.ruslanys.ifunny.channel.MemeInfo 10 | import me.ruslanys.ifunny.domain.S3File 11 | import me.ruslanys.ifunny.grab.event.ResourceDownloadRequest 12 | import me.ruslanys.ifunny.property.AwsS3Properties 13 | import me.ruslanys.ifunny.service.MemeService 14 | import me.ruslanys.ifunny.util.mockGetBytes 15 | import org.assertj.core.api.Assertions.assertThat 16 | import org.junit.jupiter.api.BeforeEach 17 | import org.junit.jupiter.api.Test 18 | import org.junit.jupiter.api.assertThrows 19 | import org.mockito.ArgumentMatchers.anyString 20 | import org.mockito.BDDMockito.given 21 | import org.mockito.Mockito.* 22 | import org.springframework.core.io.ClassPathResource 23 | import org.springframework.web.reactive.function.client.WebClient 24 | import software.amazon.awssdk.core.async.AsyncRequestBody 25 | import software.amazon.awssdk.services.s3.S3AsyncClient 26 | import software.amazon.awssdk.services.s3.S3AsyncClientBuilder 27 | import software.amazon.awssdk.services.s3.model.PutObjectRequest 28 | import software.amazon.awssdk.services.s3.model.PutObjectResponse 29 | import java.util.concurrent.CompletableFuture 30 | 31 | class ResourceDownloaderTests { 32 | 33 | private val webClient: WebClient = mock() 34 | private val memeService: MemeService = mock() 35 | private val s3ClientBuilder: S3AsyncClientBuilder = mock() 36 | private val s3Client: S3AsyncClient = mock() 37 | private val s3Properties: AwsS3Properties = AwsS3Properties(region = "eu-central-1", bucket = "bucket") 38 | 39 | private val resourceDownloader = ResourceDownloader(webClient, memeService, s3ClientBuilder, s3Properties) 40 | 41 | @BeforeEach 42 | fun setUp() { 43 | given(s3ClientBuilder.build()).willReturn(s3Client) 44 | } 45 | 46 | @Test 47 | fun downloadResourceShouldPutDataToS3AndMongoDb() = runBlocking { 48 | val originUrl = "http://debeste.de/meme.jpg" 49 | val resource = ClassPathResource("picture.jpg", ResourceDownloaderTests::class.java) 50 | val channel = DebesteChannel() 51 | val request = ResourceDownloadRequest(channel, MemeInfo(originUrl = originUrl)) 52 | val future = CompletableFuture() 53 | future.complete(PutObjectResponse.builder().build()) 54 | 55 | // -- 56 | given(memeService.isExists(any(), anyString())).willReturn(false) 57 | given(s3Client.putObject(any(), any())).willReturn(future) // Reactive stub 58 | mockGetBytes(webClient, originUrl, resource) 59 | 60 | // -- 61 | resourceDownloader.handleEvent(request) 62 | 63 | // -- 64 | val channelCaptor = argumentCaptor() 65 | val fileCaptor = argumentCaptor() 66 | val fingerprintCaptor = argumentCaptor() 67 | 68 | verify(memeService, times(1)).add(channelCaptor.capture(), any(), fileCaptor.capture(), fingerprintCaptor.capture()) 69 | verify(s3Client, times(1)).putObject(any(), any()) 70 | 71 | // -- 72 | assertThat(channelCaptor.firstValue).isEqualTo(channel) 73 | assertThat(fileCaptor.firstValue.checksum).isEqualTo("5A267D4B21F34C2900D7BEC44684A4B2") 74 | assertThat(fingerprintCaptor.firstValue).isEqualTo("F3F3C40425813979") 75 | } 76 | 77 | @Test 78 | fun downloadResourceShouldAvoidDuplicatesByFingerprint() = runBlocking { 79 | val originUrl = "http://debeste.de/meme.jpg" 80 | val resource = ClassPathResource("picture.jpg", ResourceDownloaderTests::class.java) 81 | val channel = DebesteChannel() 82 | val request = ResourceDownloadRequest(channel, MemeInfo(originUrl = originUrl)) 83 | 84 | // -- 85 | given(memeService.isExists(any(), anyString())).willReturn(true) 86 | mockGetBytes(webClient, originUrl, resource) 87 | 88 | // -- 89 | resourceDownloader.handleEvent(request) 90 | 91 | // -- 92 | verify(s3Client, never()).putObject(any(), any()) 93 | verify(memeService, never()).add(any(), any(), any(), any()) 94 | } 95 | 96 | @Test 97 | fun downloadResourceShouldThrowIllegalStateExceptionWhenThereIsEmptyBody() = runBlocking { 98 | val originUrl = "http://debeste.de/meme.jpg" 99 | val channel = DebesteChannel() 100 | val request = ResourceDownloadRequest(channel, MemeInfo(originUrl = originUrl)) 101 | 102 | // -- 103 | given(memeService.isExists(any(), anyString())).willReturn(true) 104 | mockGetBytes(webClient, originUrl, ByteArray(0)) 105 | 106 | // -- 107 | assertThrows { 108 | runBlocking { 109 | resourceDownloader.handleEvent(request) 110 | } 111 | } 112 | 113 | // -- 114 | verify(s3Client, never()).putObject(any(), any()) 115 | verify(memeService, never()).add(any(), any(), any(), any()) 116 | } 117 | 118 | } 119 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/repository/MemeRepositoryTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.repository 2 | 3 | import me.ruslanys.ifunny.base.MongoRepositoryTests 4 | import me.ruslanys.ifunny.domain.Language 5 | import me.ruslanys.ifunny.domain.Meme 6 | import me.ruslanys.ifunny.util.createDummyMeme 7 | import org.assertj.core.api.Assertions.assertThat 8 | import org.junit.jupiter.api.AfterEach 9 | import org.junit.jupiter.api.DisplayName 10 | import org.junit.jupiter.api.Test 11 | import org.junit.jupiter.api.assertThrows 12 | import org.springframework.beans.factory.annotation.Autowired 13 | import org.springframework.dao.DuplicateKeyException 14 | import org.springframework.data.domain.PageRequest 15 | import org.springframework.data.domain.Sort 16 | import java.time.LocalDateTime 17 | import java.util.* 18 | 19 | class MemeRepositoryTests : MongoRepositoryTests() { 20 | 21 | @Autowired 22 | private lateinit var repository: MemeRepository 23 | 24 | @AfterEach 25 | fun tearDown() { 26 | repository.deleteAll().block() 27 | } 28 | 29 | @Test 30 | fun persistTest() { 31 | val meme = repository.save(createDummyMeme(publishDateTime = LocalDateTime.of(2019, 12, 12, 12, 12, 12))).block()!! 32 | 33 | // -- 34 | val memeFromDb = mongoTemplate.findById(meme.id, Meme::class.java) 35 | 36 | // -- 37 | assertThat(memeFromDb).isNotNull 38 | assertThat(memeFromDb).isEqualTo(meme) 39 | } 40 | 41 | @Test 42 | fun breakUniqueConstraintTest() { 43 | val firstMeme = createDummyMeme(pageUrl = UUID.randomUUID().toString()) 44 | val secondMeme = createDummyMeme(pageUrl = firstMeme.pageUrl) 45 | 46 | repository.save(firstMeme).block() 47 | 48 | assertThrows { 49 | repository.save(secondMeme).block() 50 | } 51 | } 52 | 53 | @Test 54 | fun findByLanguageTest() { 55 | for (i in 1..30) { 56 | repository.save(createDummyMeme(language = Language.PORTUGUESE)).block() 57 | } 58 | repository.save(createDummyMeme(language = Language.RUSSIAN)).block() 59 | 60 | // -- 61 | val list = repository.findByLanguage(Language.PORTUGUESE.code, PageRequest.of(0, 10, Sort.by(Sort.Direction.DESC, "id"))) 62 | .collectList() 63 | .block()!! 64 | 65 | val count = repository.countByLanguage(Language.PORTUGUESE.code).block()!! 66 | 67 | // -- 68 | assertThat(count).isEqualTo(30) 69 | assertThat(list.size).isEqualTo(10) 70 | } 71 | 72 | @DisplayName(""" 73 | The page should display memes with publish date sorted by date DESC, and after that, memes without publishing date sorted by ID DESC. 74 | """) 75 | @Test 76 | fun sortByDateTest() { 77 | val firstMeme = repository.save(createDummyMeme(publishDateTime = LocalDateTime.of(2019, 12, 1, 10, 0))).block() 78 | val secondMeme = repository.save(createDummyMeme(publishDateTime = LocalDateTime.of(2019, 12, 2, 10, 0))).block() 79 | val thirdMeme = repository.save(createDummyMeme(publishDateTime = LocalDateTime.of(2019, 12, 3, 10, 0))).block() 80 | val emptyDateFirst = repository.save(createDummyMeme(publishDateTime = null)).block() 81 | val emptyDateSecond = repository.save(createDummyMeme(publishDateTime = null)).block() 82 | val emptyDateThird = repository.save(createDummyMeme(publishDateTime = null)).block() 83 | 84 | // -- 85 | val count = repository.countByLanguage(Language.GERMAN.code).block()!! 86 | val list = repository.findByLanguage(Language.GERMAN.code, PageRequest.of(0, 10, Sort.by(Sort.Direction.DESC, "publishDateTime", "id"))) 87 | .collectList() 88 | .block()!! 89 | 90 | // -- 91 | assertThat(count).isEqualTo(6) 92 | assertThat(list).containsExactly(thirdMeme, secondMeme, firstMeme, emptyDateThird, emptyDateSecond, emptyDateFirst) 93 | } 94 | 95 | @Test 96 | fun shouldExistsByLanguageAndFingerprint() { 97 | val fingerprint = "FINGERPRINT" 98 | val language = Language.GERMAN 99 | 100 | mongoTemplate.save(createDummyMeme(language = language, fingerprint = fingerprint)) 101 | 102 | // -- 103 | val result = repository.existsByLanguageAndFingerprint(language.code, fingerprint).block() 104 | 105 | // -- 106 | assertThat(result).isTrue() 107 | } 108 | 109 | @Test 110 | fun shouldNotExistsByLanguageAndFingerprintWhereLanguageIsDifferent() { 111 | val fingerprint = "FINGERPRINT" 112 | 113 | mongoTemplate.save(createDummyMeme(language = Language.GERMAN, fingerprint = fingerprint)) 114 | 115 | // -- 116 | val result = repository.existsByLanguageAndFingerprint(Language.RUSSIAN.code, fingerprint).block() 117 | 118 | // -- 119 | assertThat(result).isFalse() 120 | } 121 | 122 | @Test 123 | fun shouldNotExistsByLanguageAndFingerprintWhereFingerprintIsDifferent() { 124 | val language = Language.GERMAN 125 | 126 | mongoTemplate.save(createDummyMeme(language = language, fingerprint = "123")) 127 | 128 | // -- 129 | val result = repository.existsByLanguageAndFingerprint(language.code, "321").block() 130 | 131 | // -- 132 | assertThat(result).isFalse() 133 | } 134 | 135 | @Test 136 | fun findByPageUrls() { 137 | val urls = arrayListOf("http://debeste.de/meme-1", "http://debeste.de/meme-2", "http://debeste.de/meme-3") 138 | urls.forEach { 139 | mongoTemplate.save(createDummyMeme(pageUrl = it)) 140 | } 141 | 142 | // -- 143 | val result = repository.findByPageUrlIn(urls) 144 | .collectList() 145 | .block()!! 146 | 147 | // -- 148 | assertThat(result).hasSize(urls.size) 149 | assertThat(result.map { it.pageUrl }).containsExactlyElementsOf(urls) 150 | } 151 | 152 | } 153 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/repository/PageRepositoryTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.repository 2 | 3 | import me.ruslanys.ifunny.base.RedisRepositoryTests 4 | import me.ruslanys.ifunny.channel.DebesteChannel 5 | import org.assertj.core.api.Assertions.assertThat 6 | import org.junit.jupiter.api.BeforeEach 7 | import org.junit.jupiter.api.Test 8 | import java.time.Duration 9 | 10 | class PageRepositoryTests : RedisRepositoryTests() { 11 | 12 | private lateinit var repository: PageRepository 13 | 14 | @BeforeEach 15 | fun setUp() { 16 | repository = PageRepository(redisTemplate) 17 | redisTemplate.execute { 18 | it.serverCommands().flushAll() 19 | }.blockFirst() 20 | } 21 | 22 | @Test 23 | fun getCurrentShouldReturnFirst() { 24 | val channel = DebesteChannel() 25 | val currentPageNumber = repository.getCurrent(channel).block() 26 | assertThat(currentPageNumber).isEqualTo(1) 27 | } 28 | 29 | @Test 30 | fun getCurrentShouldReturnPersistedValue() { 31 | val channel = DebesteChannel() 32 | redisTemplate.opsForValue().set("$channel:current", 777).block() 33 | 34 | val pageNumber = repository.getCurrent(channel).block() 35 | 36 | assertThat(pageNumber).isEqualTo(777) 37 | } 38 | 39 | @Test 40 | fun incCurrentShouldReturnFirstByDefault() { 41 | val channel = DebesteChannel() 42 | 43 | val pageNumber = repository.incCurrent(channel).block() 44 | assertThat(pageNumber).isEqualTo(1) 45 | } 46 | 47 | @Test 48 | fun incCurrentShouldReturnSequence() { 49 | val channel = DebesteChannel() 50 | 51 | assertThat(repository.incCurrent(channel).block()).isEqualTo(1) 52 | assertThat(repository.incCurrent(channel).block()).isEqualTo(2) 53 | assertThat(repository.incCurrent(channel).block()).isEqualTo(3) 54 | } 55 | 56 | @Test 57 | fun incCurrentShouldReturnNext() { 58 | val channel = DebesteChannel() 59 | redisTemplate.opsForValue().set("$channel:current", 123).block() 60 | 61 | val pageNumber = repository.incCurrent(channel).block() 62 | 63 | assertThat(pageNumber).isEqualTo(124) 64 | } 65 | 66 | @Test 67 | fun getLastShouldReturnNullIfAbsent() { 68 | val last = repository.getLast(DebesteChannel()).block() 69 | assertThat(last).isNull() 70 | } 71 | 72 | @Test 73 | fun lastPageShouldExpire() { 74 | val channel = DebesteChannel() 75 | val duration = Duration.ofSeconds(2) 76 | 77 | repository.setLast(channel, 123, duration).block() 78 | 79 | // -- 80 | val last = repository.getLast(channel).block() 81 | assertThat(last).isEqualTo(123) 82 | 83 | // -- 84 | Thread.sleep(3_000) 85 | val lastExpired = repository.getLast(channel).block() 86 | assertThat(lastExpired).isNull() 87 | } 88 | 89 | @Test 90 | fun clearCurrentShouldRemoveKey() { 91 | val channel = DebesteChannel() 92 | val key = "$channel:current" 93 | 94 | redisTemplate.opsForValue().set(key, 123).block() 95 | assertThat(repository.getCurrent(channel).block()).isEqualTo(123) 96 | 97 | repository.clearCurrent(channel).block() 98 | assertThat(redisTemplate.opsForValue().get(key).block()).isNull() 99 | } 100 | 101 | 102 | } 103 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/service/DefaultMemeServiceTests.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.service 2 | 3 | import com.nhaarman.mockitokotlin2.any 4 | import com.nhaarman.mockitokotlin2.eq 5 | import com.nhaarman.mockitokotlin2.given 6 | import kotlinx.coroutines.runBlocking 7 | import me.ruslanys.ifunny.base.ServiceTests 8 | import me.ruslanys.ifunny.channel.DebesteChannel 9 | import me.ruslanys.ifunny.channel.MemeInfo 10 | import me.ruslanys.ifunny.controller.dto.PageRequest 11 | import me.ruslanys.ifunny.domain.Language 12 | import me.ruslanys.ifunny.domain.Meme 13 | import me.ruslanys.ifunny.exception.NotFoundException 14 | import me.ruslanys.ifunny.repository.MemeRepository 15 | import me.ruslanys.ifunny.util.createDummyFile 16 | import me.ruslanys.ifunny.util.createDummyMeme 17 | import org.assertj.core.api.Assertions.assertThat 18 | import org.junit.jupiter.api.Assertions.assertEquals 19 | import org.junit.jupiter.api.Assertions.assertThrows 20 | import org.junit.jupiter.api.BeforeEach 21 | import org.junit.jupiter.api.Test 22 | import org.mockito.Mock 23 | import reactor.core.publisher.Flux 24 | import reactor.core.publisher.Mono 25 | 26 | class DefaultMemeServiceTests : ServiceTests() { 27 | 28 | @Mock private lateinit var memeRepository: MemeRepository 29 | 30 | private lateinit var service: DefaultMemeService 31 | 32 | @BeforeEach 33 | fun setUp() { 34 | service = DefaultMemeService(memeRepository) 35 | } 36 | 37 | @Test 38 | fun addShouldReturnValueAfterSave() = runBlocking { 39 | val meme = createDummyMeme() 40 | val info = MemeInfo(pageUrl = "", title = "", originUrl = "") 41 | 42 | given(memeRepository.save(any())).willReturn(Mono.just(meme)) 43 | 44 | val result = service.add(DebesteChannel(), info, createDummyFile(), "fingerprint") 45 | 46 | assertThat(result).isEqualTo(meme) 47 | } 48 | 49 | @Test 50 | fun addShouldThrowExceptionWhenPageUrlIsNull() { 51 | assertThrows(IllegalArgumentException::class.java) { 52 | runBlocking { 53 | service.add(DebesteChannel(), MemeInfo(title = "", originUrl = "", pageUrl = null), createDummyFile(), null) 54 | } 55 | } 56 | } 57 | 58 | @Test 59 | fun addShouldThrowExceptionWhenTitleIsNull() { 60 | assertThrows(IllegalArgumentException::class.java) { 61 | runBlocking { 62 | service.add(DebesteChannel(), MemeInfo(title = null, originUrl = "", pageUrl = ""), createDummyFile(), null) 63 | } 64 | } 65 | } 66 | 67 | @Test 68 | fun addShouldThrowExceptionWhenOriginUrlIsNull() { 69 | assertThrows(IllegalArgumentException::class.java) { 70 | runBlocking { 71 | service.add(DebesteChannel(), MemeInfo(title = "", originUrl = null, pageUrl = ""), createDummyFile(), null) 72 | } 73 | } 74 | } 75 | 76 | @Test 77 | fun isExistsShouldReturnFalseWhenThereIsNoSuchDocument() = runBlocking { 78 | given(memeRepository.existsByLanguageAndFingerprint("de", "fingerprint")).willReturn(Mono.just(false)) 79 | 80 | val result = service.isExists(Language.GERMAN, "fingerprint") 81 | 82 | assertThat(result).isFalse() 83 | } 84 | 85 | @Test 86 | fun isExistsShouldReturnTrueWhenThereIsSuchDocument() = runBlocking { 87 | given(memeRepository.existsByLanguageAndFingerprint("de", "fingerprint")).willReturn(Mono.just(true)) 88 | 89 | val result = service.isExists(Language.GERMAN, "fingerprint") 90 | 91 | assertThat(result).isTrue() 92 | } 93 | 94 | @Test 95 | fun findByPageUrlsShouldReturnList() = runBlocking { 96 | val memes = arrayOf(createDummyMeme(), createDummyMeme(), createDummyMeme()) 97 | given(memeRepository.findByPageUrlIn(any())).willReturn(Flux.just(*memes)) 98 | 99 | val result = service.findByPageUrls(listOf()) 100 | 101 | assertThat(result).contains(*memes) 102 | } 103 | 104 | @Test 105 | fun getPageShouldReturnPageableResponse() = runBlocking { 106 | val memes = arrayOf(createDummyMeme(), createDummyMeme()) 107 | given(memeRepository.countByLanguage(Language.GERMAN.code)).willReturn(Mono.just(101)) 108 | given(memeRepository.findByLanguage(eq(Language.GERMAN.code), any())).willReturn(Flux.just(*memes)) 109 | 110 | val page = service.getPage(Language.GERMAN, PageRequest()) 111 | 112 | assertThat(page.totalElements).isEqualTo(101) 113 | assertThat(page.content).contains(*memes) 114 | } 115 | 116 | @Test 117 | fun getByIdShouldThrowNotFoundException() { 118 | given(memeRepository.findById(any())).willReturn(Mono.empty()) 119 | 120 | assertThrows(NotFoundException::class.java) { 121 | runBlocking { 122 | service.getById("123") 123 | } 124 | } 125 | } 126 | 127 | @Test 128 | fun getByIdShouldReturnValueFromRepository() = runBlocking { 129 | val meme = createDummyMeme() 130 | given(memeRepository.findById("321")).willReturn(Mono.just(meme)) 131 | 132 | val actual = service.getById("321") 133 | 134 | assertEquals(meme, actual) 135 | } 136 | 137 | } 138 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/util/DummyData.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.util 2 | 3 | import me.ruslanys.ifunny.domain.Language 4 | import me.ruslanys.ifunny.domain.Meme 5 | import me.ruslanys.ifunny.domain.S3File 6 | import org.bson.types.ObjectId 7 | import java.time.LocalDateTime 8 | import java.util.* 9 | import java.util.concurrent.ThreadLocalRandom 10 | 11 | fun createDummyMeme( 12 | language: Language = Language.GERMAN, 13 | channelName: String = "TestChannel", 14 | pageUrl: String = UUID.randomUUID().toString(), 15 | title: String = "Title", 16 | resourceUrl: String = "resourceUrl", 17 | file: S3File = createDummyFile(), 18 | fingerprint: String = "fingerprint", 19 | publishDateTime: LocalDateTime? = LocalDateTime.now(), 20 | author: String = "ruslanys", 21 | likes: Int = ThreadLocalRandom.current().nextInt(1, 100), 22 | comments: Int = ThreadLocalRandom.current().nextInt(1, 100), 23 | id: ObjectId = ObjectId() 24 | ) = Meme(language.code, channelName, pageUrl, title, resourceUrl, file, fingerprint, publishDateTime, author, likes, comments, id) 25 | 26 | fun createDummyFile( 27 | region: String = "region", 28 | bucket: String = "bucket", 29 | name: String = "picture.jpg", 30 | contentType: String = "image/jpg", 31 | checksum: String = "checksum", 32 | size: Long = 123L 33 | ): S3File = S3File(region, bucket, name, contentType, checksum, size) 34 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/util/IOUtils.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.util 2 | 3 | inline fun readResource(path: String): String = 4 | T::class.java.getResourceAsStream(path).bufferedReader().use { 5 | it.readText() 6 | } 7 | -------------------------------------------------------------------------------- /src/test/kotlin/me/ruslanys/ifunny/util/MockClient.kt: -------------------------------------------------------------------------------- 1 | package me.ruslanys.ifunny.util 2 | 3 | import com.nhaarman.mockitokotlin2.any 4 | import com.nhaarman.mockitokotlin2.given 5 | import com.nhaarman.mockitokotlin2.mock 6 | import org.springframework.core.ParameterizedTypeReference 7 | import org.springframework.core.io.ClassPathResource 8 | import org.springframework.http.HttpHeaders 9 | import org.springframework.web.reactive.function.client.ClientResponse 10 | import org.springframework.web.reactive.function.client.WebClient 11 | import reactor.core.publisher.Flux 12 | import reactor.core.publisher.Mono 13 | 14 | /** 15 | * Different options to mock the client: https://www.baeldung.com/spring-mocking-webclient 16 | */ 17 | 18 | fun mockGet(webClient: WebClient, path: String, response: String) { 19 | val get = mock>() 20 | val uri = mock>() 21 | val responseSpec = mock() 22 | 23 | given(webClient.get()).willReturn(get) 24 | given(get.uri(path)).willReturn(uri) 25 | given(uri.retrieve()).willReturn(responseSpec) 26 | given(responseSpec.bodyToMono(any>())).willReturn(Mono.just(response)) 27 | } 28 | 29 | fun mockGet(webClient: WebClient, path: String, resource: ClassPathResource) { 30 | val response = resource.inputStream.bufferedReader().use { 31 | it.readText() 32 | } 33 | mockGet(webClient, path, response) 34 | } 35 | 36 | fun mockGetBytes(webClient: WebClient, path: String, bytes: ByteArray) { 37 | val get = mock>() 38 | val uri = mock>() 39 | val response = mock() 40 | val headers = mock() 41 | val httpHeaders = HttpHeaders() 42 | 43 | given(webClient.get()).willReturn(get) 44 | given(get.uri(path)).willReturn(uri) 45 | given(uri.exchange()).willReturn(Mono.just(response)) 46 | given(response.headers()).willReturn(headers) 47 | given(headers.asHttpHeaders()).willReturn(httpHeaders) 48 | given(response.bodyToFlux(any>())).willReturn(Flux.just(bytes)) 49 | } 50 | 51 | fun mockGetBytes(webClient: WebClient, path: String, resource: ClassPathResource) { 52 | val bytes = resource.inputStream.buffered().use { 53 | it.readAllBytes() 54 | } 55 | mockGetBytes(webClient, path, bytes) 56 | } 57 | -------------------------------------------------------------------------------- /src/test/kotlin/phash/AverageHashTests.kt: -------------------------------------------------------------------------------- 1 | package phash 2 | 3 | import com.github.kilianB.hash.Hash 4 | import com.github.kilianB.hashAlgorithms.AverageHash 5 | import org.assertj.core.api.Assertions.assertThat 6 | import org.junit.jupiter.api.Disabled 7 | import org.junit.jupiter.api.Test 8 | import org.springframework.core.io.Resource 9 | import org.springframework.core.io.support.PathMatchingResourcePatternResolver 10 | 11 | @Disabled 12 | class AverageHashTests { 13 | 14 | private val hasher = AverageHash(64) 15 | 16 | @Test 17 | fun differentWhiteTextOnBlackBackground() { 18 | val hashedResources = hashedResources("dataset-1") 19 | val hashes = hashedResources.map { it.hash.hashValue.toString(16) } 20 | assertThat(hashes.toSet()).hasSize(hashedResources.size) 21 | } 22 | 23 | @Test 24 | fun theSamePictureWithDifferentWhiteLabels() { 25 | val hashedResources = hashedResources("dataset-2") 26 | val hashes = hashedResources.map { it.hash.hashValue.toString(16) } 27 | assertThat(hashes.toSet()).hasSize(1) 28 | } 29 | 30 | @Test 31 | fun theSameTextWithDifferentWhiteLabels() { 32 | val hashedResources = hashedResources("dataset-3") 33 | val hashes = hashedResources.map { it.hash.hashValue.toString(16) } 34 | assertThat(hashes.toSet()).hasSize(1) 35 | } 36 | 37 | @Test 38 | fun theSameGifWithDifferentWhiteLabels() { 39 | val hashedResources = hashedResources("dataset-4") 40 | val hashes = hashedResources.map { it.hash.hashValue.toString(16) } 41 | assertThat(hashes.toSet()).hasSize(1) 42 | } 43 | 44 | @Test 45 | fun differentYellowTextOnGrayBackground() { 46 | val hashedResources = hashedResources("dataset-5") 47 | val hashes = hashedResources.map { it.hash.hashValue.toString(16) } 48 | assertThat(hashes.toSet()).hasSize(hashedResources.size) 49 | } 50 | 51 | private fun hashedResources(path: String): List { 52 | val resources = RESOURCE_RESOLVER.getResources("$ROOT_DIRECTORY/$path/*") 53 | return resources.map { 54 | HashedResource(it, hasher.hash(it.file)) 55 | } 56 | } 57 | 58 | data class HashedResource(val resource: Resource, val hash: Hash) 59 | 60 | companion object { 61 | private const val ROOT_DIRECTORY = "phash" 62 | private val RESOURCE_RESOLVER = PathMatchingResourcePatternResolver() 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /src/test/kotlin/phash/CompoundHashTests.kt: -------------------------------------------------------------------------------- 1 | package phash 2 | 3 | import com.github.kilianB.hash.Hash 4 | import com.github.kilianB.hashAlgorithms.AverageColorHash 5 | import com.github.kilianB.hashAlgorithms.WaveletHash 6 | import org.assertj.core.api.Assertions.assertThat 7 | import org.junit.jupiter.api.Disabled 8 | import org.junit.jupiter.api.Test 9 | import org.springframework.core.io.Resource 10 | import org.springframework.core.io.support.PathMatchingResourcePatternResolver 11 | 12 | @Disabled 13 | class CompoundHashTests { 14 | 15 | private val fastHasher = WaveletHash(32, 5) 16 | private val accurateHasher = AverageColorHash(128) 17 | 18 | private val similarityThreshold: Double = 0.2 19 | 20 | 21 | @Test 22 | fun differentWhiteTextOnBlackBackground() { 23 | assertNotEqualsHashes("dataset-1") 24 | } 25 | 26 | @Test 27 | fun theSamePictureWithDifferentWhiteLabels() { 28 | assertEqualsHashes("dataset-2") 29 | } 30 | 31 | @Test 32 | fun theSameTextWithDifferentWhiteLabels() { 33 | assertEqualsHashes("dataset-3") 34 | } 35 | 36 | @Test 37 | fun theSameGifWithDifferentWhiteLabels() { 38 | assertEqualsHashes("dataset-4") 39 | } 40 | 41 | @Test 42 | fun differentYellowTextOnGrayBackground() { 43 | assertNotEqualsHashes("dataset-5") 44 | } 45 | 46 | private fun assertEqualsHashes(path: String) { 47 | val hashedResources = hashedResources(path) 48 | val first = hashedResources.first() 49 | val distances = distances(hashedResources) 50 | 51 | assertThat(hashedResources).allMatch { it.fastHash == first.fastHash } 52 | assertThat(distances).allMatch { it < similarityThreshold } 53 | } 54 | 55 | private fun assertNotEqualsHashes(path: String) { 56 | val hashedResources = hashedResources(path) 57 | val first = hashedResources.first() 58 | val distances = distances(hashedResources) 59 | 60 | assertThat(hashedResources).anyMatch { it.fastHash != first.fastHash } 61 | assertThat(distances).allMatch { it > similarityThreshold } 62 | } 63 | 64 | private fun hashedResources(path: String): List { 65 | val resources = RESOURCE_RESOLVER.getResources("$ROOT_DIRECTORY/$path/*") 66 | return resources.map { 67 | HashedResource(it, fastHasher.hash(it.file), accurateHasher.hash(it.file)) 68 | } 69 | } 70 | 71 | private fun distances(hashedResources: List): List { 72 | val distances = arrayListOf() 73 | for (i in hashedResources.indices) { 74 | for (j in i + 1 until hashedResources.size) { 75 | val distance = hashedResources[i].accurateHash.normalizedHammingDistance(hashedResources[j].accurateHash) 76 | distances.add(distance) 77 | } 78 | } 79 | return distances 80 | } 81 | 82 | 83 | data class HashedResource(val resource: Resource, val fastHash: Hash, val accurateHash: Hash) 84 | 85 | companion object { 86 | private const val ROOT_DIRECTORY = "phash" 87 | private val RESOURCE_RESOLVER = PathMatchingResourcePatternResolver() 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /src/test/resources/me/ruslanys/ifunny/channel/funpot/meme_gif.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/me/ruslanys/ifunny/channel/funpot/meme_gif.html -------------------------------------------------------------------------------- /src/test/resources/me/ruslanys/ifunny/channel/funpot/meme_picture.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/me/ruslanys/ifunny/channel/funpot/meme_picture.html -------------------------------------------------------------------------------- /src/test/resources/me/ruslanys/ifunny/channel/funpot/meme_video.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/me/ruslanys/ifunny/channel/funpot/meme_video.html -------------------------------------------------------------------------------- /src/test/resources/me/ruslanys/ifunny/channel/funpot/meme_video_Direct.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/me/ruslanys/ifunny/channel/funpot/meme_video_Direct.html -------------------------------------------------------------------------------- /src/test/resources/me/ruslanys/ifunny/channel/funpot/page.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/me/ruslanys/ifunny/channel/funpot/page.html -------------------------------------------------------------------------------- /src/test/resources/me/ruslanys/ifunny/channel/funpot/page_last.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/me/ruslanys/ifunny/channel/funpot/page_last.html -------------------------------------------------------------------------------- /src/test/resources/me/ruslanys/ifunny/channel/fuoriditesta-video/frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Il cavallo della bambina decide di giocare nel fango 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 30 | 31 | -------------------------------------------------------------------------------- /src/test/resources/me/ruslanys/ifunny/controller/api/feed_getById.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "5e107cc18b12145d4624f23c", 3 | "language": "de", 4 | "channelName": "TestChannel", 5 | "title": "Title", 6 | "url": "https://bucket.s3.region.amazonaws.com/picture.jpg" 7 | } 8 | -------------------------------------------------------------------------------- /src/test/resources/me/ruslanys/ifunny/controller/api/feed_getPage.json: -------------------------------------------------------------------------------- 1 | { 2 | "totalCount": 3, 3 | "list": [ 4 | { 5 | "id": "5e107cc18b12145d4624f23c", 6 | "language": "de", 7 | "channelName": "TestChannel", 8 | "title": "Title 1", 9 | "url": "https://bucket.s3.region.amazonaws.com/1.jpg" 10 | }, 11 | { 12 | "id": "5e107cc18b12145d4624f23d", 13 | "language": "de", 14 | "channelName": "TestChannel", 15 | "title": "Title 2", 16 | "url": "https://bucket.s3.region.amazonaws.com/2.jpg" 17 | }, 18 | { 19 | "id": "5e107cc18b12145d4624f23e", 20 | "language": "de", 21 | "channelName": "TestChannel", 22 | "title": "Title 3", 23 | "url": "https://bucket.s3.region.amazonaws.com/3.jpg" 24 | } 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /src/test/resources/me/ruslanys/ifunny/grab/picture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/me/ruslanys/ifunny/grab/picture.jpg -------------------------------------------------------------------------------- /src/test/resources/phash/dataset-1/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/phash/dataset-1/1.jpg -------------------------------------------------------------------------------- /src/test/resources/phash/dataset-1/2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/phash/dataset-1/2.jpg -------------------------------------------------------------------------------- /src/test/resources/phash/dataset-1/3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/phash/dataset-1/3.jpg -------------------------------------------------------------------------------- /src/test/resources/phash/dataset-1/4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/phash/dataset-1/4.jpg -------------------------------------------------------------------------------- /src/test/resources/phash/dataset-1/5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/phash/dataset-1/5.jpg -------------------------------------------------------------------------------- /src/test/resources/phash/dataset-2/debeste.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/phash/dataset-2/debeste.jpg -------------------------------------------------------------------------------- /src/test/resources/phash/dataset-2/funpot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/phash/dataset-2/funpot.jpg -------------------------------------------------------------------------------- /src/test/resources/phash/dataset-2/funpot_smaller.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/phash/dataset-2/funpot_smaller.jpg -------------------------------------------------------------------------------- /src/test/resources/phash/dataset-3/debeste.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/phash/dataset-3/debeste.jpg -------------------------------------------------------------------------------- /src/test/resources/phash/dataset-3/debeste_smaller.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/phash/dataset-3/debeste_smaller.jpg -------------------------------------------------------------------------------- /src/test/resources/phash/dataset-3/funpot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/phash/dataset-3/funpot.jpg -------------------------------------------------------------------------------- /src/test/resources/phash/dataset-4/debeste.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/phash/dataset-4/debeste.gif -------------------------------------------------------------------------------- /src/test/resources/phash/dataset-4/funpot.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/phash/dataset-4/funpot.gif -------------------------------------------------------------------------------- /src/test/resources/phash/dataset-5/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/phash/dataset-5/1.jpg -------------------------------------------------------------------------------- /src/test/resources/phash/dataset-5/2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/phash/dataset-5/2.jpg -------------------------------------------------------------------------------- /src/test/resources/phash/dataset-5/3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/phash/dataset-5/3.jpg -------------------------------------------------------------------------------- /src/test/resources/phash/dataset-5/4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/phash/dataset-5/4.jpg -------------------------------------------------------------------------------- /src/test/resources/phash/dataset-5/5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruslanys/ifunny/baf3948d7092a3cc2fc39b411dfaec2ab9456be1/src/test/resources/phash/dataset-5/5.jpg --------------------------------------------------------------------------------