├── .gitignore ├── .mvn └── wrapper │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── LICENSE ├── README.md ├── Vagrantfile ├── default.conf ├── documentation ├── Arhitecture.png └── demo.gif ├── mvnw ├── pom.xml ├── provision-vagrant.yml ├── requirements.yml └── src ├── main ├── java │ └── com │ │ └── rhcloud │ │ └── analytics4github │ │ ├── Application.java │ │ ├── config │ │ ├── AddOAuthTokenInterceptor.java │ │ ├── ChangeAcceptTypeForStargazersInterceptor.java │ │ ├── CorsSupportConfig.java │ │ ├── FiltersConfiguration.java │ │ ├── GitHubApiEndpoints.java │ │ ├── MondoDbOpenshiftConfig.java │ │ ├── RestTemplateConfig.java │ │ └── SwaggerConfig.java │ │ ├── controller │ │ ├── CommitsController.java │ │ ├── GitHubTrendingController.java │ │ ├── NumberRequestsLeftController.java │ │ ├── StargazersController.java │ │ ├── StatisticsController.java │ │ └── UniqueContributorsController.java │ │ ├── domain │ │ ├── Author.java │ │ └── RequestToAPI.java │ │ ├── dto │ │ ├── RequestFromFrontendDto.java │ │ └── ResponceForFrontendDto.java │ │ ├── exception │ │ ├── GitHubRESTApiException.java │ │ ├── GlobalDefaultExceptionHandler.java │ │ └── TrendingException.java │ │ ├── repository │ │ └── RequestToApiRepository.java │ │ ├── service │ │ ├── CommitsService.java │ │ ├── GitHubTrendingService.java │ │ ├── StargazersService.java │ │ ├── StatisticsService.java │ │ └── UniqueContributorsService.java │ │ └── util │ │ ├── GitHubApiIterator.java │ │ └── Utils.java └── resources │ ├── META-INF │ └── additional-spring-configuration-metadata.json │ ├── application-dev.properties │ ├── application-docker.properties │ ├── application-openshift.properties │ ├── application.properties │ ├── mockWeekStargazers.json │ ├── monthStargazersFrequencyForHighChart.json │ └── static │ ├── bootstrap.min.css │ ├── custom.css │ ├── customScript.js │ ├── googleAnalitics.js │ ├── images │ ├── arrow-left.svg │ ├── arrow-right.svg │ ├── favicon.png │ ├── git-commit.svg │ ├── mark-github.svg │ ├── organization.svg │ ├── repo.svg │ ├── social-media.png │ └── star.svg │ └── index.html └── test ├── java └── com │ └── rhcloud │ └── analytics4github │ ├── ApplicationTest.java │ ├── TestApplicationContext.java │ ├── config │ ├── FiltersConfigurationTest.java │ └── InterceptorsIntegrationalTest.java │ ├── controller │ ├── CommitsControllerTest.java │ ├── GitHubTrendingControllerTest.java │ ├── StargazersControllerTest.java │ ├── StatisticsControllerTest.java │ └── UniqueContributorsControllerTest.java │ ├── repository │ └── RequestToApiRepositoryTest.java │ ├── service │ ├── CommitsServiceTest.java │ ├── GitHubTrendingServiceTest.java │ ├── StargazersServiceTest.java │ ├── StatisticsServiceTest.java │ └── UniqueContributorsServiceTest.java │ └── util │ ├── GitHubApiIteratorTest.java │ └── UtilsTest.java └── resources ├── AuthorsForTest.txt ├── RepositoriesForTest.txt ├── application-test.properties ├── monthStargazersFrequencyForHighChart.json ├── monthStargazersFrequencyMap.ser ├── stargazersPage1.json ├── weekStargazersFrequencyForHIghChart.json └── weekStargazersFrequencyMap.ser /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .gradle 3 | build 4 | classes 5 | token.txt 6 | *.iml 7 | target 8 | .vagrant 9 | *.retry -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LyashenkoGS/analytics4github/ddb2dd22e78675efb2c023c58188784063bbf1f3/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.2/apache-maven-3.5.2-bin.zip -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Grigoriy Lyashenko, https://www.linkedin.com/in/lyashenkogs/ 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # analytics4github 2 | [![Build Status](http://195.211.154.179:8081/view/analytics4github/job/master-branch-poling%20and%20redeploy/badge/icon)](http://195.211.154.179:8081/view/analytics4github/job/master-branch-poling%20and%20redeploy/) 3 | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/d3a472531c4b46749c7eda1439d746db)](https://www.codacy.com/app/lyashenkogs/analytics4github?utm_source=github.com&utm_medium=referral&utm_content=LyashenkoGS/analytics4github&utm_campaign=Badge_Grade) 4 | [![codecov](https://codecov.io/gh/LyashenkoGS/analytics4github/branch/master/graph/badge.svg)](https://codecov.io/gh/LyashenkoGS/analytics4github) 5 | [![GitHub license](https://img.shields.io/github/license/mashape/apistatus.svg)](https://github.com/LyashenkoGS/analytics4github/blob/master/LICENCE) 6 | 7 | 8 | [http://195.211.154.179](http://195.211.154.179) 9 | Java web application to enhance github.com search mechanism. 10 | Provided options facilitate search of new perspective projects Preview changes 11 | with good commits/stars/contributors grown. 12 | Internally, the existed GitHub API is using. 13 | To access a REST API documentation - run the application and access 14 | [http://195.211.154.179/swagger-ui.html](http://195.211.154.179/swagger-ui.html) 15 | 16 | ![Demo](./documentation/demo.gif) 17 | 18 | 19 | ## Prerequisites 20 | 21 | * JDK 1.8 22 | * access to [GitHub REST API ](https://developer.github.com/v3/) 23 | * access to [GitHub trending page](https://github.com/trending) 24 | * Generate OAuth token for GitHub [https://github.com/settings/tokens](https://github.com/settings/tokens) and copy it to /var/token.txt or export to an environment variable GITHUB_TOKEN. 25 | * MongoDB 3.6 (docker run --name someMongo -p 27017:27017 mongo:3.6.0-jessie) 26 | 27 | ## Deployment 28 | To run locally execute 29 | 30 | ./mvnw install -Dmaven.test.skip=true 31 | java -jar target/analytics4github*.jar 32 | 33 | accessible by default at http://127.0.0.1:8080 34 | 35 | 36 | ## Development 37 | ![architecture](./documentation/Arhitecture.png) 38 | 39 | Tests stopped to work properly via maven but you can run them via Intellij 40 | ##### IntellijIdea 41 | 42 | ###### reload backend 43 | To reload controllers after editing - press ctl + f9 and wait till application restart. 44 | It'll execute "Make" and trigger hot-redeploy via spring-boot-devtools. 45 | 46 | ### working with frontend 47 | adjust application.properties to use static resources from a file system, not a jar file. It will simplify frontend 48 | development 49 | 50 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure(2) do |config| 5 | 6 | config.vm.box = "centos/7" 7 | 8 | # Create a private network, which allows host-only access to the machine using a specific IP. 9 | config.vm.network "private_network", ip: "192.168.10.10" 10 | 11 | config.vm.provision "ansible" do |ansible| 12 | ansible.playbook = "provision-vagrant.yml" 13 | ansible.verbose = "vv" 14 | end 15 | 16 | end 17 | -------------------------------------------------------------------------------- /default.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80; 3 | 4 | # reverse proxying all requests from localhost:80 to localhost:8080 5 | location / { 6 | proxy_pass http://127.0.0.1:8080; 7 | proxy_set_header Host $host; 8 | proxy_set_header X-Real-IP $remote_addr; 9 | } 10 | 11 | # redirect server error pages to the static page /50x.html 12 | error_page 500 502 503 504 /50x.html; 13 | location = /50x.html { 14 | root /usr/share/nginx/html; 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /documentation/Arhitecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LyashenkoGS/analytics4github/ddb2dd22e78675efb2c023c58188784063bbf1f3/documentation/Arhitecture.png -------------------------------------------------------------------------------- /documentation/demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LyashenkoGS/analytics4github/ddb2dd22e78675efb2c023c58188784063bbf1f3/documentation/demo.gif -------------------------------------------------------------------------------- /mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home 58 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html 59 | if [ -z "$JAVA_HOME" ]; then 60 | if [ -x "/usr/libexec/java_home" ]; then 61 | export JAVA_HOME="`/usr/libexec/java_home`" 62 | else 63 | export JAVA_HOME="/Library/Java/Home" 64 | fi 65 | fi 66 | ;; 67 | esac 68 | 69 | if [ -z "$JAVA_HOME" ] ; then 70 | if [ -r /etc/gentoo-release ] ; then 71 | JAVA_HOME=`java-config --jre-home` 72 | fi 73 | fi 74 | 75 | if [ -z "$M2_HOME" ] ; then 76 | ## resolve links - $0 may be a link to maven's home 77 | PRG="$0" 78 | 79 | # need this for relative symlinks 80 | while [ -h "$PRG" ] ; do 81 | ls=`ls -ld "$PRG"` 82 | link=`expr "$ls" : '.*-> \(.*\)$'` 83 | if expr "$link" : '/.*' > /dev/null; then 84 | PRG="$link" 85 | else 86 | PRG="`dirname "$PRG"`/$link" 87 | fi 88 | done 89 | 90 | saveddir=`pwd` 91 | 92 | M2_HOME=`dirname "$PRG"`/.. 93 | 94 | # make it fully qualified 95 | M2_HOME=`cd "$M2_HOME" && pwd` 96 | 97 | cd "$saveddir" 98 | # echo Using m2 at $M2_HOME 99 | fi 100 | 101 | # For Cygwin, ensure paths are in UNIX format before anything is touched 102 | if $cygwin ; then 103 | [ -n "$M2_HOME" ] && 104 | M2_HOME=`cygpath --unix "$M2_HOME"` 105 | [ -n "$JAVA_HOME" ] && 106 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 107 | [ -n "$CLASSPATH" ] && 108 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 109 | fi 110 | 111 | # For Mingw, ensure paths are in UNIX format before anything is touched 112 | if $mingw ; then 113 | [ -n "$M2_HOME" ] && 114 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 115 | [ -n "$JAVA_HOME" ] && 116 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 117 | # TODO classpath? 118 | fi 119 | 120 | if [ -z "$JAVA_HOME" ]; then 121 | javaExecutable="`which javac`" 122 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 123 | # readlink(1) is not available as standard on Solaris 10. 124 | readLink=`which readlink` 125 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 126 | if $darwin ; then 127 | javaHome="`dirname \"$javaExecutable\"`" 128 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 129 | else 130 | javaExecutable="`readlink -f \"$javaExecutable\"`" 131 | fi 132 | javaHome="`dirname \"$javaExecutable\"`" 133 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 134 | JAVA_HOME="$javaHome" 135 | export JAVA_HOME 136 | fi 137 | fi 138 | fi 139 | 140 | if [ -z "$JAVACMD" ] ; then 141 | if [ -n "$JAVA_HOME" ] ; then 142 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 143 | # IBM's JDK on AIX uses strange locations for the executables 144 | JAVACMD="$JAVA_HOME/jre/sh/java" 145 | else 146 | JAVACMD="$JAVA_HOME/bin/java" 147 | fi 148 | else 149 | JAVACMD="`which java`" 150 | fi 151 | fi 152 | 153 | if [ ! -x "$JAVACMD" ] ; then 154 | echo "Error: JAVA_HOME is not defined correctly." >&2 155 | echo " We cannot execute $JAVACMD" >&2 156 | exit 1 157 | fi 158 | 159 | if [ -z "$JAVA_HOME" ] ; then 160 | echo "Warning: JAVA_HOME environment variable is not set." 161 | fi 162 | 163 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 164 | 165 | # traverses directory structure from process work directory to filesystem root 166 | # first directory with .mvn subdirectory is considered project base directory 167 | find_maven_basedir() { 168 | 169 | if [ -z "$1" ] 170 | then 171 | echo "Path not specified to find_maven_basedir" 172 | return 1 173 | fi 174 | 175 | basedir="$1" 176 | wdir="$1" 177 | while [ "$wdir" != '/' ] ; do 178 | if [ -d "$wdir"/.mvn ] ; then 179 | basedir=$wdir 180 | break 181 | fi 182 | # workaround for JBEAP-8937 (on Solaris 10/Sparc) 183 | if [ -d "${wdir}" ]; then 184 | wdir=`cd "$wdir/.."; pwd` 185 | fi 186 | # end of workaround 187 | done 188 | echo "${basedir}" 189 | } 190 | 191 | # concatenates all lines of a file 192 | concat_lines() { 193 | if [ -f "$1" ]; then 194 | echo "$(tr -s '\n' ' ' < "$1")" 195 | fi 196 | } 197 | 198 | BASE_DIR=`find_maven_basedir "$(pwd)"` 199 | if [ -z "$BASE_DIR" ]; then 200 | exit 1; 201 | fi 202 | 203 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} 204 | if [ "$MVNW_VERBOSE" = true ]; then 205 | echo $MAVEN_PROJECTBASEDIR 206 | fi 207 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 208 | 209 | # For Cygwin, switch paths to Windows format before running java 210 | if $cygwin; then 211 | [ -n "$M2_HOME" ] && 212 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 213 | [ -n "$JAVA_HOME" ] && 214 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 215 | [ -n "$CLASSPATH" ] && 216 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 217 | [ -n "$MAVEN_PROJECTBASEDIR" ] && 218 | MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` 219 | fi 220 | 221 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 222 | 223 | exec "$JAVACMD" \ 224 | $MAVEN_OPTS \ 225 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 226 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 227 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" 228 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | analytics4github 6 | analytics4github 7 | 1.0-SNAPSHOT 8 | jar 9 | 10 | analytics4github 11 | 12 | 13 | org.springframework.boot 14 | spring-boot-starter-parent 15 | 1.5.9.RELEASE 16 | 17 | 18 | 19 | UTF-8 20 | UTF-8 21 | 1.8 22 | 23 | 24 | 25 | 26 | junit 27 | junit 28 | 4.12 29 | test 30 | 31 | 32 | org.springframework.boot 33 | spring-boot-starter-data-mongodb 34 | 35 | 36 | org.springframework.boot 37 | spring-boot-starter-web 38 | 39 | 40 | org.springframework.boot 41 | spring-boot-devtools 42 | 43 | 44 | org.springframework.boot 45 | spring-boot-starter-undertow 46 | 47 | 48 | org.springframework.boot 49 | spring-boot-starter-test 50 | test 51 | 52 | 53 | de.flapdoodle.embed 54 | de.flapdoodle.embed.mongo 55 | 1.50.5 56 | test 57 | 58 | 59 | io.springfox 60 | springfox-swagger2 61 | 2.4.0 62 | 63 | 64 | io.springfox 65 | springfox-swagger-ui 66 | 2.4.0 67 | 68 | 69 | org.jsoup 70 | jsoup 71 | 1.8.3 72 | 73 | 74 | 75 | 76 | 77 | 78 | org.springframework.boot 79 | spring-boot-maven-plugin 80 | 81 | 82 | 83 | -------------------------------------------------------------------------------- /provision-vagrant.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # contains tasks to install: Nginx, MongoDB 3 | - hosts: all 4 | become: yes 5 | # install and setup Nginx as a reverse proxy 6 | roles: 7 | - role: geerlingguy.nginx 8 | tasks: 9 | - name: Allow http acces to applications 10 | shell: "sudo setsebool httpd_can_network_connect 1 -P" 11 | - service: 12 | name: nginx 13 | state: stopped 14 | - copy: 15 | src: default.conf 16 | dest: /etc/nginx/conf.d/default.conf 17 | owner: nginx 18 | group: nginx 19 | mode: 0644 20 | - service: 21 | name: nginx 22 | state: started 23 | # download JDK https://stackoverflow.com/questions/10268583/downloading-java-jdk-on-linux-via-wget-is-shown-license-page-instead 24 | # This takes about a minute, doesn't have a progress indicator and may become broken anytime thanks to Oracle 25 | # - name: install wget 26 | # yum: 27 | # name: wget 28 | # state: present 29 | # - name: download JDK 30 | # shell: 'wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/8u151-b12/e758a0de34e24606bca991d704f6dcbf/jdk-8u151-linux-x64.rpm"' 31 | # - name: install JDK 32 | # shell: 'sudo yum localinstall -y jdk-8u151-linux-x64.rpm' 33 | # Install and setup MongoDB 34 | - name: Add mongodb repo 35 | yum_repository: 36 | name: mongodb-org-3.6 37 | description: MongoDB 3.6 Repository 38 | baseurl: https://repo.mongodb.org/yum/redhat/$releasever/mongodb-org/3.6/x86_64/ 39 | gpgkey: https://www.mongodb.org/static/pgp/server-3.6.asc 40 | - name: install mongoDB 41 | yum: 42 | name: mongodb-org 43 | state: present 44 | - name: Creates a folder for mongodb 45 | file: 46 | path: /data/db 47 | state: directory 48 | owner: mongod 49 | group: mongod 50 | mode: 0775 51 | - service: 52 | name: mongod 53 | state: started 54 | enabled: yes 55 | -------------------------------------------------------------------------------- /requirements.yml: -------------------------------------------------------------------------------- 1 | - src: robdyke.maven 2 | - src: geerlingguy.nginx -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/Application.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github; 2 | 3 | import com.rhcloud.analytics4github.exception.TrendingException; 4 | import com.rhcloud.analytics4github.service.GitHubTrendingService; 5 | import com.rhcloud.analytics4github.util.GitHubApiIterator; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.boot.SpringApplication; 10 | import org.springframework.boot.autoconfigure.SpringBootApplication; 11 | import org.springframework.scheduling.annotation.EnableScheduling; 12 | import org.springframework.web.client.RestTemplate; 13 | 14 | import javax.annotation.PostConstruct; 15 | 16 | 17 | @SpringBootApplication 18 | @EnableScheduling 19 | public class Application { 20 | 21 | private static Logger LOG = LoggerFactory.getLogger(Application.class); 22 | 23 | @Autowired 24 | private GitHubTrendingService gitHubTrendingService; 25 | 26 | @Autowired 27 | private RestTemplate restTemplate; 28 | 29 | public static void main(String[] args) { 30 | SpringApplication.run(Application.class, args); 31 | } 32 | 33 | /** 34 | * with start application shows requests number left and parses top trending repositories 35 | */ 36 | @PostConstruct 37 | public void initIt() { 38 | try { 39 | GitHubApiIterator.initializeRequestsLeft(restTemplate); 40 | gitHubTrendingService.parseTrendingReposWebPage(); 41 | } catch (TrendingException exception) { 42 | LOG.error(String.valueOf(exception)); 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/config/AddOAuthTokenInterceptor.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.config; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.beans.factory.annotation.Value; 7 | import org.springframework.http.HttpHeaders; 8 | import org.springframework.http.HttpRequest; 9 | import org.springframework.http.client.ClientHttpRequestExecution; 10 | import org.springframework.http.client.ClientHttpRequestInterceptor; 11 | import org.springframework.http.client.ClientHttpResponse; 12 | import org.springframework.stereotype.Component; 13 | 14 | import java.io.BufferedReader; 15 | import java.io.FileInputStream; 16 | import java.io.IOException; 17 | import java.io.InputStreamReader; 18 | 19 | /** 20 | * Add OAuth token to all RestTemplate initialized by this class 21 | * 22 | * @author lyashenkogs 23 | */ 24 | @Component 25 | public class AddOAuthTokenInterceptor implements ClientHttpRequestInterceptor { 26 | private static Logger LOG = LoggerFactory.getLogger(AddOAuthTokenInterceptor.class); 27 | 28 | private String tokenValue = ""; 29 | 30 | @Autowired 31 | public AddOAuthTokenInterceptor(@Value("${token.file}") String tokenFile) { 32 | //Todo replace with NIO features; Getting token procedure should be execute once during the context load 33 | try (FileInputStream fstream = new FileInputStream(tokenFile); 34 | BufferedReader br = new BufferedReader(new InputStreamReader(fstream))) { 35 | this.tokenValue = br.readLine(); 36 | } catch (IOException e) { 37 | e.printStackTrace(); 38 | } 39 | 40 | LOG.debug("token value: " + tokenValue); 41 | if (tokenValue.isEmpty()) { 42 | LOG.warn("token value from " + tokenFile + " is empty"); 43 | LOG.warn("cant get token from a file, trying to get from the environment variable GITHUB_TOKEN"); 44 | this.tokenValue = System.getenv("GITHUB_TOKEN"); 45 | LOG.debug("token value: " + tokenValue); 46 | } 47 | } 48 | 49 | @Override 50 | public ClientHttpResponse intercept(HttpRequest request, byte[] body, 51 | ClientHttpRequestExecution execution) throws IOException { 52 | 53 | HttpHeaders headers = request.getHeaders(); 54 | headers.add("Authorization", "token " + tokenValue); 55 | return execution.execute(request, body); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/config/ChangeAcceptTypeForStargazersInterceptor.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.config; 2 | 3 | import org.springframework.http.HttpHeaders; 4 | import org.springframework.http.HttpRequest; 5 | import org.springframework.http.client.ClientHttpRequestExecution; 6 | import org.springframework.http.client.ClientHttpRequestInterceptor; 7 | import org.springframework.http.client.ClientHttpResponse; 8 | import org.springframework.stereotype.Component; 9 | 10 | import java.io.IOException; 11 | 12 | /** 13 | * Interceptor to change request header for Github API 14 | * to include timestamp in response for GET /repos/:owner/:repo/stargazers endpoint 15 | * 16 | * @author lyashenkogs 17 | * @see https://developer.github.com/v3/activity/starring/ 18 | */ 19 | @Component 20 | public class ChangeAcceptTypeForStargazersInterceptor implements ClientHttpRequestInterceptor { 21 | 22 | @Override 23 | public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException { 24 | HttpHeaders headers = request.getHeaders(); 25 | headers.add("Accept", "application/vnd.github.v3.star+json"); 26 | return execution.execute(request, body); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/config/CorsSupportConfig.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.config; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.web.servlet.config.annotation.CorsRegistry; 6 | import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; 7 | import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter; 8 | 9 | /** 10 | * Allow the application to proceed request from another domains 11 | * 12 | * @author lyashenkogs. 13 | */ 14 | @Configuration 15 | public class CorsSupportConfig { 16 | 17 | @Bean 18 | public WebMvcConfigurer corsConfigurer() { 19 | return new WebMvcConfigurerAdapter() { 20 | @Override 21 | public void addCorsMappings(CorsRegistry registry) { 22 | registry.addMapping("/**").allowedOrigins("*"); 23 | } 24 | }; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/config/FiltersConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.config; 2 | 3 | import com.rhcloud.analytics4github.domain.RequestToAPI; 4 | import com.rhcloud.analytics4github.repository.RequestToApiRepository; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.boot.web.servlet.FilterRegistrationBean; 9 | import org.springframework.context.annotation.Bean; 10 | import org.springframework.context.annotation.Configuration; 11 | 12 | import javax.servlet.*; 13 | import javax.servlet.http.Cookie; 14 | import javax.servlet.http.HttpServletRequest; 15 | import javax.servlet.http.HttpServletResponse; 16 | import java.io.IOException; 17 | import java.util.Arrays; 18 | import java.util.Optional; 19 | 20 | /** 21 | * @author lyashenkogs. 22 | */ 23 | @Configuration 24 | public class FiltersConfiguration { 25 | private static Logger LOG = LoggerFactory.getLogger(FiltersConfiguration.class); 26 | 27 | static final String FREE_REQUESTS_NUMBER_PER_NEW_USER = "20"; 28 | static final String FREE_REQUESTS_COOKIE_NAME = "freeRequests"; 29 | private static final int COOKIE_MAX_AGE = 24 * 60 * 60; 30 | 31 | @Autowired 32 | private RequestToApiRepository requestToApiRepository; 33 | 34 | @Bean 35 | public FilterRegistrationBean requestsStatisticsRegistrationBean() { 36 | FilterRegistrationBean bean = new FilterRegistrationBean(); 37 | bean.setName("requestsStatisticsRegistrationBean");//need to set name to avoid conflicts with other FilterRegistrationBeans' 38 | bean.setFilter(requestsStatisticAggregatorFilter()); 39 | bean.addUrlPatterns("/commitsPerMonth"); 40 | bean.addUrlPatterns("/stargazersPerMonth"); 41 | bean.addUrlPatterns("/uniqueContributorsPerMonth"); 42 | return bean; 43 | } 44 | 45 | public Filter requestsStatisticAggregatorFilter() { 46 | return new Filter() { 47 | @Override 48 | public void init(FilterConfig filterConfig) throws ServletException { 49 | } 50 | 51 | @Override 52 | public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { 53 | if (request instanceof HttpServletRequest) { 54 | StringBuffer requestURL = ((HttpServletRequest) request).getRequestURL(); 55 | String url = requestURL.toString(); 56 | String queryString = ((HttpServletRequest) request).getQueryString(); 57 | LOG.info(url + "?" + queryString); 58 | RequestToAPI interceptedRequest = new RequestToAPI(request.getParameter("projectName"), url); 59 | LOG.info(interceptedRequest.toString()); 60 | try { 61 | requestToApiRepository.save(interceptedRequest); 62 | } catch (Exception ex) { 63 | //TODO: implement notification and failover policy 64 | LOG.error("CANT SAVE STATISTIC DO DB !!"); 65 | ex.printStackTrace(); 66 | } 67 | } 68 | chain.doFilter(request, response); 69 | } 70 | 71 | @Override 72 | public void destroy() { 73 | } 74 | }; 75 | } 76 | 77 | @Bean 78 | public FilterRegistrationBean userRequestsLimitationRegistrationBean() { 79 | FilterRegistrationBean filterRequestsLimitationBean = new FilterRegistrationBean(); 80 | filterRequestsLimitationBean.setFilter(userRequestsLimitationFilter()); 81 | filterRequestsLimitationBean.setName("userRequestsLimitationRegistrationBean");//need to set name to avoid conflicts with other FilterRegistrationBeans' 82 | filterRequestsLimitationBean.addUrlPatterns("/commits"); 83 | filterRequestsLimitationBean.addUrlPatterns("/stargazers"); 84 | filterRequestsLimitationBean.addUrlPatterns("/uniqueContributors"); 85 | return filterRequestsLimitationBean; 86 | } 87 | 88 | public Filter userRequestsLimitationFilter() { 89 | return new Filter() { 90 | @Override 91 | public void init(FilterConfig filterConfig) throws ServletException { 92 | 93 | } 94 | 95 | /** 96 | * Manages cookies to account number of requests a user can perform without registration. 97 | *

98 | * 1. For returned user, decreases the cookie value by 1 99 | * 2. For new user, sets a cookie with value FREE_REQUESTS_NUMBER_PER_NEW_USER
100 | * @param request http request 101 | * @param response http response 102 | * @param chain filters chain 103 | * @throws IOException can't delegate to the filters chain 104 | * @throws ServletException can't delegate to the filters chain 105 | */ 106 | @Override 107 | public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { 108 | if (request instanceof HttpServletRequest) { 109 | HttpServletRequest httpServletRequest = (HttpServletRequest) request; 110 | LOG.info("intercepted " + httpServletRequest + "\n" 111 | + "in the userRequestsLimitationFilter"); 112 | //avoiding null pointer exception 113 | Cookie[] cookies = Optional.ofNullable(httpServletRequest.getCookies()) 114 | .orElse(new Cookie[]{}); 115 | Cookie updatedCookie = Arrays.stream(cookies) 116 | .filter(cookie -> cookie.getName().equals(FREE_REQUESTS_COOKIE_NAME)) 117 | .map(cookie -> { 118 | LOG.debug("Decreases freeRequestsCookie value by 1"); 119 | int freeRequestsLeftNumber = Integer.parseInt(cookie.getValue()); 120 | cookie.setValue(String.valueOf(freeRequestsLeftNumber - 1)); 121 | LOG.info("free requests number: " + cookie.getValue()); 122 | return cookie; 123 | }) 124 | .findFirst() 125 | .orElseGet(() -> { 126 | LOG.debug("Create a new cookies"); 127 | Cookie newCookie = new Cookie(FREE_REQUESTS_COOKIE_NAME, FREE_REQUESTS_NUMBER_PER_NEW_USER); 128 | newCookie.setMaxAge(COOKIE_MAX_AGE); 129 | newCookie.setPath("/"); 130 | LOG.info("free requests number: " + newCookie.getValue()); 131 | return newCookie; 132 | }); 133 | ((HttpServletResponse) response).addCookie(updatedCookie); 134 | } 135 | 136 | chain.doFilter(request, response); 137 | } 138 | 139 | @Override 140 | public void destroy() { 141 | 142 | } 143 | }; 144 | } 145 | 146 | } 147 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/config/GitHubApiEndpoints.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.config; 2 | 3 | /** 4 | * @author lyashenkogs. 5 | */ 6 | public enum GitHubApiEndpoints { 7 | COMMITS, STARGAZERS 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/config/MondoDbOpenshiftConfig.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.config; 2 | 3 | import com.mongodb.MongoClient; 4 | import com.mongodb.MongoCredential; 5 | import com.mongodb.ServerAddress; 6 | import org.springframework.beans.factory.annotation.Value; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | import org.springframework.context.annotation.Profile; 10 | import org.springframework.data.mongodb.core.MongoTemplate; 11 | import org.springframework.data.mongodb.core.SimpleMongoDbFactory; 12 | 13 | import java.net.UnknownHostException; 14 | import java.util.Collections; 15 | 16 | /** 17 | * Setting to connect to MongoDb 2.4 on openshift.com 18 | * 19 | * @author lyashenkogs. 20 | * @since 8/31/16 21 | */ 22 | @Profile("openshift") 23 | @Configuration 24 | public class MondoDbOpenshiftConfig { 25 | 26 | @Value("${mongodb_db_password}") 27 | private String password; 28 | 29 | @Value("${mongodb_db_host}") 30 | private String host; 31 | 32 | @Value("${mongodb_db_username}") 33 | private String username; 34 | 35 | @Value("${mongodb_database}") 36 | private String database; 37 | 38 | @Value("${mongodb_db_port}") 39 | private String port; 40 | 41 | @Bean 42 | public MongoTemplate mongoTemplate() throws UnknownHostException { 43 | char[] passwordChar = password.toCharArray(); 44 | int openshiftMongoDbPort = Integer.parseInt(port); 45 | return getMongoTemplate(host, openshiftMongoDbPort, database, database, username, passwordChar); 46 | } 47 | 48 | private MongoTemplate getMongoTemplate(String host, int port, 49 | String authenticationDB,//TODO: is it redundant ? 50 | String database, 51 | String user, char[] password) 52 | throws UnknownHostException { 53 | return new MongoTemplate( 54 | new SimpleMongoDbFactory( 55 | new MongoClient( 56 | new ServerAddress(host, port), 57 | Collections.singletonList( 58 | MongoCredential.createCredential( 59 | user, 60 | authenticationDB, 61 | password 62 | ) 63 | ) 64 | ), 65 | database 66 | ) 67 | ); 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/config/RestTemplateConfig.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.config; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.context.annotation.Bean; 5 | import org.springframework.context.annotation.Configuration; 6 | import org.springframework.http.client.ClientHttpRequestInterceptor; 7 | import org.springframework.web.client.RestTemplate; 8 | 9 | import java.util.LinkedList; 10 | 11 | /** 12 | * Configure RestTemplate for Github API 13 | * 14 | * @see https://developer.github.com/v3/ 15 | */ 16 | @Configuration 17 | public class RestTemplateConfig { 18 | 19 | @Autowired 20 | private AddOAuthTokenInterceptor oAuthTokenInterceptor; 21 | 22 | @Autowired 23 | private ChangeAcceptTypeForStargazersInterceptor changeAcceptTypeForStargazersInterceptor; 24 | 25 | @Bean 26 | RestTemplate restTemplate() { 27 | RestTemplate restTemplate = new RestTemplate(); 28 | LinkedList clientHttpRequestInterceptors = new LinkedList<>(); 29 | clientHttpRequestInterceptors.add(oAuthTokenInterceptor); 30 | clientHttpRequestInterceptors.add(changeAcceptTypeForStargazersInterceptor); 31 | 32 | restTemplate.setInterceptors(clientHttpRequestInterceptors); 33 | return restTemplate; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/config/SwaggerConfig.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.config; 2 | 3 | import com.google.common.base.Predicate; 4 | import org.springframework.context.annotation.Bean; 5 | import org.springframework.context.annotation.Configuration; 6 | import springfox.documentation.builders.ApiInfoBuilder; 7 | import springfox.documentation.builders.RequestHandlerSelectors; 8 | import springfox.documentation.service.ApiInfo; 9 | import springfox.documentation.spi.DocumentationType; 10 | import springfox.documentation.spring.web.plugins.Docket; 11 | import springfox.documentation.swagger.web.UiConfiguration; 12 | import springfox.documentation.swagger2.annotations.EnableSwagger2; 13 | 14 | import java.time.LocalDate; 15 | 16 | import static com.google.common.base.Predicates.or; 17 | import static springfox.documentation.builders.PathSelectors.regex; 18 | 19 | /** 20 | * @author Grigoriy Lyashenko (Grog). 21 | */ 22 | @Configuration 23 | @EnableSwagger2 24 | public class SwaggerConfig { 25 | 26 | @Bean 27 | public Docket api() { 28 | return new Docket(DocumentationType.SWAGGER_2) 29 | .apiInfo(apiInfo()) 30 | //a workaround to correctly display LocalDate type 31 | .directModelSubstitute(LocalDate.class, java.sql.Date.class) 32 | .select() 33 | .apis(RequestHandlerSelectors.any()) 34 | .paths(analyticsPaths()) 35 | .build(); 36 | } 37 | 38 | private Predicate analyticsPaths() { 39 | return or( 40 | regex(".*/commits.*"), 41 | regex(".*/stargazers.*"), 42 | regex(".*/uniqueContributors.*"), 43 | regex("/statistics.*") 44 | ); 45 | } 46 | 47 | private ApiInfo apiInfo() { 48 | return new ApiInfoBuilder() 49 | .title("analytics4github API") 50 | .description("A public REST API to perform analytics on GitHub repositories") 51 | .license("MIT") 52 | .licenseUrl("https://github.com/LyashenkoGS/analytics4github/blob/master/LICENSE") 53 | .version("1.0") 54 | .build(); 55 | } 56 | 57 | @Bean 58 | UiConfiguration uiConfig() { 59 | return new UiConfiguration( 60 | "", 61 | "list", 62 | "alpha", 63 | "schema", 64 | true, 65 | true 66 | ); 67 | } 68 | } -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/controller/CommitsController.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.controller; 2 | 3 | import com.rhcloud.analytics4github.dto.RequestFromFrontendDto; 4 | import com.rhcloud.analytics4github.dto.ResponceForFrontendDto; 5 | import com.rhcloud.analytics4github.exception.GitHubRESTApiException; 6 | import com.rhcloud.analytics4github.service.CommitsService; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.web.bind.annotation.GetMapping; 9 | import org.springframework.web.bind.annotation.ModelAttribute; 10 | import org.springframework.web.bind.annotation.RestController; 11 | 12 | import java.io.IOException; 13 | import java.net.URISyntaxException; 14 | import java.util.Collections; 15 | import java.util.List; 16 | import java.util.concurrent.ExecutionException; 17 | 18 | /** 19 | * @author lyashenkogs. 20 | */ 21 | @RestController 22 | public class CommitsController { 23 | 24 | @Autowired 25 | private CommitsService commitsService; 26 | 27 | @GetMapping("/{author}/{repository}/commits") 28 | List getCommits(@ModelAttribute RequestFromFrontendDto requestFromFrontendDto) throws IOException, URISyntaxException, ClassNotFoundException, ExecutionException, InterruptedException, GitHubRESTApiException { 29 | return Collections.singletonList(commitsService.getCommitsFrequency(requestFromFrontendDto)); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/controller/GitHubTrendingController.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.controller; 2 | 3 | 4 | import com.rhcloud.analytics4github.exception.TrendingException; 5 | import com.rhcloud.analytics4github.service.GitHubTrendingService; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.http.HttpStatus; 10 | import org.springframework.http.MediaType; 11 | import org.springframework.http.ResponseEntity; 12 | import org.springframework.web.bind.annotation.GetMapping; 13 | import org.springframework.web.bind.annotation.RestController; 14 | 15 | /** 16 | * @author lyashenkogs. 17 | * @since 9/3/16 18 | */ 19 | @RestController 20 | public class GitHubTrendingController { 21 | private static Logger LOG = LoggerFactory.getLogger(GitHubTrendingController.class); 22 | 23 | @Autowired 24 | private GitHubTrendingService trendingService; 25 | 26 | @GetMapping(value = "/randomRequestTrendingRepoName", consumes = MediaType.ALL_VALUE, produces = MediaType.TEXT_PLAIN_VALUE) 27 | ResponseEntity getRandomTrendingRepo() { 28 | try { 29 | return ResponseEntity 30 | .status(HttpStatus.OK) 31 | .body(trendingService.getRandomTrendingRepo()); 32 | } catch (TrendingException e) { 33 | LOG.error(e.getMessage()); 34 | return ResponseEntity 35 | .status(HttpStatus.INTERNAL_SERVER_ERROR) 36 | .body(e.getMessage()); 37 | } 38 | } 39 | } 40 | 41 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/controller/NumberRequestsLeftController.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.controller; 2 | 3 | import com.rhcloud.analytics4github.util.GitHubApiIterator; 4 | import org.springframework.web.bind.annotation.GetMapping; 5 | import org.springframework.web.bind.annotation.RestController; 6 | 7 | 8 | @RestController 9 | public class NumberRequestsLeftController { 10 | 11 | @GetMapping(value = "/getRequestsNumberLeft") 12 | Integer getRequestsLeft() { 13 | return GitHubApiIterator.requestsLeft; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/controller/StargazersController.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.controller; 2 | 3 | import com.rhcloud.analytics4github.dto.RequestFromFrontendDto; 4 | import com.rhcloud.analytics4github.dto.ResponceForFrontendDto; 5 | import com.rhcloud.analytics4github.exception.GitHubRESTApiException; 6 | import com.rhcloud.analytics4github.service.StargazersService; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.web.bind.annotation.GetMapping; 9 | import org.springframework.web.bind.annotation.ModelAttribute; 10 | import org.springframework.web.bind.annotation.RestController; 11 | 12 | import java.io.IOException; 13 | import java.net.URISyntaxException; 14 | import java.util.Collections; 15 | import java.util.List; 16 | import java.util.concurrent.ExecutionException; 17 | 18 | @RestController 19 | public class StargazersController { 20 | 21 | @Autowired 22 | private StargazersService stargazersService; 23 | 24 | @GetMapping("/{author}/{repository}/stargazers") 25 | List stargazers(@ModelAttribute RequestFromFrontendDto requestFromFrontendDto) throws IOException, URISyntaxException, ClassNotFoundException, ExecutionException, InterruptedException, GitHubRESTApiException { 26 | return Collections.singletonList(stargazersService.stargazersFrequency(requestFromFrontendDto)); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/controller/StatisticsController.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.controller; 2 | 3 | import com.rhcloud.analytics4github.domain.RequestToAPI; 4 | import com.rhcloud.analytics4github.service.StatisticsService; 5 | import io.swagger.annotations.ApiOperation; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.web.bind.annotation.GetMapping; 8 | import org.springframework.web.bind.annotation.RequestMapping; 9 | import org.springframework.web.bind.annotation.RestController; 10 | 11 | import java.util.List; 12 | 13 | /** 14 | * @author lyashenkogs. 15 | * @since 8/31/16 16 | */ 17 | @RestController 18 | @RequestMapping(value = "/statistics") 19 | public class StatisticsController { 20 | 21 | @Autowired 22 | private StatisticsService statisticsService; 23 | 24 | @ApiOperation(value = "get the application visitors requests list", notes = "get requests statistic") 25 | @GetMapping(value = "/requests") 26 | List getRequests() { 27 | return statisticsService.getRequestsStatistic(); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/controller/UniqueContributorsController.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.controller; 2 | 3 | import com.rhcloud.analytics4github.dto.RequestFromFrontendDto; 4 | import com.rhcloud.analytics4github.dto.ResponceForFrontendDto; 5 | import com.rhcloud.analytics4github.exception.GitHubRESTApiException; 6 | import com.rhcloud.analytics4github.service.UniqueContributorsService; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.web.bind.annotation.GetMapping; 9 | import org.springframework.web.bind.annotation.ModelAttribute; 10 | import org.springframework.web.bind.annotation.RestController; 11 | 12 | import java.io.IOException; 13 | import java.net.URISyntaxException; 14 | import java.util.Collections; 15 | import java.util.List; 16 | import java.util.concurrent.ExecutionException; 17 | 18 | @RestController 19 | public class UniqueContributorsController { 20 | 21 | @Autowired 22 | private UniqueContributorsService uniqueContributorsService; 23 | 24 | @GetMapping("/{author}/{repository}/uniqueContributors") 25 | List stargazersFrequency(@ModelAttribute RequestFromFrontendDto requestFromFrontendDto) throws IOException, URISyntaxException, ClassNotFoundException, ExecutionException, InterruptedException, GitHubRESTApiException { 26 | return Collections.singletonList(uniqueContributorsService.getUniqueContributorsFrequency(requestFromFrontendDto)); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/domain/Author.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.domain; 2 | 3 | /** 4 | * represent a commit author 5 | * 6 | * @author lyashenkogs. 7 | */ 8 | public class Author { 9 | private final String name; 10 | private final String email; 11 | 12 | /** 13 | * @param name represent author name. Add an empty String of zero length, if not present 14 | * @param email represent author email. Add an empty String of zero length, if not present 15 | */ 16 | public Author(String name, String email) { 17 | this.email = email; 18 | this.name = name; 19 | } 20 | 21 | @Override 22 | public boolean equals(Object o) { 23 | if (this == o) return true; 24 | if (o == null || getClass() != o.getClass()) return false; 25 | 26 | Author author = (Author) o; 27 | 28 | if (name != null ? !name.equals(author.name) : author.name != null) return false; 29 | return email != null ? email.equals(author.email) : author.email == null; 30 | 31 | } 32 | 33 | @Override 34 | public int hashCode() { 35 | int result = name != null ? name.hashCode() : 0; 36 | result = 31 * result + (email != null ? email.hashCode() : 0); 37 | return result; 38 | } 39 | 40 | public String getName() { 41 | return name; 42 | } 43 | 44 | public String getEmail() { 45 | return email; 46 | } 47 | 48 | @Override 49 | public String toString() { 50 | final StringBuilder sb = new StringBuilder("Author{"); 51 | sb.append("name='").append(name).append('\''); 52 | sb.append(", email='").append(email).append('\''); 53 | sb.append('}'); 54 | return sb.toString(); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/domain/RequestToAPI.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.domain; 2 | 3 | import org.springframework.data.annotation.Id; 4 | import org.springframework.data.mongodb.core.mapping.Document; 5 | 6 | import java.time.Instant; 7 | import java.time.temporal.ChronoUnit; 8 | 9 | /** 10 | * @author lyashenkogs. 11 | */ 12 | @Document 13 | public class RequestToAPI { 14 | private final String repository; 15 | private final String requestTime; 16 | private final String endpoint; 17 | @Id 18 | private String id; 19 | 20 | public RequestToAPI(String repository, String endpoint) { 21 | this.repository = repository; 22 | this.endpoint = endpoint; 23 | this.requestTime = Instant.now().truncatedTo(ChronoUnit.SECONDS).toString(); 24 | } 25 | 26 | public String getRequestTime() { 27 | return requestTime; 28 | } 29 | 30 | public String getEndpoint() { 31 | return endpoint; 32 | } 33 | 34 | public String getRepository() { 35 | return repository; 36 | } 37 | 38 | @Override 39 | public String toString() { 40 | final StringBuilder sb = new StringBuilder("RequestToAPI{"); 41 | sb.append("id='").append(id).append('\''); 42 | sb.append(", repository='").append(repository).append('\''); 43 | sb.append(", requestTime='").append(requestTime).append('\''); 44 | sb.append(", endpoint='").append(endpoint).append('\''); 45 | sb.append('}'); 46 | return sb.toString(); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/dto/RequestFromFrontendDto.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.dto; 2 | 3 | import io.swagger.annotations.ApiModel; 4 | import io.swagger.annotations.ApiParam; 5 | import org.springframework.format.annotation.DateTimeFormat; 6 | 7 | import java.time.LocalDate; 8 | 9 | /** 10 | * Created by Iron on 27.12.2016. 11 | */ 12 | @ApiModel 13 | public class RequestFromFrontendDto { 14 | @ApiParam(required = true, example = "mewo2", value = "a repository author. Example: mewo2") 15 | private String author; 16 | @ApiParam(required = true, example = "terrain", value = "a repository author. Example: terrain") 17 | private String repository; 18 | @ApiParam(required = true, example = "2016-08-10", value = "a data to start analyze from. Example: 2016-08-10") 19 | @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) 20 | private LocalDate startPeriod; 21 | @ApiParam(required = true, example = "2016-08-17", value = "a data until analyze. Example: 2016-08-17") 22 | @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) 23 | private LocalDate endPeriod; 24 | 25 | public RequestFromFrontendDto() { 26 | } 27 | 28 | public String getAuthor() { 29 | return author; 30 | } 31 | 32 | public void setAuthor(String author) { 33 | this.author = author; 34 | } 35 | 36 | public String getRepository() { 37 | return repository; 38 | } 39 | 40 | public void setRepository(String repository) { 41 | this.repository = repository; 42 | } 43 | 44 | public LocalDate getStartPeriod() { 45 | return startPeriod; 46 | } 47 | 48 | public void setStartPeriod(LocalDate startPeriod) { 49 | this.startPeriod = startPeriod; 50 | } 51 | 52 | public LocalDate getEndPeriod() { 53 | return endPeriod; 54 | } 55 | 56 | public void setEndPeriod(LocalDate endPeriod) { 57 | this.endPeriod = endPeriod; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/dto/ResponceForFrontendDto.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.dto; 2 | 3 | import io.swagger.annotations.ApiModel; 4 | 5 | import java.util.List; 6 | 7 | @ApiModel 8 | public class ResponceForFrontendDto { 9 | private String name; 10 | private int requestsLeft; 11 | private List data; 12 | public ResponceForFrontendDto() { 13 | } 14 | 15 | public ResponceForFrontendDto(String name, List data) { 16 | this.name = name; 17 | this.data = data; 18 | } 19 | 20 | public ResponceForFrontendDto(String name, List data, int requestsLeft) { 21 | this.name = name; 22 | this.data = data; 23 | this.requestsLeft = requestsLeft; 24 | } 25 | 26 | @Override 27 | public String toString() { 28 | return "ResponceForFrontendDto{" + 29 | "name='" + name + '\'' + 30 | ", requestsLeft=" + requestsLeft + 31 | ", data=" + data + 32 | '}'; 33 | } 34 | 35 | public String getName() { 36 | return name; 37 | } 38 | 39 | public void setName(String name) { 40 | this.name = name; 41 | } 42 | 43 | public List getData() { 44 | return data; 45 | } 46 | 47 | public void setData(List data) { 48 | this.data = data; 49 | } 50 | 51 | public Integer getRequestsLeft() { 52 | return requestsLeft; 53 | } 54 | 55 | public void setRequestsLeft(Integer requestsLeft) { 56 | this.requestsLeft = requestsLeft; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/exception/GitHubRESTApiException.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.exception; 2 | 3 | /** 4 | * Created by nazar on 14.01.17. 5 | * Custom exception for GitHub REST API 6 | */ 7 | 8 | public class GitHubRESTApiException extends Exception { 9 | 10 | public GitHubRESTApiException(String message, Throwable cause) { 11 | super(message, cause); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/exception/GlobalDefaultExceptionHandler.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.exception; 2 | 3 | 4 | import org.slf4j.Logger; 5 | import org.springframework.http.HttpStatus; 6 | import org.springframework.http.ResponseEntity; 7 | import org.springframework.web.bind.annotation.ControllerAdvice; 8 | import org.springframework.web.bind.annotation.ExceptionHandler; 9 | import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; 10 | 11 | import static org.slf4j.LoggerFactory.getLogger; 12 | 13 | /** 14 | * Created by Nazar on 28.12.2016. 15 | * exception handling apply across the whole application, not just to an individual controller. 16 | */ 17 | @ControllerAdvice 18 | public class GlobalDefaultExceptionHandler extends ResponseEntityExceptionHandler { 19 | private static Logger LOG = getLogger(GlobalDefaultExceptionHandler.class); 20 | 21 | 22 | @ExceptionHandler(value = {GitHubRESTApiException.class}) 23 | public ResponseEntity gitHubRESTAPIException(GitHubRESTApiException exception) { 24 | LOG.error(exception.getMessage(), exception); 25 | return ResponseEntity 26 | .status(HttpStatus.NOT_FOUND) 27 | .body(exception.getMessage()); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/exception/TrendingException.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.exception; 2 | 3 | /** 4 | * Created by Nazar on 28.12.2016. 5 | * Represents an exception during interaction with github trending service 6 | */ 7 | public class TrendingException extends Exception { 8 | 9 | public TrendingException(String message) { 10 | super(message); 11 | } 12 | 13 | public TrendingException(String message, Throwable cause) { 14 | super(message, cause); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/repository/RequestToApiRepository.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.repository; 2 | 3 | import com.rhcloud.analytics4github.domain.RequestToAPI; 4 | import org.springframework.data.mongodb.repository.MongoRepository; 5 | 6 | import java.util.List; 7 | 8 | /** 9 | * @author lyashenkogs. 10 | */ 11 | public interface RequestToApiRepository extends MongoRepository { 12 | List findByRepository(String repository); 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/service/CommitsService.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.service; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import com.rhcloud.analytics4github.config.GitHubApiEndpoints; 5 | import com.rhcloud.analytics4github.dto.RequestFromFrontendDto; 6 | import com.rhcloud.analytics4github.dto.ResponceForFrontendDto; 7 | import com.rhcloud.analytics4github.exception.GitHubRESTApiException; 8 | import com.rhcloud.analytics4github.util.GitHubApiIterator; 9 | import com.rhcloud.analytics4github.util.Utils; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | import org.springframework.beans.factory.annotation.Autowired; 13 | import org.springframework.stereotype.Service; 14 | import org.springframework.web.client.RestTemplate; 15 | 16 | import java.io.IOException; 17 | import java.net.URISyntaxException; 18 | import java.time.LocalDate; 19 | import java.time.temporal.ChronoUnit; 20 | import java.util.LinkedList; 21 | import java.util.List; 22 | import java.util.TreeMap; 23 | import java.util.concurrent.ExecutionException; 24 | 25 | /** 26 | * @author lyashenkogs. 27 | */ 28 | @Service 29 | public class CommitsService { 30 | private static Logger LOG = LoggerFactory.getLogger(CommitsService.class); 31 | @Autowired 32 | private RestTemplate template; 33 | 34 | public List getCommitsList(RequestFromFrontendDto requestFromFrontendDto) throws URISyntaxException, ExecutionException, InterruptedException, GitHubRESTApiException { 35 | List thisMonthCommitsDateList = new LinkedList<>(); 36 | GitHubApiIterator iterator = new GitHubApiIterator(requestFromFrontendDto.getAuthor() + "/" + requestFromFrontendDto.getRepository(), template, 37 | GitHubApiEndpoints.COMMITS, Utils.getPeriodInstant(requestFromFrontendDto.getStartPeriod()), 38 | Utils.getPeriodInstant(requestFromFrontendDto.getEndPeriod())); 39 | while (iterator.hasNext()) { 40 | List commitPagesBatch = iterator.next(5); 41 | //Get localDatesList 42 | for (JsonNode commitPage : commitPagesBatch) { 43 | for (JsonNode commit : commitPage) { 44 | LOG.debug("commit: " + commit); 45 | LOG.debug("commitDate: " + commit.get("commit").get("author").get("date").textValue()); 46 | LocalDate localDate = Utils.parseTimestamp(commit.get("commit").get("author").get("date").textValue()); 47 | LOG.debug("parsed commit sinceMonth: " + localDate.toString()); 48 | thisMonthCommitsDateList.add(localDate); 49 | } 50 | } 51 | } 52 | iterator.close(); 53 | LOG.debug("finish parsing commits" + thisMonthCommitsDateList.toString()); 54 | return thisMonthCommitsDateList; 55 | } 56 | 57 | 58 | public ResponceForFrontendDto getCommitsFrequency(RequestFromFrontendDto requestFromFrontendDto) throws IOException, InterruptedException, ExecutionException, URISyntaxException, ClassNotFoundException, GitHubRESTApiException { 59 | TreeMap frequencyMap = Utils.buildStargazersFrequencyMap(getCommitsList(requestFromFrontendDto)); 60 | long period = ChronoUnit.DAYS.between(requestFromFrontendDto.getStartPeriod(), requestFromFrontendDto.getEndPeriod()); 61 | //if week 62 | if (period <= 7) { 63 | List frequencyList = Utils.parseWeekStargazersMapFrequencyToWeekFrequencyList(frequencyMap); 64 | ResponceForFrontendDto responceForFrontendDto = Utils.buildJsonForFrontend(frequencyList); 65 | LOG.debug("builded json for highchart.js :" + responceForFrontendDto); 66 | return responceForFrontendDto; 67 | //if month 68 | } else { 69 | List frequencyList = Utils.parseMonthFrequencyMapToFrequencyLIst(frequencyMap); 70 | ResponceForFrontendDto responceForFrontendDto = Utils.buildJsonForFrontend(frequencyList); 71 | LOG.debug("builded json for highchart.js :" + responceForFrontendDto); 72 | return responceForFrontendDto; 73 | } 74 | } 75 | 76 | } 77 | 78 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/service/GitHubTrendingService.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.service; 2 | 3 | import com.rhcloud.analytics4github.exception.TrendingException; 4 | import org.jsoup.Jsoup; 5 | import org.jsoup.nodes.Document; 6 | import org.jsoup.select.Elements; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.scheduling.annotation.Scheduled; 10 | import org.springframework.stereotype.Service; 11 | 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | import java.util.Random; 15 | 16 | /** 17 | * Parses https://github.com/trending and gets this month most popular repositories 18 | * Parser start on the class instantiation and saves data to an internal cachedTrendingRepos. 19 | * 20 | * @author lyashenkogs. 21 | * @since 9/3/16 22 | */ 23 | @Service 24 | public class GitHubTrendingService { 25 | 26 | static String GITHUB_TRENDING_URL = "https://github.com/trending?since=monthly"; 27 | private static Logger LOG = LoggerFactory.getLogger(GitHubTrendingService.class); 28 | private List trendingRepos = new ArrayList<>(); 29 | 30 | /** 31 | * Parses http://github/trending and retrieves 10 top repositories 32 | * The method cash will be invoked every 2 minutes. 33 | * 34 | * @throws TrendingException can't get or parse the web page 35 | */ 36 | @Scheduled(fixedRate = 120000) 37 | public void parseTrendingReposWebPage() throws TrendingException { 38 | try { 39 | Document webPageDocument = Jsoup.connect(GITHUB_TRENDING_URL) 40 | .userAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.122 Safari/534.30") 41 | .get(); 42 | Elements elements = webPageDocument.select("h3>a"); 43 | LOG.info("Top trending repositories according to " + GITHUB_TRENDING_URL); 44 | elements.forEach(element -> { 45 | LOG.info(element.attr("href")); 46 | trendingRepos.add(element.attr("href")); 47 | }); 48 | } catch (Exception exception) { 49 | throw new TrendingException(" Can not access GitHub Trending page https://github.com/trending", exception); 50 | } 51 | } 52 | 53 | public List getTrendingRepos() throws TrendingException { 54 | if (!trendingRepos.isEmpty()) { 55 | return trendingRepos; 56 | } else 57 | throw new TrendingException(" Can not access GitHub Trending page https://github.com/trending "); 58 | } 59 | 60 | public String getRandomTrendingRepo() throws TrendingException { 61 | List trendingRepos = getTrendingRepos(); 62 | int index = new Random().nextInt(trendingRepos.size()); 63 | String randomRepoName = trendingRepos.get(index); 64 | LOG.info("Random trending repo: " + randomRepoName); 65 | return randomRepoName; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/service/StargazersService.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.service; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import com.rhcloud.analytics4github.config.GitHubApiEndpoints; 5 | import com.rhcloud.analytics4github.dto.RequestFromFrontendDto; 6 | import com.rhcloud.analytics4github.dto.ResponceForFrontendDto; 7 | import com.rhcloud.analytics4github.exception.GitHubRESTApiException; 8 | import com.rhcloud.analytics4github.util.GitHubApiIterator; 9 | import com.rhcloud.analytics4github.util.Utils; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | import org.springframework.beans.factory.annotation.Autowired; 13 | import org.springframework.stereotype.Service; 14 | import org.springframework.web.client.RestTemplate; 15 | 16 | import java.io.IOException; 17 | import java.net.URISyntaxException; 18 | import java.time.LocalDate; 19 | import java.time.temporal.ChronoUnit; 20 | import java.util.LinkedList; 21 | import java.util.List; 22 | import java.util.TreeMap; 23 | import java.util.concurrent.ExecutionException; 24 | import java.util.stream.Collectors; 25 | 26 | /** 27 | * @author lyashenkogs 28 | */ 29 | @Service 30 | public class StargazersService { 31 | private static Logger LOG = LoggerFactory.getLogger(StargazersService.class); 32 | @Autowired 33 | private RestTemplate template; 34 | 35 | 36 | public ResponceForFrontendDto stargazersFrequency(RequestFromFrontendDto requestFromFrontendDto) throws InterruptedException, ExecutionException, URISyntaxException, IOException, ClassNotFoundException, GitHubRESTApiException { 37 | //if week 38 | long period = ChronoUnit.DAYS.between(requestFromFrontendDto.getStartPeriod(), requestFromFrontendDto.getEndPeriod()); 39 | if (period <= 7) { 40 | TreeMap stargazersFrequencyMap = Utils.buildStargazersFrequencyMap(getWeekStargazersList(requestFromFrontendDto)); 41 | List frequencyList = Utils.parseWeekStargazersMapFrequencyToWeekFrequencyList(stargazersFrequencyMap); 42 | ResponceForFrontendDto buildedDtoForFrontend = Utils.buildJsonForFrontend(frequencyList); 43 | LOG.debug("builded json for highchart.js :" + buildedDtoForFrontend); 44 | return buildedDtoForFrontend; 45 | } 46 | //if month 47 | else { 48 | TreeMap stargazersFrequencyMap = Utils.buildStargazersFrequencyMap(getMonthStargazersList(requestFromFrontendDto)); 49 | List frequencyList = Utils.parseMonthFrequencyMapToFrequencyLIst(stargazersFrequencyMap); 50 | ResponceForFrontendDto buildedDtoForFrontend = Utils.buildJsonForFrontend(frequencyList); 51 | LOG.debug("builded json for highchart.js :" + buildedDtoForFrontend); 52 | return buildedDtoForFrontend; 53 | } 54 | } 55 | 56 | public List getWeekStargazersList(RequestFromFrontendDto requestFromFrontendDto) throws URISyntaxException, ExecutionException, InterruptedException, GitHubRESTApiException { 57 | List thisWeekAllStargazersDateList = new LinkedList<>(); 58 | GitHubApiIterator stargazersIterator = new GitHubApiIterator(requestFromFrontendDto.getAuthor() + "/" + requestFromFrontendDto.getRepository(), template, GitHubApiEndpoints.STARGAZERS); 59 | while (stargazersIterator.hasNext()) { 60 | List stargazerPagesBatch = stargazersIterator.next(5); 61 | 62 | List stargazersFrequency = stargazerPagesBatch.stream() 63 | .map(jsonNode -> jsonNode.get(0).get("starred_at"))//get element "starred_at from each JSON inside the node 64 | .map(starredAt -> Utils.parseTimestamp(starredAt.textValue())) 65 | .filter(Utils::isWithinThisWeekRange) 66 | .collect(Collectors.toList()); 67 | LOG.debug(stargazersFrequency.toString()); 68 | 69 | thisWeekAllStargazersDateList.addAll(stargazersFrequency); 70 | 71 | boolean batchContainStargazersOutOfRange = stargazerPagesBatch.stream() 72 | .map(jsonNode -> jsonNode.get(0).get("starred_at"))//get element "starred_at from each JSON inside the node 73 | .map(starredAt -> Utils.parseTimestamp(starredAt.textValue())) 74 | .anyMatch((starredAtData) -> !Utils.isWithinThisWeekRange(starredAtData)); 75 | if (batchContainStargazersOutOfRange) break; 76 | } 77 | stargazersIterator.close(); 78 | 79 | LOG.debug("finish parsing stargazers" + thisWeekAllStargazersDateList.toString()); 80 | return thisWeekAllStargazersDateList; 81 | } 82 | 83 | public List getMonthStargazersList(RequestFromFrontendDto requestFromFrontendDto) throws URISyntaxException, ExecutionException, InterruptedException, GitHubRESTApiException { 84 | List thisMonthAllStargazersDateList = new LinkedList<>(); 85 | GitHubApiIterator stargazersIterator = new GitHubApiIterator(requestFromFrontendDto.getAuthor() + "/" + requestFromFrontendDto.getRepository(), template, GitHubApiEndpoints.STARGAZERS); 86 | while (stargazersIterator.hasNext()) { 87 | List stargazerPagesBatch = stargazersIterator.next(5); 88 | //parse pages where localDate withing month period 89 | List stargazersFrequency = stargazerPagesBatch.stream() 90 | .map(jsonNode -> jsonNode.get(0).get("starred_at"))//get element "starred_at from each JSON inside the node 91 | .map(jsonNode -> Utils.parseTimestamp(jsonNode.textValue())) 92 | .filter(localDate -> Utils.isWithinThisMonthRange(localDate, requestFromFrontendDto)) 93 | .collect(Collectors.toList()); 94 | LOG.debug(stargazersFrequency.toString()); 95 | 96 | thisMonthAllStargazersDateList.addAll(stargazersFrequency); 97 | 98 | boolean batchContainStargazersOutOfRange = stargazerPagesBatch.stream() 99 | .map(jsonNode -> jsonNode.get(0).get("starred_at"))//get element "starred_at from each JSON inside the node 100 | .map(jsonNode -> Utils.parseTimestamp(jsonNode.textValue())) 101 | .anyMatch((localDate) -> !Utils.isWithinThisMonthRange(localDate, requestFromFrontendDto)); 102 | if (batchContainStargazersOutOfRange) break; 103 | } 104 | stargazersIterator.close(); 105 | 106 | LOG.debug("finish parsing stargazers" + thisMonthAllStargazersDateList.toString()); 107 | return thisMonthAllStargazersDateList; 108 | } 109 | 110 | } 111 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/service/StatisticsService.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.service; 2 | 3 | import com.rhcloud.analytics4github.domain.RequestToAPI; 4 | import com.rhcloud.analytics4github.repository.RequestToApiRepository; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.stereotype.Service; 7 | 8 | import java.util.List; 9 | 10 | /** 11 | * @author lyashenkogs. 12 | * @since 8/31/16 13 | */ 14 | @Service 15 | public class StatisticsService { 16 | @Autowired 17 | private RequestToApiRepository repository; 18 | 19 | 20 | public List getRequestsStatistic() { 21 | return repository.findAll(); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/service/UniqueContributorsService.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.service; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import com.rhcloud.analytics4github.config.GitHubApiEndpoints; 5 | import com.rhcloud.analytics4github.domain.Author; 6 | import com.rhcloud.analytics4github.dto.RequestFromFrontendDto; 7 | import com.rhcloud.analytics4github.dto.ResponceForFrontendDto; 8 | import com.rhcloud.analytics4github.exception.GitHubRESTApiException; 9 | import com.rhcloud.analytics4github.util.GitHubApiIterator; 10 | import com.rhcloud.analytics4github.util.Utils; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | import org.springframework.beans.factory.annotation.Autowired; 14 | import org.springframework.stereotype.Service; 15 | import org.springframework.web.client.RestTemplate; 16 | import org.springframework.web.util.UriComponentsBuilder; 17 | 18 | import java.io.IOException; 19 | import java.net.URISyntaxException; 20 | import java.time.Instant; 21 | import java.time.LocalDate; 22 | import java.time.temporal.ChronoUnit; 23 | import java.util.*; 24 | import java.util.concurrent.ExecutionException; 25 | import java.util.stream.Collectors; 26 | import java.util.stream.StreamSupport; 27 | 28 | /** 29 | * @author lyashenkogs. 30 | */ 31 | @Service 32 | public class UniqueContributorsService { 33 | private static Logger LOG = LoggerFactory.getLogger(UniqueContributorsService.class); 34 | 35 | @Autowired 36 | private RestTemplate restTemplate; 37 | 38 | boolean isUniqueContributor(String repository, Author author, Instant uniqueSince) { 39 | String queryByAuthorName = UriComponentsBuilder.fromHttpUrl("https://api.github.com/repos/") 40 | .path(repository) 41 | .path("/" + GitHubApiEndpoints.COMMITS.toString().toLowerCase()) 42 | .queryParam("author", author.getName()) 43 | .queryParam("until", uniqueSince) 44 | .build().encode() 45 | .toUriString(); 46 | LOG.debug(queryByAuthorName); 47 | JsonNode commitsPage = restTemplate.getForObject(queryByAuthorName, JsonNode.class); 48 | LOG.debug(commitsPage.toString()); 49 | //if an author seems to be unique, check one more time by author email 50 | if (!commitsPage.has(0)) {//return true if a commits page look like [] 51 | LOG.debug("No commits by an author name: " + author.getName() + " untill " + uniqueSince); 52 | LOG.debug("Recheck by the author email: " + author.getEmail()); 53 | String queryByAuthorEmail = UriComponentsBuilder.fromHttpUrl("https://api.github.com/repos/") 54 | .path(repository) 55 | .path("/" + GitHubApiEndpoints.COMMITS.toString().toLowerCase()) 56 | .queryParam("author", author.getEmail()) 57 | .queryParam("until", uniqueSince) 58 | .build().encode() 59 | .toUriString(); 60 | LOG.debug(queryByAuthorEmail); 61 | commitsPage = restTemplate.getForObject(queryByAuthorEmail, JsonNode.class); 62 | LOG.debug(commitsPage.toString()); 63 | return !commitsPage.has(0);//return true if a commits page look like [] 64 | } else return false; 65 | } 66 | 67 | List getCommits(String repository, Instant since, Instant until) throws URISyntaxException, ExecutionException, InterruptedException, GitHubRESTApiException { 68 | GitHubApiIterator gitHubApiIterator = new GitHubApiIterator(repository, restTemplate, GitHubApiEndpoints.COMMITS, since, until); 69 | List commitPages = new ArrayList<>(); 70 | List commits = new ArrayList<>(); 71 | while (gitHubApiIterator.hasNext()) { 72 | List commitPagesBatch = gitHubApiIterator.next(5); 73 | commitPages.addAll(commitPagesBatch); 74 | } 75 | gitHubApiIterator.close(); 76 | LOG.debug(commitPages.toString()); 77 | for (JsonNode page : commitPages) { 78 | for (JsonNode commit : page) { 79 | commits.add(commit); 80 | LOG.debug(commit.toString()); 81 | } 82 | } 83 | return commits; 84 | } 85 | 86 | Set getAuthorNameAndEmail(List commits) { 87 | Set authors = new HashSet<>(); 88 | for (JsonNode commit : commits) { 89 | JsonNode authorEmail = commit.get("commit") 90 | .get("author").get("email"); 91 | JsonNode authorName = commit.get("commit") 92 | .get("author").get("name"); 93 | LOG.debug(authorEmail.textValue()); 94 | LOG.debug(authorName.textValue()); 95 | Author author = new Author(authorName.textValue(), authorEmail.textValue()); 96 | authors.add(author); 97 | } 98 | LOG.debug("Authors : " + authors); 99 | return authors; 100 | } 101 | 102 | Set getUniqueContributors(String projectName, Instant uniqueSince, Instant uniqueUntil) throws InterruptedException, ExecutionException, URISyntaxException, GitHubRESTApiException { 103 | Set authorsPerPeriod = getAuthorNameAndEmail(getCommits(projectName, uniqueSince, uniqueUntil)); 104 | Set newAuthors = authorsPerPeriod.parallelStream() 105 | .filter(author -> isUniqueContributor(projectName, author, uniqueSince)) 106 | .collect(Collectors.toSet()); 107 | LOG.debug("since" + uniqueSince + " are " + newAuthors.size() + " new authors: " + newAuthors.toString()); 108 | return newAuthors; 109 | } 110 | 111 | LocalDate getFirstContributionDate(Author author, String repository) throws GitHubRESTApiException { 112 | int reliableLastPageNumber = 0; 113 | String URL; 114 | if (author.getEmail() != null && !author.getEmail().isEmpty()) { 115 | reliableLastPageNumber = Utils.getLastPageNumber(repository, restTemplate, GitHubApiEndpoints.COMMITS, author.getEmail(), null, null); 116 | URL = UriComponentsBuilder 117 | .fromHttpUrl("https://api.github.com/repos/") 118 | .path(repository).path("/" + GitHubApiEndpoints.COMMITS.toString().toLowerCase()) 119 | .queryParam("page", reliableLastPageNumber) 120 | .queryParam("author", author.getEmail()) 121 | .build().encode() 122 | .toUriString(); 123 | } else { 124 | reliableLastPageNumber = Utils.getLastPageNumber(repository, restTemplate, GitHubApiEndpoints.COMMITS, author.getName(), null, null); 125 | URL = UriComponentsBuilder 126 | .fromHttpUrl("https://api.github.com/repos/") 127 | .path(repository).path("/" + GitHubApiEndpoints.COMMITS.toString().toLowerCase()) 128 | .queryParam("page", reliableLastPageNumber) 129 | .queryParam("author", author.getName()) 130 | .build().encode() 131 | .toUriString(); 132 | } 133 | LOG.debug(String.valueOf(reliableLastPageNumber)); 134 | LOG.info(URL); 135 | JsonNode commitsPage = restTemplate.getForObject(URL, JsonNode.class); 136 | for (JsonNode commit : commitsPage) { 137 | LOG.debug(commit.toString()); 138 | } 139 | List commits = StreamSupport.stream(commitsPage.spliterator(), false).collect(Collectors.toList()); 140 | JsonNode commit; 141 | try { 142 | commit = commits.get(commits.size() - 1); 143 | LOG.info("First commit by " + author + " : " + commit.toString()); 144 | String date = commit.get("commit").get("author").get("date").textValue(); 145 | LOG.debug(date); 146 | LocalDate firstContributionDate = Utils.parseTimestamp(date); 147 | LOG.info(firstContributionDate.toString()); 148 | return firstContributionDate; 149 | } catch (ArrayIndexOutOfBoundsException ex) { 150 | LOG.error("Cant properly get commits for :" + author); 151 | ex.printStackTrace(); 152 | throw ex; 153 | } 154 | } 155 | 156 | List getFirstAuthorCommitFrequencyList(String repository, Instant since, Instant until) throws InterruptedException, ExecutionException, URISyntaxException, GitHubRESTApiException { 157 | Set uniqueContributors = getUniqueContributors(repository, since, until); 158 | List firstAuthorCommitFrequencyList = new ArrayList<>(); 159 | for (Author author : uniqueContributors) { 160 | try { 161 | firstAuthorCommitFrequencyList.add(getFirstContributionDate(author, repository)); 162 | } catch (ArrayIndexOutOfBoundsException ex) { 163 | LOG.error("cant properly getFirstContributionDate :" + author); 164 | LOG.error("don't add any to firstAuthorCommitFrequencyList for " + repository); 165 | } 166 | } 167 | LOG.info(firstAuthorCommitFrequencyList.toString()); 168 | return firstAuthorCommitFrequencyList; 169 | } 170 | 171 | public ResponceForFrontendDto getUniqueContributorsFrequency(RequestFromFrontendDto requestFromFrontendDto) throws IOException, ClassNotFoundException, InterruptedException, ExecutionException, URISyntaxException, GitHubRESTApiException { 172 | //if week 173 | long period = ChronoUnit.DAYS.between(requestFromFrontendDto.getStartPeriod(), requestFromFrontendDto.getEndPeriod()); 174 | if (period <= 7) { 175 | List firstAuthorCommitFrequencyList = getFirstAuthorCommitFrequencyList(requestFromFrontendDto.getAuthor() + "/" + requestFromFrontendDto.getRepository(), Utils.getThisWeekBeginInstant(), null); 176 | TreeMap weekContributorsFrequencyMap = Utils.buildStargazersFrequencyMap(firstAuthorCommitFrequencyList); 177 | List frequencyList = Utils.parseWeekStargazersMapFrequencyToWeekFrequencyList(weekContributorsFrequencyMap); 178 | ResponceForFrontendDto responceForFrontendDto = Utils.buildJsonForFrontend(frequencyList); 179 | LOG.debug("builded json for highchart.js :" + responceForFrontendDto); 180 | return responceForFrontendDto; 181 | } 182 | //if month 183 | else { 184 | TreeMap commitsFrequencyMap = Utils.buildStargazersFrequencyMap(getFirstAuthorCommitFrequencyList 185 | (requestFromFrontendDto.getAuthor() + "/" + requestFromFrontendDto.getRepository(), Utils.getPeriodInstant(requestFromFrontendDto.getStartPeriod()), 186 | Utils.getPeriodInstant(requestFromFrontendDto.getEndPeriod()))); 187 | List frequencyList = Utils.parseMonthFrequencyMapToFrequencyLIst(commitsFrequencyMap); 188 | ResponceForFrontendDto responceForFrontendDto = Utils.buildJsonForFrontend(frequencyList); 189 | LOG.debug("builded json for highchart.js :" + responceForFrontendDto); 190 | return responceForFrontendDto; 191 | } 192 | } 193 | 194 | 195 | } 196 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/util/GitHubApiIterator.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.util; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import com.rhcloud.analytics4github.config.GitHubApiEndpoints; 6 | import com.rhcloud.analytics4github.exception.GitHubRESTApiException; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.http.HttpEntity; 10 | import org.springframework.http.HttpHeaders; 11 | import org.springframework.web.client.RestTemplate; 12 | import org.springframework.web.util.UriComponents; 13 | import org.springframework.web.util.UriComponentsBuilder; 14 | 15 | import java.net.URISyntaxException; 16 | import java.time.Instant; 17 | import java.util.Iterator; 18 | import java.util.List; 19 | import java.util.concurrent.CompletableFuture; 20 | import java.util.concurrent.ExecutionException; 21 | import java.util.concurrent.ExecutorService; 22 | import java.util.concurrent.Executors; 23 | import java.util.concurrent.atomic.AtomicInteger; 24 | import java.util.stream.Collectors; 25 | import java.util.stream.IntStream; 26 | 27 | /** 28 | * Iterate over commits array of specified project on Github using RestTemplate 29 | * 30 | * @author lyashenkogs. 31 | */ 32 | public class GitHubApiIterator implements Iterator { 33 | private static Logger LOG = LoggerFactory.getLogger(GitHubApiIterator.class); 34 | 35 | private final int numberOfPages; 36 | private final String projectName; 37 | private final GitHubApiEndpoints githubEndpoint; 38 | private final RestTemplate restTemplate; 39 | private final ExecutorService executor = Executors.newFixedThreadPool(5); 40 | private Instant since = null; 41 | private Instant until = null; 42 | private String author; 43 | private volatile AtomicInteger counter = new AtomicInteger(); 44 | public static int requestsLeft; 45 | 46 | public GitHubApiIterator(String projectName, RestTemplate restTemplate, GitHubApiEndpoints endpoint 47 | ) throws URISyntaxException, GitHubRESTApiException { 48 | this.restTemplate = restTemplate; 49 | this.projectName = projectName; 50 | this.githubEndpoint = endpoint; 51 | this.numberOfPages = getLastPageNumber(projectName); 52 | this.counter.set(numberOfPages); 53 | } 54 | 55 | public GitHubApiIterator(String projectName, String author, RestTemplate restTemplate, GitHubApiEndpoints endpoint 56 | ) throws URISyntaxException, GitHubRESTApiException { 57 | this.author = author; 58 | this.restTemplate = restTemplate; 59 | this.projectName = projectName; 60 | this.githubEndpoint = endpoint; 61 | this.numberOfPages = getLastPageNumber(projectName); 62 | this.counter.set(numberOfPages); 63 | } 64 | 65 | public GitHubApiIterator(String projectName, RestTemplate restTemplate, GitHubApiEndpoints endpoint, 66 | Instant since, Instant until) throws URISyntaxException, GitHubRESTApiException { 67 | this.since = since; 68 | this.until = until; 69 | this.restTemplate = restTemplate; 70 | this.projectName = projectName; 71 | this.githubEndpoint = endpoint; 72 | this.numberOfPages = getLastPageNumber(projectName); 73 | this.counter.set(numberOfPages); 74 | } 75 | 76 | public int getNumberOfPages() { 77 | return numberOfPages; 78 | } 79 | 80 | public String getProjectName() { 81 | return projectName; 82 | } 83 | 84 | public int getLastPageNumber(String projectName) throws URISyntaxException, GitHubRESTApiException { 85 | return Utils.getLastPageNumber(projectName, restTemplate, githubEndpoint, author, since, until); 86 | } 87 | 88 | public synchronized boolean hasNext() { 89 | return counter.get() > 0; 90 | } 91 | 92 | /** 93 | * Performs side efects (parsing a header and updating the static field 94 | * @return JsonNode that represents a stargazers list 95 | * 96 | */ 97 | public JsonNode next() { 98 | if (counter.get() > 0) { 99 | String basicURL = "https://api.github.com/repos/" + projectName + "/" + githubEndpoint.toString().toLowerCase(); 100 | UriComponents page; 101 | 102 | if (since != null && until!=null) { 103 | page = UriComponentsBuilder.fromHttpUrl(basicURL) 104 | .queryParam("page", counter.getAndDecrement()) 105 | .queryParam("since", since) 106 | .queryParam("until",until) 107 | .build(); 108 | } else if (since != null){ 109 | page = UriComponentsBuilder.fromHttpUrl(basicURL) 110 | .queryParam("page", counter.getAndDecrement()) 111 | .queryParam("since", since) 112 | .build(); 113 | } else if (author != null) { 114 | page = UriComponentsBuilder.fromHttpUrl(basicURL) 115 | .queryParam("page", counter.getAndDecrement()) 116 | .queryParam("author", author) 117 | .build(); 118 | } else { 119 | page = UriComponentsBuilder.fromHttpUrl(basicURL) 120 | .queryParam("page", counter.getAndDecrement()) 121 | .build(); 122 | } 123 | String URL = page.encode().toUriString(); 124 | LOG.debug(URL); 125 | //sent request 126 | HttpEntity entity = restTemplate.getForEntity(URL, JsonNode.class); 127 | initializeRequestsLeft(restTemplate); 128 | JsonNode node = new ObjectMapper().convertValue(entity.getBody(), JsonNode.class); 129 | LOG.debug(node.toString()); 130 | return node; 131 | } else throw new IndexOutOfBoundsException("there is no next element"); 132 | } 133 | 134 | public static void initializeRequestsLeft(RestTemplate restTemplate){ 135 | try{ 136 | HttpEntity entity = restTemplate.getForEntity("https://api.github.com/users/whatever", JsonNode.class); 137 | HttpHeaders headers = entity.getHeaders(); 138 | GitHubApiIterator.requestsLeft = Integer.parseInt(headers.get("X-RateLimit-Remaining").get(0)); 139 | } catch (Exception e){ 140 | LOG.debug(e.getMessage()); 141 | } 142 | } 143 | 144 | /** 145 | * @param batchSize number of numberOfPages to return 146 | * @return list of stargazer numberOfPages according to batchSize 147 | * @throws IndexOutOfBoundsException if there is no elements left to return 148 | */ 149 | public List next(int batchSize) throws ExecutionException, InterruptedException { 150 | int reliableBatchSize = batchSize; 151 | if (batchSize > counter.get()) { 152 | LOG.warn("batch size is bigger then number of elements left, decrease batch size to " + counter); 153 | reliableBatchSize = counter.get(); 154 | } 155 | 156 | List> completableFutures = IntStream.range(0, reliableBatchSize) 157 | .mapToObj( 158 | e -> CompletableFuture 159 | .supplyAsync(this::next, executor) 160 | ).collect(Collectors.toList()); 161 | 162 | CompletableFuture> result = CompletableFuture 163 | .allOf(completableFutures 164 | .toArray(new CompletableFuture[completableFutures.size()])) 165 | .thenApply(v -> completableFutures.stream() 166 | .map(CompletableFuture::join) 167 | .collect(Collectors.toList()));//compose all in one task 168 | List jsonNodes = result.get();// 169 | LOG.debug("batch completed, counter:" + counter.get()); 170 | return jsonNodes; 171 | } 172 | 173 | /** 174 | * invoke explicitly after every class usage to close ThreadPool correctly 175 | */ 176 | public void close() { 177 | this.executor.shutdown(); 178 | } 179 | } 180 | -------------------------------------------------------------------------------- /src/main/java/com/rhcloud/analytics4github/util/Utils.java: -------------------------------------------------------------------------------- 1 | package com.rhcloud.analytics4github.util; 2 | 3 | import com.rhcloud.analytics4github.config.GitHubApiEndpoints; 4 | import com.rhcloud.analytics4github.dto.RequestFromFrontendDto; 5 | import com.rhcloud.analytics4github.dto.ResponceForFrontendDto; 6 | import com.rhcloud.analytics4github.exception.GitHubRESTApiException; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.http.HttpHeaders; 10 | import org.springframework.web.client.RestTemplate; 11 | import org.springframework.web.util.UriComponentsBuilder; 12 | 13 | import java.io.IOException; 14 | import java.net.URISyntaxException; 15 | import java.time.*; 16 | import java.time.format.DateTimeFormatter; 17 | import java.time.temporal.ChronoUnit; 18 | import java.time.temporal.TemporalAdjusters; 19 | import java.util.*; 20 | import java.util.concurrent.ExecutionException; 21 | import java.util.function.Function; 22 | import java.util.regex.Matcher; 23 | import java.util.regex.Pattern; 24 | import java.util.stream.Collectors; 25 | 26 | import static java.time.DayOfWeek.MONDAY; 27 | import static java.time.DayOfWeek.SUNDAY; 28 | import static java.time.temporal.TemporalAdjusters.nextOrSame; 29 | import static java.time.temporal.TemporalAdjusters.previousOrSame; 30 | 31 | /** 32 | * Common utilities for complex operation on objects representing date and time. 33 | * 34 | * @author lyashenkogs. 35 | */ 36 | public class Utils { 37 | static String HTTPS_API_GITHUB_COM_REPOS = "https://api.github.com/repos/"; 38 | private static Logger LOG = LoggerFactory.getLogger(Utils.class); 39 | private Utils() { 40 | } 41 | 42 | /** 43 | * Assert that given Date is in the range from current monday to sunday 44 | * include border values. 45 | * Must return true for all days from this week include monday and sunday. 46 | */ 47 | public static boolean isWithinThisWeekRange(LocalDate timestamp) { 48 | LOG.debug("Check is the " + timestamp + " is within this week range"); 49 | LocalDate today = LocalDate.now(); 50 | LocalDate monday = today.with(previousOrSame(MONDAY)); 51 | LocalDate sunday = today.with(nextOrSame(SUNDAY)); 52 | boolean isWithinThisWeekRange = (timestamp.isAfter(monday.minusDays(1))) && timestamp.isBefore(sunday.plusDays(1)); 53 | LOG.debug(String.valueOf(isWithinThisWeekRange)); 54 | return isWithinThisWeekRange; 55 | } 56 | 57 | /** 58 | * 59 | * @param timestamp String representing date and time in ISO 8601 YYYY-MM-DDTHH:MM:SSZ 60 | * @param requestFromFrontendDto DTO came from frontend 61 | * @return if commit is withing analitics period 62 | */ 63 | public static boolean isWithinThisMonthRange(LocalDate timestamp, RequestFromFrontendDto requestFromFrontendDto) { 64 | LOG.debug("Check is the " + timestamp + " is within this month range"); 65 | LocalDate monthStart = requestFromFrontendDto.getStartPeriod(); 66 | LocalDate monthEnd = requestFromFrontendDto.getEndPeriod(); 67 | 68 | boolean isWithinThisMonthRange = (timestamp.isAfter(monthStart) || timestamp.isEqual(monthStart)) 69 | && (timestamp.isBefore(monthEnd) || timestamp.isEqual(monthEnd)); 70 | LOG.debug(String.valueOf(isWithinThisMonthRange)); 71 | return isWithinThisMonthRange; 72 | } 73 | 74 | /** 75 | * @param timestamp String representing date and time in ISO 8601 YYYY-MM-DDTHH:MM:SSZ 76 | * @return LocalDate parsed from the given @param. Example (2007-12-03) 77 | */ 78 | public static LocalDate parseTimestamp(String timestamp) { 79 | return LocalDate.parse(timestamp, DateTimeFormatter.ISO_DATE_TIME); 80 | } 81 | 82 | public static Instant getThisMonthBeginInstant() { 83 | LocalDateTime localDate = LocalDateTime.now().withSecond(0).withHour(0).withMinute(0) 84 | .with(TemporalAdjusters.firstDayOfMonth()).truncatedTo(ChronoUnit.SECONDS); 85 | return localDate.toInstant(ZoneOffset.UTC); 86 | } 87 | 88 | public static Instant getPeriodInstant(LocalDate localDate) { 89 | LocalDateTime localDateTime = localDate.atStartOfDay(); 90 | return localDateTime.toInstant(ZoneOffset.UTC); 91 | } 92 | 93 | public static Instant getThisWeekBeginInstant() { 94 | LocalDateTime localDate = LocalDateTime.now().withSecond(0).withHour(0).withMinute(0) 95 | .with((MONDAY)).truncatedTo(ChronoUnit.SECONDS); 96 | return localDate.toInstant(ZoneOffset.UTC); 97 | } 98 | 99 | public static ResponceForFrontendDto buildJsonForFrontend(List stargazersFrequencyList) throws IOException, ClassNotFoundException { 100 | ResponceForFrontendDto outputDto = new ResponceForFrontendDto(); 101 | outputDto.setName("Stars"); 102 | outputDto.setData(stargazersFrequencyList); 103 | return outputDto; 104 | } 105 | 106 | public static List parseWeekStargazersMapFrequencyToWeekFrequencyList(TreeMap weekStargazersFrequenyMap) { 107 | LOG.debug("parseWeekStargazersMapFrequencyToWeekFrequencyList"); 108 | List output = new ArrayList<>(); 109 | for (DayOfWeek dayOfWeek : DayOfWeek.values()) { 110 | LOG.debug(dayOfWeek.toString()); 111 | Optional optional = weekStargazersFrequenyMap.keySet().stream() 112 | .filter(e -> DayOfWeek.from(e).equals(dayOfWeek)) 113 | .findFirst(); 114 | if (optional.isPresent()) { 115 | LOG.debug("match " + optional.get() + " with frequency " + weekStargazersFrequenyMap.get(optional.get())); 116 | output.add(weekStargazersFrequenyMap.get(optional.get())); 117 | } else { 118 | LOG.debug("no match from " + weekStargazersFrequenyMap.keySet()); 119 | LOG.debug("add 0"); 120 | output.add(0); 121 | } 122 | 123 | } 124 | LOG.debug("Output is" + output.toString()); 125 | return output; 126 | } 127 | 128 | public static List parseMonthFrequencyMapToFrequencyLIst(TreeMap mockWeekStargazersFrequencyMap) throws IOException { 129 | int lastDayOfMonth = LocalDate.now().with(TemporalAdjusters.lastDayOfMonth()).getDayOfMonth(); 130 | LOG.debug(String.valueOf(lastDayOfMonth)); 131 | List monthStargazersFrequency = new ArrayList<>(lastDayOfMonth); 132 | for (int dayOfMonth = 1; dayOfMonth < lastDayOfMonth + 1; dayOfMonth++) { 133 | LOG.debug("day of month: " + dayOfMonth); 134 | Optional frequency = Optional.empty(); 135 | for (LocalDate localDate : mockWeekStargazersFrequencyMap.keySet()) { 136 | if (dayOfMonth == localDate.getDayOfMonth()) { 137 | frequency = Optional.of(mockWeekStargazersFrequencyMap.get(localDate)); 138 | } 139 | } 140 | if (frequency.isPresent()) { 141 | monthStargazersFrequency.add(frequency.get()); 142 | } else { 143 | monthStargazersFrequency.add(0); 144 | } 145 | LOG.debug(monthStargazersFrequency.toString()); 146 | } 147 | return monthStargazersFrequency; 148 | } 149 | 150 | 151 | public static TreeMap buildStargazersFrequencyMap(List stargazersList) throws IOException, URISyntaxException, ExecutionException, InterruptedException { 152 | //temporary set 153 | Set stargazersDateSet = new HashSet<>(stargazersList); 154 | Map stargazersFrequencyMap = stargazersDateSet.stream().collect(Collectors 155 | .toMap(Function.identity(), e -> Collections.frequency(stargazersList, e))); 156 | TreeMap localDateIntegerNavigableMap = new TreeMap<>(stargazersFrequencyMap); 157 | LOG.debug("stargazers week/month frequency map:" + localDateIntegerNavigableMap.toString()); 158 | return localDateIntegerNavigableMap; 159 | } 160 | 161 | public static int getLastPageNumber(String repository, RestTemplate restTemplate, GitHubApiEndpoints githubEndpoint, String author, Instant since, Instant until) throws GitHubRESTApiException{ 162 | String url; 163 | try { 164 | if (since != null) { 165 | url = UriComponentsBuilder.fromHttpUrl(HTTPS_API_GITHUB_COM_REPOS) 166 | .path(repository).path("/" + githubEndpoint.toString().toLowerCase()) 167 | .queryParam("since", since) 168 | .queryParam("until", until) 169 | .build().encode() 170 | .toUriString(); 171 | } else if (author != null) { 172 | url = UriComponentsBuilder.fromHttpUrl(HTTPS_API_GITHUB_COM_REPOS) 173 | .path(repository).path("/" + githubEndpoint.toString().toLowerCase()) 174 | .queryParam("author", author) 175 | .queryParam("until", until) 176 | .build().encode() 177 | .toUriString(); 178 | } else { 179 | url = UriComponentsBuilder.fromHttpUrl(HTTPS_API_GITHUB_COM_REPOS) 180 | .path(repository).path("/" + githubEndpoint.toString().toLowerCase()) 181 | .build().encode() 182 | .toUriString(); 183 | } 184 | LOG.debug("URL to get the last commits page number:" + url); 185 | HttpHeaders headers = restTemplate.headForHeaders(url); 186 | String link = headers.getFirst("Link"); 187 | LOG.debug("Link: " + link); 188 | LOG.debug("parse link by regexp"); 189 | Pattern p = Pattern.compile("page=(\\d*)>; rel=\"last\""); 190 | int lastPageNum = 0; 191 | try { 192 | Matcher m = p.matcher(link); 193 | if (m.find()) { 194 | lastPageNum = Integer.valueOf(m.group(1)); 195 | LOG.debug("parse result: " + lastPageNum); 196 | } 197 | } catch (NullPointerException npe) { 198 | // npe.printStackTrace(); 199 | LOG.info("Propably " + repository + "commits consists from only one page"); 200 | return 1; 201 | } 202 | return lastPageNum; 203 | } catch (Exception e) { 204 | throw new GitHubRESTApiException(" Can't access GitHub REST ", e); 205 | } 206 | } 207 | } 208 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/additional-spring-configuration-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "properties": [ 3 | { 4 | "name": "token.file", 5 | "type": "java.lang.String", 6 | "description": "the absolute path to a file with a GitHub OAuth token. Example '/var/token.txt'." 7 | } 8 | ] 9 | } -------------------------------------------------------------------------------- /src/main/resources/application-dev.properties: -------------------------------------------------------------------------------- 1 | logging.level.org.springframework.web=warn 2 | logging.level.com.rhcloud=debug 3 | spring.cache.type=none 4 | #enable in production for actuator features 5 | #management.context-path=/manage 6 | spring.data.mongodb.uri=mongodb://localhost/test 7 | spring.data.mongodb.repositories.enabled=true 8 | 9 | # Enable constant static resources reloading during development 10 | #spring.resources.static-locations=file:./src/main/resources/static/ 11 | spring.resources.cache-period=0 -------------------------------------------------------------------------------- /src/main/resources/application-docker.properties: -------------------------------------------------------------------------------- 1 | #MongoDB settings 2 | spring.data.mongodb.host=mongodb 3 | -------------------------------------------------------------------------------- /src/main/resources/application-openshift.properties: -------------------------------------------------------------------------------- 1 | token.file=/var/lib/openshift/57c7040389f5cfa0540001c3/diy/var/token.txt 2 | logging.level.org.springframework.web=info 3 | logging.level.com.rhcloud=debug 4 | mongodb_db_password=${OPENSHIFT_MONGODB_DB_PASSWORD} 5 | mongodb_db_host=${OPENSHIFT_MONGODB_DB_HOST} 6 | mongodb_db_username=${OPENSHIFT_MONGODB_DB_USERNAME} 7 | mongodb_database=${OPENSHIFT_APP_NAME} 8 | mongodb_db_port=${OPENSHIFT_MONGODB_DB_PORT} -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.data.mongodb.repositories.enabled=true 2 | #DEFAULT PROFILE - application-dev.properties 3 | spring.profiles.active=dev 4 | token.file=/var/token.txt 5 | #enable working under a reverse proxy 6 | server.tomcat.remote_ip_header=x-forwarded-for 7 | server.tomcat.protocol_header=x-forwarded-proto 8 | 9 | -------------------------------------------------------------------------------- /src/main/resources/mockWeekStargazers.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Stars", 4 | "data": [ 5 | 42.4, 6 | 33.2, 7 | 34.5, 8 | 39.7, 9 | 52.6, 10 | 46.8, 11 | 51.1 12 | ] 13 | } 14 | ] -------------------------------------------------------------------------------- /src/main/resources/monthStargazersFrequencyForHighChart.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Stars", 4 | "data": [ 5 | 0, 6 | 0, 7 | 0, 8 | 0, 9 | 0, 10 | 0, 11 | 0, 12 | 0, 13 | 0, 14 | 14, 15 | 18, 16 | 7, 17 | 3, 18 | 2, 19 | 4, 20 | 3, 21 | 2, 22 | 2, 23 | 1, 24 | 1, 25 | 1, 26 | 0, 27 | 0, 28 | 0, 29 | 0, 30 | 0, 31 | 0, 32 | 0, 33 | 0, 34 | 0, 35 | 0 36 | ] 37 | } 38 | ] 39 | -------------------------------------------------------------------------------- /src/main/resources/static/custom.css: -------------------------------------------------------------------------------- 1 | 2 | #about:hover { 3 | color: #4078c0; 4 | cursor: pointer; 5 | } 6 | 7 | #about { 8 | font-size: 14px; 9 | color: black; 10 | font-weight: 600; 11 | } 12 | 13 | #requestsLeft{ 14 | color: black; 15 | text-align: right; 16 | } 17 | 18 | footer { 19 | margin-top: 20px; 20 | } 21 | 22 | footer a { 23 | font-size: 85%; 24 | } 25 | #button{ 26 | display: inline-block; 27 | } 28 | #current-month-interval { 29 | display: inline-block; 30 | padding-left: 3px; 31 | padding-right: 3px; 32 | } 33 | .prevDate,.nexDate{ 34 | display: inline-block; 35 | padding-left: 5px; 36 | padding-right: 5px; 37 | } 38 | 39 | .prevDate:hover,.nexDate:hover{ 40 | background-color: #e7e7e7; 41 | border-radius: 1px; 42 | border-color: #080808; 43 | } 44 | 45 | -------------------------------------------------------------------------------- /src/main/resources/static/customScript.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This file include all JavaScript methods that renders UI 3 | * and sends ajax requests to backend REST endpoints 4 | */ 5 | 6 | //Global variables 7 | var sinceMonth = new Date(); 8 | var untilMonth = new Date(); 9 | 10 | var sinceWeek = new Date(); 11 | var untilWeek = new Date(); 12 | 13 | /** 14 | * Performs a call to the backend GET "/getRequestsLeft" and displays a number of requests that left on UI 15 | */ 16 | function renderNumberOfRequestsLeft() { 17 | $.ajax({ 18 | url: "getRequestsNumberLeft", 19 | success: function (data) { 20 | document.getElementById('requestsLeft').innerHTML = "Requests Left: " + data; 21 | } 22 | }) 23 | } 24 | 25 | /** 26 | * Display a charts based on analytics result. 27 | * 1. performs a REST call to analyze stargazers|commits|stars per week 28 | * 2. performs a REST call to analyze stargazers|commits|stars per month 29 | * implicit parameters are: currently selected or clicked tab on UI and currently selected month on UI 30 | */ 31 | function analyze(e) { 32 | var inputValue = $('#projectName').val(); 33 | console.log("repository to analyse: " + inputValue); 34 | var analyticsArea; 35 | if (e === undefined || $(this).attr('id') == 'analyze-btn') { 36 | analyticsArea = $('ul.nav-tabs .active').attr('id'); 37 | console.log("active tab: " + analyticsArea); 38 | } 39 | else { 40 | analyticsArea = $(this).attr('id'); 41 | console.log("clicked tab: " + analyticsArea); 42 | } 43 | //render frequency chart per week 44 | $.ajax({ 45 | //thoughout front-end development use http://localhost:8080/stargazers" + "?projectName=" + inputValue/stargazers" + "?projectName=" + inputValue 46 | url: inputValue + "/" + analyticsArea + "?startPeriod=" + parseDateToISOString(sinceWeek) + "&endPeriod=" + parseDateToISOString(untilWeek), 47 | beforeSend: function () { 48 | $('#week-frequency-plot') 49 | .html("

Crunching the latest sinceMonth, just for you.
"); 50 | } 51 | }) 52 | .done(function (msg) { 53 | var response = msg; 54 | $('#week-frequency-plot').highcharts({ 55 | chart: { 56 | type: 'line', 57 | height: '200' 58 | }, 59 | legend: { 60 | enabled: false 61 | }, 62 | title: { 63 | style: { 64 | "display": "none" 65 | } 66 | }, 67 | xAxis: { 68 | categories: [ 69 | 'Mon', 70 | 'Tue', 71 | 'Wed', 72 | 'Thu', 73 | 'Fri', 74 | 'Sat', 75 | 'Sun' 76 | ] 77 | }, 78 | yAxis: { 79 | min: 0, 80 | tickInterval: 1 81 | }, 82 | tooltip: { 83 | headerFormat: '{point.key}', 84 | pointFormat: '' 85 | + 86 | '', 87 | footerFormat: '
{series.name}: {point.y:.1f}
', 88 | shared: true, 89 | useHTML: true 90 | }, 91 | plotOptions: { 92 | series: { 93 | color: '#1db34f' 94 | } 95 | }, 96 | series: response 97 | }); 98 | }) 99 | .fail(function (jqXHR) { 100 | $('#week-frequency-plot') 101 | .html("