├── Procfile ├── src └── org │ ├── loklak │ ├── server │ │ ├── WeiboUserInfo.java │ │ ├── HttpsMode.java │ │ ├── BaseUserRole.java │ │ ├── APIException.java │ │ ├── APIHandler.java │ │ └── Client.java │ ├── http │ │ └── CookieRequest.java │ ├── data │ │ ├── Insertable.java │ │ └── IndexEntry.java │ ├── objects │ │ ├── ObjectEntry.java │ │ ├── Peers.java │ │ ├── Peer.java │ │ ├── ProviderType.java │ │ └── ResultList.java │ ├── tools │ │ ├── storage │ │ │ ├── JsonFactory.java │ │ │ ├── JsonReader.java │ │ │ └── StorageMetadata.java │ │ └── CacheStats.java │ ├── stream │ │ ├── StreamServlet.java │ │ └── MqttEventSource.java │ ├── geo │ │ ├── LocationSource.java │ │ └── AbstractGeoPoint.java │ ├── ir │ │ ├── UserFactory.java │ │ ├── QueryFactory.java │ │ ├── MessageFactory.java │ │ ├── AccountFactory.java │ │ ├── BulkWriteResult.java │ │ └── ImportProfileFactory.java │ ├── api │ │ ├── cms │ │ │ ├── TopMenuService.java │ │ │ └── SettingsManagementService.java │ │ └── iot │ │ │ ├── FreifunkNodePushServlet.java │ │ │ ├── OpenWifiMapPushServlet.java │ │ │ └── PushReport.java │ └── harvester │ │ └── HarvestingFrequency.java │ └── json │ ├── JSONPointerException.java │ ├── JSONPropertyIgnore.java │ ├── JSONString.java │ └── JSONPropertyName.java ├── system.properties ├── settings.gradle ├── html ├── robots.txt ├── images │ ├── icon.png │ ├── favicon.png │ ├── loklak_org.png │ ├── loklak_share.png │ ├── loklak_collect.png │ ├── loklak_anonymous.png │ ├── loklak_org_black.png │ ├── concept_searchengine.png │ ├── component_jsonlistp2p.png │ ├── component_searchindex.png │ ├── concept_messagesearch.png │ ├── forkme_right_green_007200.png │ ├── component_contentharvester.png │ └── concept_messagesearchdetail.png ├── fonts │ ├── DroidSansMono.eot │ ├── DroidSansMono.ttf │ ├── DroidSansMono.woff │ ├── glyphicons-halflings-regular.eot │ ├── glyphicons-halflings-regular.ttf │ └── glyphicons-halflings-regular.woff ├── css │ └── morris.css ├── js │ ├── ie10-viewport-bug-workaround.js │ └── angular-modules │ │ ├── ngStorage.min.js │ │ └── angular-momentjs.min.js └── settings.html ├── conf ├── http_auth ├── keystore.jks ├── config.properties ├── conversion │ ├── fossasia.json │ ├── nodelist-node.json │ ├── netmon-node.json │ ├── freifunk-node.json │ └── openwifimap.json ├── templates │ ├── reset-mail.txt │ └── verification-mail.txt ├── keystore.readme ├── logs │ ├── log4j2.properties │ └── log-to-file.properties ├── elasticsearch │ └── mappings │ │ ├── accounts.json │ │ ├── users.json │ │ ├── import_profiles.json │ │ └── queries.json └── schema │ ├── openwifimap.json │ └── freifunk-node.json ├── installation ├── favicon.png ├── fonts │ ├── DroidSansMono.ttf │ ├── glyphicons-halflings-regular.eot │ ├── glyphicons-halflings-regular.ttf │ └── glyphicons-halflings-regular.woff └── css │ └── loklak.css ├── test ├── queries │ ├── polish_00.txt │ ├── README.txt │ ├── chinese_01.txt │ ├── cantonese_00.txt │ ├── chinese_00.txt │ ├── spanish_01.txt │ ├── german_00.txt │ └── english_03.txt ├── org │ ├── loklak │ │ ├── tools │ │ │ ├── storage │ │ │ │ ├── JsonMinifierTest.java │ │ │ │ ├── JsonFileTest.java │ │ │ │ ├── CommonPatternTest.java │ │ │ │ ├── AsciiTest.java │ │ │ │ └── JsonDatasetTest.java │ │ │ └── UTF8Test.java │ │ ├── http │ │ │ └── ClientConnectionTest.java │ │ ├── TestRunner.java │ │ ├── geo │ │ │ └── GeoLocationTest.java │ │ ├── harvester │ │ │ └── RedirectUnshortenerTest.java │ │ ├── api │ │ │ └── search │ │ │ │ └── WordpressCrawlerServiceTest.java │ │ └── data │ │ │ └── ElasticsearchClientTest.java │ └── json │ │ └── JSONObjectTest.java └── api_test.sh ├── kubernetes ├── yamls │ ├── lego │ │ ├── 00-namespace.yml │ │ ├── configmap.yml │ │ └── deployment.yaml │ ├── api │ │ ├── web │ │ │ ├── 00-namespace.yml │ │ │ ├── ingress-notls.yml │ │ │ └── ingress-tls.yml │ │ ├── elasticsearch │ │ │ ├── 00-namespace.yml │ │ │ ├── service-account.yaml │ │ │ ├── es-service.yaml │ │ │ └── es-deployment.yaml │ │ └── api-server │ │ │ ├── configmap.yml │ │ │ ├── api-service.yml │ │ │ └── api-deployment.yml │ ├── mosquitto │ │ ├── 00-namespace.yaml │ │ ├── mqtt-service-account.yaml │ │ ├── mqtt-service.yaml │ │ └── mqtt-deployment.yaml │ ├── nginx │ │ ├── 00-namespace.yml │ │ ├── default-service.yml │ │ ├── service.yml │ │ ├── configmap.yml │ │ ├── default-deployment.yml │ │ └── deployment.yml │ └── staging │ │ ├── web │ │ ├── 00-namespace.yml │ │ ├── ingress-notls.yml │ │ └── ingress-tls.yml │ │ ├── elasticsearch │ │ ├── 00-namespace.yml │ │ ├── service-account.yaml │ │ ├── es-service.yaml │ │ └── es-deployment.yaml │ │ └── api-server │ │ ├── configmap.yml │ │ ├── api-service.yml │ │ └── api-deployment.yml ├── gcloud-credentials.json.enc ├── bin │ ├── update-image.sh │ ├── update-deployment-travis.sh │ └── deploy.sh └── images │ ├── staging │ └── Dockerfile │ └── api │ └── Dockerfile ├── bin ├── restart.sh ├── heroku_start.sh ├── kill.sh ├── init.sh ├── upgrade.sh ├── build.sbt ├── search_loklak_api.sh ├── search.sh ├── test.sh ├── start.bat ├── stop.sh ├── utility.sh ├── search_nerdwithak.py ├── apicall.sh ├── search.rb ├── search.py ├── search_with_usernames.py ├── checkalive.sh ├── search1.pl ├── test_piotr.py ├── search.pl ├── search_fossasia.py ├── bluemix_start.sh ├── profile.py ├── search(python2&3).py ├── generate_test_queries.py ├── profile_queries.py ├── installation.sh └── .preload.sh ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── .settings ├── org.eclipse.core.resources.prefs ├── org.eclipse.jdt.ui.prefs └── org.eclipse.buildship.core.prefs ├── docs ├── _assets │ ├── component_jsonlistp2p.png │ ├── component_searchindex.png │ ├── concept_messagesearch.png │ ├── concept_searchengine.png │ ├── component_contentharvester.png │ └── concept_messagesearchdetail.png ├── misc │ ├── _assets │ │ ├── architecture_phase1.png │ │ ├── architecture_phase2.png │ │ ├── architecture_phase3.png │ │ ├── architecture_topology.png │ │ └── architecture_large_scaling.png │ ├── stream.html │ ├── StreamChannels.md │ └── StreamExample.md ├── installation │ ├── installation_linux.md │ ├── installation_mac.md │ ├── installation_cloud9.md │ ├── installation_heroku.md │ ├── deploying-development-kubernetes.md │ └── tutorials.md ├── development │ └── eclipseSetup.md └── Dockerfile-learnings.md ├── docker-cloud.yml ├── ssi ├── common_body_back.include └── common_head_front.include ├── .yaydoc.yml ├── .editorconfig ├── azuredeploy.parameters.json ├── docker-compose.yml ├── metadata.json ├── .gitignore ├── cloud9-setup.sh ├── manifest.yml ├── .github ├── PULL_REQUEST_TEMPLATE.md └── ISSUE_TEMPLATE.md ├── app.json ├── scalingo.json ├── .project ├── .flake8 ├── res └── log4j2.properties ├── .classpath ├── .bluemix └── pipeline.yml ├── .travis.yml ├── .utility ├── docker-compose.yml └── push-docker.sh ├── Dockerfile ├── Dockerfile-arm32v6 └── Dockerfile-arm32v7 /Procfile: -------------------------------------------------------------------------------- 1 | web: sh bin/heroku_start.sh 2 | -------------------------------------------------------------------------------- /src/org/loklak/server/WeiboUserInfo.java: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /system.properties: -------------------------------------------------------------------------------- 1 | java.runtime.version=1.8 2 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'loklak_server' 2 | -------------------------------------------------------------------------------- /html/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Sitemap: http://loklak.org/api/sitemap.xml 3 | -------------------------------------------------------------------------------- /conf/http_auth: -------------------------------------------------------------------------------- 1 | # test : test 2 | test: MD5:098f6bcd4621d373cade4e832627b4f6,user 3 | -------------------------------------------------------------------------------- /conf/keystore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/conf/keystore.jks -------------------------------------------------------------------------------- /html/images/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/images/icon.png -------------------------------------------------------------------------------- /conf/config.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/conf/config.properties -------------------------------------------------------------------------------- /html/images/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/images/favicon.png -------------------------------------------------------------------------------- /html/images/loklak_org.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/images/loklak_org.png -------------------------------------------------------------------------------- /installation/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/installation/favicon.png -------------------------------------------------------------------------------- /test/queries/polish_00.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/test/queries/polish_00.txt -------------------------------------------------------------------------------- /html/fonts/DroidSansMono.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/fonts/DroidSansMono.eot -------------------------------------------------------------------------------- /html/fonts/DroidSansMono.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/fonts/DroidSansMono.ttf -------------------------------------------------------------------------------- /html/images/loklak_share.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/images/loklak_share.png -------------------------------------------------------------------------------- /kubernetes/yamls/lego/00-namespace.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: kube-lego 5 | -------------------------------------------------------------------------------- /conf/conversion/fossasia.json: -------------------------------------------------------------------------------- 1 | { 2 | "rules" : 3 | [ 4 | ["name", "text"], 5 | ["url", "link"] 6 | ] 7 | } -------------------------------------------------------------------------------- /html/fonts/DroidSansMono.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/fonts/DroidSansMono.woff -------------------------------------------------------------------------------- /html/images/loklak_collect.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/images/loklak_collect.png -------------------------------------------------------------------------------- /kubernetes/yamls/api/web/00-namespace.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: api-web 5 | -------------------------------------------------------------------------------- /kubernetes/yamls/mosquitto/00-namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: mqtt 5 | -------------------------------------------------------------------------------- /kubernetes/yamls/nginx/00-namespace.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: nginx-ingress 5 | -------------------------------------------------------------------------------- /bin/restart.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | cd `dirname $0`/.. 3 | echo "re-starting loklak" 4 | bin/stop.sh 5 | bin/start.sh 6 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /html/images/loklak_anonymous.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/images/loklak_anonymous.png -------------------------------------------------------------------------------- /html/images/loklak_org_black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/images/loklak_org_black.png -------------------------------------------------------------------------------- /kubernetes/yamls/staging/web/00-namespace.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: staging-web 5 | -------------------------------------------------------------------------------- /conf/conversion/nodelist-node.json: -------------------------------------------------------------------------------- 1 | { 2 | "rules" : 3 | [ 4 | ["name", "text"], 5 | ["href", "link"] 6 | ] 7 | } -------------------------------------------------------------------------------- /html/images/concept_searchengine.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/images/concept_searchengine.png -------------------------------------------------------------------------------- /installation/fonts/DroidSansMono.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/installation/fonts/DroidSansMono.ttf -------------------------------------------------------------------------------- /.settings/org.eclipse.core.resources.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | encoding//src/org/loklak/LoklakServer.java=UTF-8 3 | -------------------------------------------------------------------------------- /docs/_assets/component_jsonlistp2p.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/docs/_assets/component_jsonlistp2p.png -------------------------------------------------------------------------------- /docs/_assets/component_searchindex.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/docs/_assets/component_searchindex.png -------------------------------------------------------------------------------- /docs/_assets/concept_messagesearch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/docs/_assets/concept_messagesearch.png -------------------------------------------------------------------------------- /docs/_assets/concept_searchengine.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/docs/_assets/concept_searchengine.png -------------------------------------------------------------------------------- /html/images/component_jsonlistp2p.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/images/component_jsonlistp2p.png -------------------------------------------------------------------------------- /html/images/component_searchindex.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/images/component_searchindex.png -------------------------------------------------------------------------------- /html/images/concept_messagesearch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/images/concept_messagesearch.png -------------------------------------------------------------------------------- /kubernetes/gcloud-credentials.json.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/kubernetes/gcloud-credentials.json.enc -------------------------------------------------------------------------------- /kubernetes/yamls/api/elasticsearch/00-namespace.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: elasticsearch 5 | -------------------------------------------------------------------------------- /conf/conversion/netmon-node.json: -------------------------------------------------------------------------------- 1 | { 2 | "rules" : 3 | [ 4 | ["description", "text"], 5 | ["update_date", "mtime"] 6 | ] 7 | } -------------------------------------------------------------------------------- /docker-cloud.yml: -------------------------------------------------------------------------------- 1 | web: 2 | name: "loklak" 3 | image: loklak/loklak_server:latest-master 4 | ports: 5 | - "80" 6 | - "443" 7 | -------------------------------------------------------------------------------- /docs/misc/_assets/architecture_phase1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/docs/misc/_assets/architecture_phase1.png -------------------------------------------------------------------------------- /docs/misc/_assets/architecture_phase2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/docs/misc/_assets/architecture_phase2.png -------------------------------------------------------------------------------- /docs/misc/_assets/architecture_phase3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/docs/misc/_assets/architecture_phase3.png -------------------------------------------------------------------------------- /html/images/forkme_right_green_007200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/images/forkme_right_green_007200.png -------------------------------------------------------------------------------- /kubernetes/yamls/staging/elasticsearch/00-namespace.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: elasticsearch 5 | -------------------------------------------------------------------------------- /docs/_assets/component_contentharvester.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/docs/_assets/component_contentharvester.png -------------------------------------------------------------------------------- /docs/misc/_assets/architecture_topology.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/docs/misc/_assets/architecture_topology.png -------------------------------------------------------------------------------- /html/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /html/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /html/images/component_contentharvester.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/images/component_contentharvester.png -------------------------------------------------------------------------------- /html/images/concept_messagesearchdetail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/images/concept_messagesearchdetail.png -------------------------------------------------------------------------------- /kubernetes/yamls/api/api-server/configmap.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: server 5 | namespace: api-web 6 | -------------------------------------------------------------------------------- /bin/heroku_start.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Make sure we're on project root 4 | cd $(dirname $0)/.. 5 | 6 | exec ./bin/start.sh -Idn 7 | -------------------------------------------------------------------------------- /bin/kill.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | cd `dirname $0`/../data 3 | kill -9 `cat loklak.pid` 2>/dev/null 4 | rm -f loklak.pid 2>/dev/null 5 | 6 | 7 | -------------------------------------------------------------------------------- /docs/_assets/concept_messagesearchdetail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/docs/_assets/concept_messagesearchdetail.png -------------------------------------------------------------------------------- /html/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/html/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /conf/conversion/freifunk-node.json: -------------------------------------------------------------------------------- 1 | { 2 | "rules" : 3 | [ 4 | ["name", "text"], 5 | ["geo", ["location_point", "location_mark"]] 6 | ] 7 | } -------------------------------------------------------------------------------- /docs/misc/_assets/architecture_large_scaling.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/docs/misc/_assets/architecture_large_scaling.png -------------------------------------------------------------------------------- /kubernetes/yamls/staging/api-server/configmap.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: server 5 | namespace: staging-web 6 | -------------------------------------------------------------------------------- /installation/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/installation/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /installation/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/installation/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /installation/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loklak/loklak_server/HEAD/installation/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /kubernetes/yamls/mosquitto/mqtt-service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: mosquitto 5 | namespace: mqtt 6 | 7 | -------------------------------------------------------------------------------- /kubernetes/yamls/api/elasticsearch/service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: elasticsearch 5 | namespace: elasticsearch 6 | -------------------------------------------------------------------------------- /bin/init.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | cd `dirname $0`/.. 3 | mkdir -p data 4 | mkdir -p data/settings 5 | cp conf/config.properties data/settings/customized_config.properties 6 | -------------------------------------------------------------------------------- /kubernetes/yamls/staging/elasticsearch/service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: elasticsearch 5 | namespace: elasticsearch 6 | -------------------------------------------------------------------------------- /bin/upgrade.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | cd `dirname $0`/.. 3 | echo "loading latest code changes" 4 | git pull -r 5 | echo "assembling loklak" 6 | ./gradlew assemble 7 | bin/restart.sh 8 | -------------------------------------------------------------------------------- /bin/build.sbt: -------------------------------------------------------------------------------- 1 | 2 | name := "loklak-query-profile" 3 | organization := "org.loklak" 4 | version := "1.0" 5 | scalaVersion := "2.11.7" 6 | libraryDependencies += "org.scalaj" %% "scalaj-http" % "2.2.0" 7 | -------------------------------------------------------------------------------- /conf/conversion/openwifimap.json: -------------------------------------------------------------------------------- 1 | { 2 | "rules" : 3 | [ 4 | ["value.hostname", "text"], 5 | ["value.latlng", ["location_point", "location_mark"]], 6 | ["value.mtime", "mtime"] 7 | ] 8 | } -------------------------------------------------------------------------------- /src/org/loklak/http/CookieRequest.java: -------------------------------------------------------------------------------- 1 | package org.loklak.http; 2 | 3 | public interface CookieRequest { 4 | public CookieRequest makeRequest(); 5 | public String body(); 6 | public String cookie(); 7 | } 8 | -------------------------------------------------------------------------------- /ssi/common_body_back.include: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /conf/templates/reset-mail.txt: -------------------------------------------------------------------------------- 1 | You just requested for a password reset for your loklak account. 2 | 3 | To do so, click on the following link: 4 | 5 | %RESET-LINK% 6 | 7 | If you didn't ask for a password reset, please ignore this email. 8 | -------------------------------------------------------------------------------- /.settings/org.eclipse.jdt.ui.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.ui.exception.name=e 3 | org.eclipse.jdt.ui.gettersetter.use.is=true 4 | org.eclipse.jdt.ui.keywordthis=false 5 | org.eclipse.jdt.ui.overrideannotation=true 6 | -------------------------------------------------------------------------------- /kubernetes/yamls/lego/configmap.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | metadata: 3 | name: kube-lego 4 | namespace: kube-lego 5 | data: 6 | lego.email: "info@loklak.org" 7 | lego.url: "https://acme-v01.api.letsencrypt.org/directory" 8 | kind: ConfigMap 9 | -------------------------------------------------------------------------------- /bin/search_loklak_api.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | curl 'loklak.org/api/search.json?q='$1 | python -c " 3 | import json 4 | import sys 5 | raw_data = sys.stdin 6 | json_data = json.load(raw_data) 7 | for t in json_data['statuses']: 8 | print(t['text']+'\n')" -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /bin/search.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This script needs jq 3 | # If you are using OSX, then you can use brew install jq 4 | # And if you are using Ubuntu, then use apt-get install jq 5 | cd "`dirname $0`" 6 | ./apicall.sh "api/search.json?q=$1" | jq -r '.statuses[].text' 7 | -------------------------------------------------------------------------------- /conf/templates/verification-mail.txt: -------------------------------------------------------------------------------- 1 | You just signed up for Loklak! 2 | 3 | To finish the process, you have to verify you email address by clicking on the following link: 4 | 5 | %VERIFICATION-LINK% 6 | 7 | If you didn't sign up for Loklak, please ignore this email. 8 | -------------------------------------------------------------------------------- /bin/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | cd $(dirname $0)/.. 3 | 4 | # Execute preload script 5 | source bin/.preload.sh 6 | 7 | echo "starting Tests" 8 | 9 | java -Xmx1024m -classpath .:build/libs/loklak_server-all.jar:build/classes/test org.junit.runner.JUnitCore org.loklak.TestRunner 10 | -------------------------------------------------------------------------------- /bin/start.bat: -------------------------------------------------------------------------------- 1 | cd .. 2 | START java -Xmx4G -Xms1G -server -XX:+AggressiveOpts -XX:NewSize=512M -cp "build/libs/loklak_server-all.jar" org.loklak.LoklakServer >> data/loklak.log 2>&1 & echo $! > data/loklak.pid 3 | echo "loklak server started at port 9000, open your browser at http://localhost:9000" 4 | -------------------------------------------------------------------------------- /kubernetes/yamls/nginx/default-service.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: default-http-backend 5 | namespace: nginx-ingress 6 | spec: 7 | ports: 8 | - port: 80 9 | targetPort: 8080 10 | protocol: TCP 11 | selector: 12 | app: default-http-backend 13 | -------------------------------------------------------------------------------- /conf/keystore.readme: -------------------------------------------------------------------------------- 1 | the keystore was generated with 2 | keytool -genkey -alias sitename -keyalg RSA -keystore keystore.jks -keysize 2048 3 | the password is 123456 4 | 5 | you can customize/replace the keystore. Just re-run the keytool, use a different 6 | password and then set the password in the properties file. -------------------------------------------------------------------------------- /kubernetes/yamls/api/api-server/api-service.yml: -------------------------------------------------------------------------------- 1 | # Expose port 80 using load balancer 2 | kind: Service 3 | apiVersion: v1 4 | metadata: 5 | name: server 6 | namespace: api-web 7 | spec: 8 | ports: 9 | - port: 80 10 | protocol: TCP 11 | targetPort: 80 12 | selector: 13 | app: server 14 | 15 | -------------------------------------------------------------------------------- /kubernetes/yamls/staging/api-server/api-service.yml: -------------------------------------------------------------------------------- 1 | # Expose port 80 using load balancer 2 | kind: Service 3 | apiVersion: v1 4 | metadata: 5 | name: server 6 | namespace: staging-web 7 | spec: 8 | ports: 9 | - port: 80 10 | protocol: TCP 11 | targetPort: 80 12 | selector: 13 | app: server 14 | 15 | -------------------------------------------------------------------------------- /.yaydoc.yml: -------------------------------------------------------------------------------- 1 | metadata: 2 | projectname: loklak_server 3 | author: FOSSASIA 4 | version: development 5 | autoindex: 6 | apidoc: 7 | javadoc: 8 | show: true 9 | 10 | build: 11 | source: docs 12 | theme: 13 | name: sphinx_fossasia_theme 14 | 15 | extras: 16 | javadoc: 17 | path: 'src/' 18 | -------------------------------------------------------------------------------- /.settings/org.eclipse.buildship.core.prefs: -------------------------------------------------------------------------------- 1 | build.commands=org.eclipse.jdt.core.javabuilder 2 | connection.arguments= 3 | connection.java.home=null 4 | connection.jvm.arguments= 5 | connection.project.dir= 6 | derived.resources=.gradle,build 7 | eclipse.preferences.version=1 8 | natures=org.eclipse.jdt.core.javanature 9 | project.path=\: 10 | -------------------------------------------------------------------------------- /bin/stop.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | cd `dirname $0`/../data 3 | echo "shut down of loklak" 4 | # If you don't want to wait, just run this concurrently 5 | kill $(cat loklak.pid 2>/dev/null) 2>/dev/null 6 | if [ $? -eq 0 ]; then while [ -f "loklak.pid" ]; do sleep 1; done; fi; 7 | rm -f loklak.pid 2>/dev/null 8 | echo "loklak has been terminated" 9 | -------------------------------------------------------------------------------- /bin/utility.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script contains only functions that can be used in other bash scripts 4 | 5 | function change_shortlink_urlstub() { 6 | # changes port_number in shortlink.urlstub=http://localhost:port_number 7 | sed -i 's/\(shortlink\.urlstub=http:.*:\)\(.*\)/\1'"$1"'/' conf/config.properties 8 | } 9 | 10 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | # Global settings 4 | [*] 5 | indent_style = space 6 | end_of_line = lf 7 | charset = utf-8 8 | 9 | # Specially for java 10 | [*.java] 11 | indent_size = 4 12 | insert_final_newline = true 13 | trim_trailing_whitespace = true 14 | 15 | 16 | # Batch files use tabs for indentation 17 | [*.bat] 18 | indent_style = tab 19 | -------------------------------------------------------------------------------- /azuredeploy.parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "name": { 6 | "value": "loklakserver" 7 | }, 8 | "image": { 9 | "value": "loklak/loklak_server:latest-master" 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /kubernetes/yamls/mosquitto/mqtt-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: mosquitto 5 | namespace: mqtt 6 | labels: 7 | component: mosquitto 8 | spec: 9 | selector: 10 | component: mosquitto 11 | ports: 12 | - name: mosquitto 13 | port: 1883 14 | - name: mosquitto-web 15 | port: 80 16 | targetPort: 9001 17 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | loklak: 5 | build: . 6 | ports: 7 | - "80:80" 8 | #Map volume-folder 9 | volumes: 10 | - data:/loklak_server/data 11 | 12 | #Create named volume 13 | volumes: 14 | data: 15 | external: false 16 | -------------------------------------------------------------------------------- /kubernetes/yamls/nginx/service.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: nginx 5 | namespace: nginx-ingress 6 | annotations: 7 | service.beta.kubernetes.io/external-traffic: "OnlyLocal" 8 | spec: 9 | type: LoadBalancer 10 | ports: 11 | - port: 80 12 | name: http 13 | - port: 443 14 | name: https 15 | selector: 16 | app: nginx 17 | -------------------------------------------------------------------------------- /metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "itemDisplayName": "loklak Cloud Instance - Linux container with public IP", 3 | "description": "Deploy a single Linux container accessible via a public IP using Azure Container Instances.", 4 | "summary": "This will create a container group with a single Linux container and a publid IP address.", 5 | "githubUsername": "kavithaenair", 6 | "dateUpdated": "2017-08-18" 7 | } 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /todo.txt 2 | /classes/ 3 | /data/ 4 | 5 | /.idea 6 | *.iml 7 | /hs_err_*.log 8 | /data.bkp/ 9 | .merge_file_* 10 | *.xml~ 11 | 12 | # python 13 | venv/* 14 | 15 | dist 16 | dist/loklak.jar 17 | /build/ 18 | /.gradle/ 19 | .DS_Store 20 | 21 | /docs/_build 22 | .DS_Store 23 | 24 | bin/org/ 25 | bin/queries/ 26 | bin/log4j2.properties 27 | bin/api_test.sh 28 | .sts4-cache/ 29 | .vscode/ 30 | -------------------------------------------------------------------------------- /bin/search_nerdwithak.py: -------------------------------------------------------------------------------- 1 | # Uses Python3 2 | import urllib.request 3 | import json 4 | import sys 5 | 6 | # Get JSON file 7 | data = urllib.request.urlopen("127.0.0.1:9000/api/search.json?q=%s" % (sys.argv[1])) 8 | 9 | # Read and parse JSON 10 | data = data.read().decode('utf-8') 11 | data = json.loads(data) 12 | 13 | # Print out tweets 14 | for tweet in data["statuses"]: 15 | print(tweet["text"]) 16 | -------------------------------------------------------------------------------- /kubernetes/yamls/api/elasticsearch/es-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: elasticsearch 5 | namespace: elasticsearch 6 | labels: 7 | component: elasticsearch 8 | spec: 9 | selector: 10 | component: elasticsearch 11 | ports: 12 | - name: http 13 | port: 9200 14 | protocol: TCP 15 | - name: transport 16 | port: 9300 17 | protocol: TCP 18 | -------------------------------------------------------------------------------- /kubernetes/yamls/api/web/ingress-notls.yml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Ingress 3 | metadata: 4 | name: web-notls 5 | namespace: api-web 6 | annotations: 7 | kubernetes.io/ingress.class: "nginx" 8 | spec: 9 | rules: 10 | - host: api.loklak.org 11 | http: 12 | paths: 13 | - path: / 14 | backend: 15 | serviceName: server 16 | servicePort: 80 17 | -------------------------------------------------------------------------------- /kubernetes/yamls/staging/elasticsearch/es-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: elasticsearch 5 | namespace: elasticsearch 6 | labels: 7 | component: elasticsearch 8 | spec: 9 | selector: 10 | component: elasticsearch 11 | ports: 12 | - name: http 13 | port: 9200 14 | protocol: TCP 15 | - name: transport 16 | port: 9300 17 | protocol: TCP 18 | -------------------------------------------------------------------------------- /bin/apicall.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | cd "`dirname $0`" 3 | port=$(grep ^port.http= ../data/settings/customized_config.properties |cut -d= -f2) 4 | if [ -z "$port" ]; then port=9000; fi 5 | 6 | if which curl &>/dev/null; then 7 | curl -m 60 -s "http://127.0.0.1:$port/$1" 8 | elif which wget &>/dev/null; then 9 | wget -q -t 1 --timeout=120 "http://127.0.0.1:$port/$1" -O - 10 | else 11 | exit 1 12 | fi 13 | 14 | -------------------------------------------------------------------------------- /kubernetes/yamls/staging/web/ingress-notls.yml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Ingress 3 | metadata: 4 | name: web-notls 5 | namespace: staging-web 6 | annotations: 7 | kubernetes.io/ingress.class: "nginx" 8 | spec: 9 | rules: 10 | - host: staging.loklak.org 11 | http: 12 | paths: 13 | - path: / 14 | backend: 15 | serviceName: server 16 | servicePort: 80 17 | -------------------------------------------------------------------------------- /cloud9-setup.sh: -------------------------------------------------------------------------------- 1 | sudo apt-get update 2 | sudo apt-get install -y openjdk-8-jdk; git clone https://github.com/loklak/loklak_server.git loklak_server 3 | cd loklak_server; 4 | sed -i.bak 's/^\(port.http=\).*/\180/' conf/config.properties 5 | sed -i.bak 's/^\(port.https=\).*/\1443/' conf/config.properties 6 | sed -i.bak 's/^\(upgradeInterval=\).*/\186400000000/' conf/config.properties 7 | ./gradlew build 8 | bin/start.sh 9 | -------------------------------------------------------------------------------- /html/css/morris.css: -------------------------------------------------------------------------------- 1 | .morris-hover{position:absolute;z-index:1000}.morris-hover.morris-default-style{border-radius:10px;padding:6px;color:#666;background:rgba(255,255,255,0.8);border:solid 2px rgba(230,230,230,0.8);font-family:sans-serif;font-size:12px;text-align:center}.morris-hover.morris-default-style .morris-hover-row-label{font-weight:bold;margin:0.25em 0} 2 | .morris-hover.morris-default-style .morris-hover-point{white-space:nowrap;margin:0.1em 0} 3 | -------------------------------------------------------------------------------- /test/queries/README.txt: -------------------------------------------------------------------------------- 1 | This folder should contain text files named like "language_00.txt" 2 | where the name 'language' shall be replaced by the actual name of 3 | the language, used for the words inside the file. 4 | Each file should contain example queries (i.e. nouns or hashtags) 5 | as you would type them in into a search field. 6 | If several files are generated for the same language, increase 7 | the default counter from '00' to '01' (and so on). 8 | -------------------------------------------------------------------------------- /manifest.yml: -------------------------------------------------------------------------------- 1 | --- 2 | applications: 3 | - name: "Loklak Server" 4 | buildpack: "https://github.com/heroku/heroku-buildpack-gradle.git" 5 | description: "Distributed Tweet Search Server" 6 | image: "loklak/loklak_server:latest-master" 7 | logo: "https://raw.githubusercontent.com/loklak/loklak_server/master/html/images/loklak_anonymous.png" 8 | repository: "https://github.com/loklak/loklak_server.git" 9 | website: "http://api.loklak.org" 10 | -------------------------------------------------------------------------------- /kubernetes/yamls/mosquitto/mqtt-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: mosquitto 5 | namespace: mqtt 6 | spec: 7 | replicas: 1 8 | template: 9 | metadata: 10 | labels: 11 | app: mosquitto 12 | spec: 13 | containers: 14 | - name: mosquitto 15 | image: toke/mosquitto 16 | ports: 17 | - containerPort: 9001 18 | - containerPort: 8883 19 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Short description 2 | I have: 3 | - [ ] There is a corresponding issue for this pull request. 4 | - [ ] Mentioned the Issue number in the pull request commit message `Fixes # commit message` 5 | - [ ] There is only strictly only one commit per issue. 6 | 7 | ### For the reviewers 8 | I have: 9 | - [ ] Reviewed this pull request by an authorized contributor. 10 | - [ ] The reviewer is assigned to the pull request. 11 | -------------------------------------------------------------------------------- /app.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Loklak Server", 3 | "description": "Distributed Tweet Search Server", 4 | "logo": "https://raw.githubusercontent.com/loklak/loklak_server/master/html/images/loklak_anonymous.png", 5 | "website": "http://api.loklak.org", 6 | "repository": "https://github.com/loklak/loklak_server.git", 7 | "image": "loklak/loklak_server:latest-master", 8 | "env": { 9 | "BUILDPACK_URL": "https://github.com/heroku/heroku-buildpack-gradle.git" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /installation/css/loklak.css: -------------------------------------------------------------------------------- 1 | /* Move down content because we have a fixed navbar that is 50px tall */ 2 | @font-face { 3 | font-family: 'Droid Sans Mono'; 4 | font-style: normal; 5 | font-weight: 400; 6 | src: local('Droid Sans Mono'), local('DroidSansMono'), url('../fonts/DroidSansMono.ttf') format('truetype'); 7 | } 8 | 9 | * { 10 | font-family: 'Droid Sans Mono'; 11 | } 12 | 13 | 14 | body { 15 | padding-top: 50px; 16 | font: DroidSansMono; 17 | } 18 | -------------------------------------------------------------------------------- /docs/installation/installation_linux.md: -------------------------------------------------------------------------------- 1 | # Installation on Linux 2 | 3 | Its very easy! 4 | 5 | ## Requirements 6 | - install git, jdk 7 and ant 7 | 8 | ## Download 9 | - `git clone https://github.com/loklak/loklak_server.git` 10 | - `cd loklak_server` 11 | 12 | ## Build 13 | - `ant` (just this, type "ant" - without quotes - and hit enter) 14 | 15 | ## Run 16 | - `bin/start.sh` 17 | 18 | ## Operate 19 | - open `http://localhost:9000` in your browser 20 | 21 | ## Shut down 22 | - `bin/stop.sh` 23 | 24 | -------------------------------------------------------------------------------- /docs/installation/installation_mac.md: -------------------------------------------------------------------------------- 1 | # Installation on Mac OS 2 | 3 | Its very easy! 4 | 5 | ## Requirements 6 | - install git, jdk 7 and ant 7 | 8 | ## Download 9 | - `git clone https://github.com/loklak/loklak_server.git` 10 | - `cd loklak_server` 11 | 12 | ## Build 13 | - `ant` (just this, type "ant" - without quotes - and hit enter) 14 | 15 | ## Run 16 | - `bin/start.sh` 17 | 18 | ## Operate 19 | - open `http://localhost:9000` in your browser 20 | 21 | ## Shut down 22 | - `bin/stop.sh` 23 | 24 | -------------------------------------------------------------------------------- /scalingo.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Loklak Server", 3 | "description": "Distributed Tweet Search Server", 4 | "logo": "https://raw.githubusercontent.com/loklak/loklak_server/master/html/images/loklak_anonymous.png", 5 | "website": "http://api.loklak.org", 6 | "repository": "https://github.com/loklak/loklak_server.git", 7 | "image": "loklak/loklak_server:latest-master", 8 | "env": { 9 | "BUILDPACK_URL": { 10 | "value": "https://github.com/heroku/heroku-buildpack-gradle.git" 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /bin/search.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'net/http' 3 | require 'json' 4 | 5 | if ARGV.length == 0 6 | puts 'Set search query after script name like "./search.rb fossasia"' 7 | exit 8 | end 9 | 10 | url = URI.parse('http://127.0.0.1:9000/api/search.json?q=' + ARGV[0]) 11 | req = Net::HTTP::Get.new(url.to_s) 12 | res = Net::HTTP.start(url.host, url.port) {|http| 13 | http.request(req) 14 | } 15 | 16 | tweets = JSON.parse(res.body) 17 | tweets['statuses'].each do |tweet| 18 | puts tweet['text'] 19 | end 20 | -------------------------------------------------------------------------------- /kubernetes/yamls/nginx/configmap.yml: -------------------------------------------------------------------------------- 1 | # 2 | # nginx ingress controller config 3 | # @ref https://github.com/kubernetes/ingress/blob/master/controllers/nginx/configuration.md 4 | # 5 | apiVersion: v1 6 | data: 7 | proxy-connect-timeout: "15" 8 | proxy-read-timeout: "600" 9 | proxy-send-imeout: "600" 10 | hsts-include-subdomains: "false" 11 | body-size: "64m" 12 | server-name-hash-bucket-size: "256" 13 | server-tokens: "false" 14 | kind: ConfigMap 15 | metadata: 16 | namespace: nginx-ingress 17 | name: nginx 18 | -------------------------------------------------------------------------------- /kubernetes/yamls/api/web/ingress-tls.yml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Ingress 3 | metadata: 4 | name: web 5 | namespace: api-web 6 | annotations: 7 | kubernetes.io/tls-acme: "true" 8 | kubernetes.io/ingress.class: "nginx" 9 | spec: 10 | tls: 11 | - hosts: 12 | - api.loklak.org 13 | secretName: loklak-api-tls 14 | rules: 15 | - host: api.loklak.org 16 | http: 17 | paths: 18 | - path: / 19 | backend: 20 | serviceName: server 21 | servicePort: 80 22 | -------------------------------------------------------------------------------- /kubernetes/yamls/staging/web/ingress-tls.yml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Ingress 3 | metadata: 4 | name: web 5 | namespace: staging-web 6 | annotations: 7 | kubernetes.io/tls-acme: "true" 8 | kubernetes.io/ingress.class: "nginx" 9 | spec: 10 | tls: 11 | - hosts: 12 | - staging.loklak.org 13 | secretName: loklak-api-tls 14 | rules: 15 | - host: staging.loklak.org 16 | http: 17 | paths: 18 | - path: / 19 | backend: 20 | serviceName: server 21 | servicePort: 80 22 | -------------------------------------------------------------------------------- /docs/installation/installation_cloud9.md: -------------------------------------------------------------------------------- 1 | Steps to setup loklak on cloud9 2 | ------------------------------- 3 | 4 | * Create a cloud9 account 5 | * Create a workspace with the relevant name (Do not fill the url with the clone url of loklak server) 6 | * Wait for the workspace to build and take you to the relevant page 7 | * Use the terminal in the page and type the following command to setup the loklak server and run on cloud9 8 | ```bash 9 | wget -O - https://raw.githubusercontent.com/loklak/loklak_server/master/cloud9-setup.sh | bash 10 | ``` 11 | -------------------------------------------------------------------------------- /bin/search.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import urllib 4 | import json 5 | import sys 6 | 7 | searchTerm = sys.argv[1] 8 | query = "http://127.0.0.1:9000/api/search.json?q={}".format(searchTerm) 9 | 10 | try: 11 | fetchData = urllib.urlopen(query).read() 12 | except Exception as e: 13 | print "! Sorry, something went wrong:" 14 | print "! Error: %s" % e 15 | sys.exit(1) 16 | 17 | statuses = json.loads(fetchData).get("statuses") 18 | texts = [tweet.get("text") for tweet in statuses] 19 | 20 | for text in texts: 21 | print text 22 | -------------------------------------------------------------------------------- /docs/misc/stream.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | loklak Streaming Example 6 | 13 | 14 | 15 | See console for magic! 16 | 17 | -------------------------------------------------------------------------------- /bin/search_with_usernames.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | import json 3 | import urllib 4 | import sys 5 | 6 | if __name__ == "__main__": 7 | args = sys.argv[1:] 8 | if args: 9 | query = args[0] 10 | else: 11 | print("Usage: ./search.py QUERY") 12 | sys.exit(1) 13 | res = urllib.urlopen("http://127.0.0.1:9000/api/search.json?q={}".format(query)).read() 14 | data = json.loads(res) 15 | for tweet in data["statuses"]: 16 | print "@{}: {}".format(tweet["screen_name"].encode("utf-8"), tweet["text"].encode("utf-8")) 17 | -------------------------------------------------------------------------------- /kubernetes/bin/update-image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ "$1" = "api" ]] 4 | then 5 | BRANCH=Api 6 | IMAGE=loklak/loklak_server:latest-kubernetes-api 7 | elif [[ "$1" = "staging" ]] 8 | then 9 | BRANCH=Staging 10 | IMAGE=loklak/loklak_server:latest-kubernetes-staging 11 | else 12 | IMAGE=$1 13 | fi 14 | 15 | echo "Setting image $IMAGE" 16 | 17 | if [[ "$1" = "staging" ]] 18 | then 19 | kubectl set image deployment/server --namespace=api-staging server=$IMAGE 20 | else 21 | kubectl set image deployment/server --namespace=api-web server=$IMAGE 22 | fi 23 | 24 | echo "Succesfully updated $BRANCH image" 25 | -------------------------------------------------------------------------------- /bin/checkalive.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | cd "`dirname $0`" 3 | 4 | # for a production environment with high-availability requirement, 5 | # add the following line in /etc/crontab (replace with correct path!) 6 | # 0 * * * * root cd /usr/share/loklak/bin && ./checkalive.sh 7 | 8 | FLAG=0 9 | if [[ -n `./apicall.sh index.html | grep "loklak"` ]]; then 10 | FLAG=1 11 | fi 12 | 13 | if [[ $FLAG -eq '0' && -f ../data/loklak.log && `tail -1 ../data/loklak.log` == *"Waiting for elasticsearch yellow status"* ]]; then 14 | FLAG=1 15 | fi 16 | 17 | if [ $FLAG -eq '0' ]; then 18 | ./stop.sh & sleep 60; kill $! 19 | ./kill.sh 20 | ./start.sh 21 | fi 22 | exit 23 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | loklak_server 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.buildship.core.gradleprojectbuilder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.jdt.core.javanature 21 | org.eclipse.buildship.core.gradleprojectnature 22 | 23 | 24 | -------------------------------------------------------------------------------- /test/org/loklak/tools/storage/JsonMinifierTest.java: -------------------------------------------------------------------------------- 1 | package org.loklak.tools.storage; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import org.json.JSONObject; 6 | import org.json.JSONObjectTest; 7 | import org.junit.Test; 8 | 9 | public class JsonMinifierTest { 10 | 11 | @Test 12 | public void string2byte() throws Exception { 13 | JSONObject json = JSONObjectTest.testJson(true); 14 | JsonMinifier minifier = new JsonMinifier(); 15 | JsonMinifier.JsonCapsuleFactory capsule = minifier.minify(json); 16 | JSONObject challenge = capsule.getJSON(); 17 | assertEquals(json.toString(), challenge.toString()); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = 3 | # Module imported but unused 4 | F401, 5 | # Line too long 6 | E501, 7 | # Indentation contains tabs 8 | W191, 9 | ZZZZ # allow comma on last entry 10 | select = 11 | # Missing whitespace around modulo 12 | E228, 13 | # Blank line at end of file 14 | W391, 15 | # Block comment should start with '# ' 16 | E265, 17 | # Whitespace before '(' 18 | E211, 19 | # Trailing whitespace 20 | W291, 21 | # Multiple spaces after operator 22 | E222, 23 | # Multiple imports on one line 24 | E401, 25 | # Indentation contains mixed spaces and tabs 26 | E101, 27 | ZZZZ # allow comma on last entry 28 | filename = *.py 29 | -------------------------------------------------------------------------------- /res/log4j2.properties: -------------------------------------------------------------------------------- 1 | ## log to console 2 | 3 | shutdownHook = disable 4 | 5 | appenders = console 6 | 7 | appender.console.type = Console 8 | appender.console.name = STDOUT 9 | appender.console.layout.type = PatternLayout 10 | appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS}:%-5level:%C{1}:%t: %msg%n 11 | 12 | loggers = onlyerror 13 | logger.onlyerror.name=org.apache.http.client.protocol.ResponseProcessCookies 14 | logger.onlyerror.level = error 15 | logger.onlyerror.appenderRefs = stdout 16 | logger.onlyerror.appenderRef.stdout.ref = STDOUT 17 | 18 | rootLogger.level = info 19 | rootLogger.appenderRefs = stdout 20 | rootLogger.appenderRef.stdout.ref = STDOUT 21 | -------------------------------------------------------------------------------- /conf/logs/log4j2.properties: -------------------------------------------------------------------------------- 1 | ## log to console 2 | 3 | shutdownHook = disable 4 | 5 | appenders = console 6 | 7 | appender.console.type = Console 8 | appender.console.name = STDOUT 9 | appender.console.layout.type = PatternLayout 10 | appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS}:%-5level:%C{1}:%t: %msg%n 11 | 12 | loggers = onlyerror 13 | logger.onlyerror.name=org.apache.http.client.protocol.ResponseProcessCookies 14 | logger.onlyerror.level = error 15 | logger.onlyerror.appenderRefs = stdout 16 | logger.onlyerror.appenderRef.stdout.ref = STDOUT 17 | 18 | rootLogger.level = info 19 | rootLogger.appenderRefs = stdout 20 | rootLogger.appenderRef.stdout.ref = STDOUT 21 | -------------------------------------------------------------------------------- /src/org/loklak/data/Insertable.java: -------------------------------------------------------------------------------- 1 | package org.loklak.data; 2 | 3 | /** 4 | * Created by hadoop on 16-8-3. 5 | */ 6 | 7 | 8 | 9 | import java.util.*; 10 | 11 | /** 12 | * This is the interface that allow you to store data in elasticseach index 13 | * This is a lower level implemention of DAOWrapper. DAOWrapper may be a better solution if you don't care how we store data in elasticsearch index 14 | * 15 | * However, this one is apparently more flexible 16 | */ 17 | public interface Insertable { 18 | String getText(); 19 | String getUsername(); 20 | String getUserID(); 21 | String getDomain(); 22 | Collection> getExtraField(); 23 | } 24 | -------------------------------------------------------------------------------- /test/org/loklak/http/ClientConnectionTest.java: -------------------------------------------------------------------------------- 1 | package org.loklak.http; 2 | 3 | import org.junit.Test; 4 | import static org.junit.Assert.assertEquals; 5 | 6 | import java.io.IOException; 7 | import java.util.HashMap; 8 | 9 | public class ClientConnectionTest { 10 | 11 | @Test 12 | public void metaUrlExtractorTest() throws IOException { 13 | HashMap linkMap = new HashMap<>(); 14 | linkMap.put("https://yacy.net/en/index.html", "https://yacy.net/index.html"); 15 | 16 | for (HashMap.Entry pair : linkMap.entrySet()) { 17 | assertEquals(pair.getValue(), ClientConnection.getRedirect(pair.getKey())); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /docs/misc/StreamChannels.md: -------------------------------------------------------------------------------- 1 | # Stream Channels 2 | 3 | ## Generic 4 | 5 | The following channel contains a stream for all the messages collected by loklak: 6 | 7 | ``` 8 | - all 9 | ``` 10 | 11 | ## Twitter 12 | 13 | The following are Twitter specific channels: 14 | 15 | ``` 16 | - twitter 17 | - twitter/mention/ # Mentions of a user 18 | - twitter/user/ # By a user 19 | - twitter/hashtag/ # Including hashtag 20 | - twitter/country/ # For a specefic country 21 | - twitter/text/ # Containing word token 22 | ``` 23 | 24 | **Note**: Country codes are in [`ISO 3166-1 alpha-2`](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) format. 25 | -------------------------------------------------------------------------------- /src/org/loklak/server/HttpsMode.java: -------------------------------------------------------------------------------- 1 | package org.loklak.server; 2 | 3 | public enum HttpsMode { 4 | OFF(0), 5 | ON(1), 6 | REDIRECT(2), 7 | ONLY(3); 8 | 9 | private Integer mode; 10 | 11 | HttpsMode(int mode) { 12 | this.mode = mode; 13 | } 14 | 15 | public boolean equals(HttpsMode other) {return this.mode == other.mode;} 16 | public boolean isSmallerThan(HttpsMode other) {return this.mode < other.mode;} 17 | public boolean isSmallerOrEqualTo(HttpsMode other) {return this.mode <= other.mode;} 18 | public boolean isGreaterThan(HttpsMode other) {return this.mode > other.mode;} 19 | public boolean isGreaterOrEqualTo(HttpsMode other) {return this.mode >= other.mode;} 20 | } 21 | -------------------------------------------------------------------------------- /test/org/loklak/TestRunner.java: -------------------------------------------------------------------------------- 1 | package org.loklak; 2 | 3 | import org.junit.runner.RunWith; 4 | import org.junit.runners.Suite; 5 | import org.loklak.harvester.TwitterScraperTest; 6 | import org.loklak.harvester.YoutubeScraperTest; 7 | import org.loklak.api.search.GithubProfileScraperTest; 8 | import org.loklak.api.search.QuoraProfileScraperTest; 9 | import org.loklak.api.search.InstagramProfileScraperTest; 10 | /* 11 | TestRunner for harvesters 12 | */ 13 | @RunWith(Suite.class) 14 | @Suite.SuiteClasses({ 15 | TwitterScraperTest.class, 16 | YoutubeScraperTest.class, 17 | GithubProfileScraperTest.class, 18 | QuoraProfileScraperTest.class, 19 | InstagramProfileScraperTest.class 20 | }) 21 | public class TestRunner { } 22 | -------------------------------------------------------------------------------- /kubernetes/yamls/api/api-server/api-deployment.yml: -------------------------------------------------------------------------------- 1 | # Main deployment for API server 2 | apiVersion: apps/v1beta1 3 | kind: Deployment 4 | metadata: 5 | name: server 6 | namespace: api-web 7 | spec: 8 | replicas: 1 9 | template: 10 | metadata: 11 | labels: 12 | app: server 13 | spec: 14 | containers: 15 | - name: server 16 | imagePullPolicy: Always 17 | image: loklak/loklak_server:latest-kubernetes-api 18 | ports: 19 | - containerPort: 80 20 | protocol: TCP 21 | livenessProbe: 22 | httpGet: 23 | path: /api/status.json 24 | port: 80 25 | initialDelaySeconds: 60 26 | timeoutSeconds: 10 27 | periodSeconds: 360 28 | -------------------------------------------------------------------------------- /kubernetes/yamls/staging/api-server/api-deployment.yml: -------------------------------------------------------------------------------- 1 | # Main deployment for API server 2 | apiVersion: apps/v1beta1 3 | kind: Deployment 4 | metadata: 5 | name: server 6 | namespace: staging-web 7 | spec: 8 | replicas: 1 9 | template: 10 | metadata: 11 | labels: 12 | app: server 13 | spec: 14 | containers: 15 | - name: server 16 | imagePullPolicy: Always 17 | image: loklak/loklak_server:latest-kubernetes-staging 18 | livenessProbe: 19 | httpGet: 20 | path: /api/status.json 21 | port: 80 22 | initialDelaySeconds: 90 23 | periodSeconds: 300 24 | timeoutSeconds: 3 25 | ports: 26 | - containerPort: 80 27 | protocol: TCP 28 | -------------------------------------------------------------------------------- /html/js/ie10-viewport-bug-workaround.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * IE10 viewport hack for Surface/desktop Windows 8 bug 3 | * Copyright 2014 Twitter, Inc. 4 | * Licensed under the Creative Commons Attribution 3.0 Unported License. For 5 | * details, see http://creativecommons.org/licenses/by/3.0/. 6 | */ 7 | 8 | // See the Getting Started docs for more information: 9 | // http://getbootstrap.com/getting-started/#support-ie10-width 10 | 11 | (function () { 12 | 'use strict'; 13 | if (navigator.userAgent.match(/IEMobile\/10\.0/)) { 14 | var msViewportStyle = document.createElement('style') 15 | msViewportStyle.appendChild( 16 | document.createTextNode( 17 | '@-ms-viewport{width:auto!important}' 18 | ) 19 | ) 20 | document.querySelector('head').appendChild(msViewportStyle) 21 | } 22 | })(); 23 | -------------------------------------------------------------------------------- /bin/search1.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | # Install required modules: 3 | # cpan install JSON 4 | # cpan install LWP 5 | 6 | use strict; 7 | use warnings; 8 | use JSON qw( decode_json ); 9 | use LWP::Simple; 10 | 11 | my $amount_of_args = $#ARGV + 1; 12 | if ($amount_of_args != 1) { 13 | print "\nUsage: search.pl search_term\n"; 14 | exit; 15 | } 16 | 17 | my $url = "http://127.0.0.1:9000/api/search.json?q=$ARGV[0]"; 18 | my $json = get( $url ); 19 | die "Sorry! Could not open $url!" unless defined $json; 20 | 21 | my $decoded_json = decode_json( $json ); 22 | 23 | foreach my $tweets(@{$decoded_json->{'statuses'}}){ 24 | my %tweets_hash = (); 25 | $tweets_hash{text} = "$tweets->{'text'}"; 26 | while (my($k, $v) = each (%tweets_hash)){ 27 | print "$v\n"; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /bin/test_piotr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import os 3 | import requests 4 | 5 | 6 | def print_response_from_file(file_name, path): 7 | current_file = open(path + file_name, 'r', encoding="utf8") 8 | for line in current_file: 9 | response = requests.get("http://127.0.0.1:9000/api/search.json?q=" + line) 10 | x = response.json() 11 | try: 12 | print(x['statuses']) 13 | except Exception as e: 14 | print(e) 15 | 16 | 17 | def print_all_files_lines_in_defined_directory(path): 18 | for fn in os.listdir(path): 19 | if fn == 'README.txt': 20 | pass 21 | elif fn.endswith(".txt"): 22 | print_response_from_file(fn, path) 23 | else: 24 | pass 25 | 26 | 27 | if __name__ == "__main__": 28 | print_all_files_lines_in_defined_directory('../test/queries/') 29 | -------------------------------------------------------------------------------- /kubernetes/yamls/nginx/default-deployment.yml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: default-http-backend 5 | namespace: nginx-ingress 6 | spec: 7 | replicas: 1 8 | template: 9 | metadata: 10 | labels: 11 | app: default-http-backend 12 | spec: 13 | containers: 14 | - name: default-http-backend 15 | image: gcr.io/google_containers/defaultbackend:1.0 16 | livenessProbe: 17 | httpGet: 18 | path: /healthz 19 | port: 8080 20 | scheme: HTTP 21 | initialDelaySeconds: 30 22 | timeoutSeconds: 5 23 | ports: 24 | - containerPort: 8080 25 | resources: 26 | limits: 27 | cpu: 10m 28 | memory: 20Mi 29 | requests: 30 | cpu: 10m 31 | memory: 20Mi 32 | -------------------------------------------------------------------------------- /bin/search.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | # Requirements Dep: CPAN:JSON 3 | # Installtion: 4 | # cpan install JSON 5 | # cpan install LWP 6 | 7 | use LWP::Simple; # From CPAN 8 | use JSON qw( decode_json ); # From CPAN 9 | use strict; # Good practice 10 | use warnings; # Good practice 11 | 12 | my $num_args = $#ARGV + 1; 13 | if ($num_args != 1) { 14 | print "\nUsage: search.pl string\n"; 15 | exit; 16 | } 17 | 18 | my $url = "http://127.0.0.1:9000/api/search.json?q=$ARGV[0]"; 19 | my $json = get( $url ); 20 | die "Could not get $url!" unless defined $json; 21 | 22 | my $decoded_json = decode_json( $json ); 23 | 24 | foreach my $tweets(@{$decoded_json->{'statuses'}}){ 25 | my %tweets_hash = (); 26 | $tweets_hash{text} = "$tweets->{'text'}"; 27 | while (my($k, $v) = each (%tweets_hash)){ 28 | print "$v\n"; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /bin/search_fossasia.py: -------------------------------------------------------------------------------- 1 | # Returns all tweet text lines related to fossasia 2 | import urllib 3 | import json 4 | import sys 5 | 6 | 7 | search_term = sys.argv[1] 8 | query = "http://loklak.org/api/search.json?q={}".format(search_term) 9 | 10 | # Get file and decode 11 | try: 12 | json_data = urllib.request.Request(query) 13 | with urllib.request.urlopen(json_data) as data: 14 | data1 = str(data.read().decode('utf-8')) 15 | non_bmp_map = dict.fromkeys(range(0x10000, sys.maxunicode + 1), 0xfffd) 16 | data1 = data1.translate(non_bmp_map) 17 | except Exception as e: 18 | print("! Sorry, something went wrong:") 19 | print("! Error: %s" % e) 20 | sys.exit(1) 21 | 22 | # Output tweets 23 | statuses = json.loads(data1).get("statuses") 24 | texts = [tweet.get("text") for tweet in statuses] 25 | 26 | for text in texts: 27 | print(text) 28 | -------------------------------------------------------------------------------- /.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /bin/bluemix_start.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Make sure we're on project root 4 | cd $(dirname $0)/.. 5 | 6 | #jdk8 7 | 8 | JAVA_DIST="openjdk-1.8.0_51" 9 | JDK8_URL="https://download.run.pivotal.io/openjdk/lucid/x86_64/${JAVA_DIST}.tar.gz" 10 | 11 | if [ -d .jdk8 ]; then 12 | echo "-----> .jdk8 folder found, moving along." 13 | 14 | else 15 | echo -n "-----> .jdk8 folder not found! " 16 | if [[ -d "$PWD/.jdk8" ]]; then 17 | echo -n "Copying jdk from cache to app... " 18 | cp -r "$PWD/.jdk8" "$PWD" 19 | echo "Done!" 20 | 21 | else 22 | echo -n "-----> Installing ${JAVA_DIST} build (to .jdk8)....." 23 | mkdir "$PWD/.jdk8" 24 | cd "$PWD/.jdk8" 25 | curl --max-time 180 --location "$JDK8_URL" | tar xz 26 | cd "$PWD" 27 | echo "Done!" 28 | fi 29 | fi 30 | 31 | cd $PWD 32 | 33 | export JAVA_HOME="$PWD/.jdk8" 34 | export PATH="$JAVA_HOME/bin:$PATH" 35 | 36 | exec ./bin/start.sh -Idn 37 | -------------------------------------------------------------------------------- /docs/installation/installation_heroku.md: -------------------------------------------------------------------------------- 1 | # How to run the Loklak server on Heroku using Toolbelt 2 | 3 | 1. Create a Heroku account https://www.heroku.com/ 4 | 2. Download the Heroku toolbelt https://toolbelt.heroku.com/ 5 | 3. Login with heroku: `heroku login` 6 | 4. Clone the Loklak server (if not already) : `git clone https://github.com/loklak/loklak_server.git` 7 | 5. Move into the cloned repository: `cd loklak_server` 8 | 6. Create a heroku app: `heroku create` 9 | 7. Set the buildpack: `heroku buildpacks:set https://github.com/heroku/heroku-buildpack-gradle.git` 10 | 8. Push your app to heroku: `git push heroku master` 11 | In case if it doesn't get pushed, use `git push heroku HEAD:master` 12 | 9. Confirm the loklak server is running: `heroku logs --tail` 13 | 10. Sometimes the server may take a while to start. The logs would show `State changed from starting to up` when the server is ready. 14 | 11. Open the URL of your server in your browser: `heroku open`. 15 | -------------------------------------------------------------------------------- /.bluemix/pipeline.yml: -------------------------------------------------------------------------------- 1 | --- 2 | stages: 3 | - name: Build Stage 4 | inputs: 5 | - type: git 6 | branch: master 7 | triggers: 8 | - type: commit 9 | jobs: 10 | - name: Build 11 | type: builder 12 | artifact_dir: '' 13 | build_type: shell 14 | script: |- 15 | #!/bin/bash 16 | echo "Run gradle build..." 17 | export JAVA_HOME=/opt/IBM/java8 18 | ./gradlew build 19 | - name: Deploy Stage 20 | inputs: 21 | - type: job 22 | stage: Build Stage 23 | job: Build 24 | triggers: 25 | - type: stage 26 | jobs: 27 | - name: Deploy to dev 28 | type: deployer 29 | target: 30 | url: https://api.ng.bluemix.net 31 | organization: ${CF_ORGANIZATION} 32 | space: ${CF_SPACE} 33 | application: ${CF_APP} 34 | script: |- 35 | #!/bin/bash 36 | cf push "${CF_APP}" -c "bin/bluemix_start.sh ;sleep 1d" -n "${CF_APP}" 37 | 38 | # View logs 39 | #cf logs "${CF_APP}" --recent 40 | -------------------------------------------------------------------------------- /src/org/loklak/server/BaseUserRole.java: -------------------------------------------------------------------------------- 1 | /** 2 | * BaseUserRole 3 | * Copyright 17.05.2016 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | 21 | package org.loklak.server; 22 | 23 | public enum BaseUserRole { 24 | 25 | ANONYMOUS, 26 | USER, 27 | PRIVILEGED, 28 | ADMIN 29 | } 30 | -------------------------------------------------------------------------------- /bin/profile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Requirements: 4 | 5 | requests (installation: pip install requests) 6 | """ 7 | import requests 8 | import os 9 | 10 | SEARCH_URL = 'http://127.0.0.1:9000/api/search.json' 11 | _ALL_FILES_ = [] 12 | 13 | for (d, f, filenames) in os.walk(os.path.join(os.getcwd(), 14 | '../test/queries/')): 15 | if filenames == "README.txt": 16 | continue 17 | else: 18 | _ALL_FILES_ = filenames 19 | 20 | for single_file in _ALL_FILES_: 21 | path_to_file = os.path.join(os.path.abspath('../test/queries'), 22 | single_file) 23 | with open(path_to_file, 'rb') as f: 24 | queries = f.readlines() 25 | 26 | for query in queries: 27 | resp = requests.get(url=SEARCH_URL, params={'source': 'cache', 28 | 'q': query.strip()}) 29 | data = resp.json() 30 | statuses = data['statuses'] 31 | for status in statuses: 32 | print status['text'] 33 | -------------------------------------------------------------------------------- /conf/elasticsearch/mappings/accounts.json: -------------------------------------------------------------------------------- 1 | { 2 | "_all": {"enabled": false}, 3 | "properties":{ 4 | "timestamp":{ 5 | "type":"date", "format":"dateOptionalTime", "include_in_all":"false", "index":"not_analyzed" 6 | }, 7 | "screen_name":{ 8 | "type":"string", "include_in_all":"false", "index":"not_analyzed" 9 | }, 10 | "source_type":{ 11 | "type":"string", "include_in_all":"false", "index":"not_analyzed" 12 | }, 13 | "authentication_first":{ 14 | "type":"date", "format":"dateOptionalTime", "include_in_all":"false", "index":"not_analyzed" 15 | }, 16 | "authentication_latest":{ 17 | "type":"date", "format":"dateOptionalTime", "include_in_all":"false", "index":"not_analyzed" 18 | }, 19 | "oauth_token":{ 20 | "type":"string", "include_in_all":"false", "index":"not_analyzed" 21 | }, 22 | "oauth_token_secret":{ 23 | "type":"string", "include_in_all":"false", "index":"not_analyzed" 24 | }, 25 | "apps":{ 26 | "type":"object", "include_in_all":"false" 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /ssi/common_head_front.include: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /bin/search(python2&3).py: -------------------------------------------------------------------------------- 1 | """ 2 | loklak Tweeet Searcher 3 | 4 | Works with Python 2 & 3 5 | """ 6 | 7 | import argparse 8 | import json 9 | try: 10 | from urllib.request import urlopen # Imports for Python 3 11 | except ImportError: 12 | from urllib import urlopen # Imports for Python 2 13 | 14 | # Gets command line arguments 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument("query", type=str, help="Query to search for") 17 | args = parser.parse_args() 18 | query = args.query 19 | 20 | # Crafts url 21 | SEARCH_URL = "http://127.0.0.1:9000/api/search.json?q=" 22 | full_url = SEARCH_URL + query 23 | 24 | # Gets page 25 | page = urlopen(full_url).read().decode('utf-8') 26 | 27 | # Parses json 28 | data = json.loads(page) 29 | statuses = data['statuses'] 30 | 31 | # Prints tweets 32 | for status in statuses: 33 | # Note: encode is used to account for special characters that cannot be displayed in some command lines 34 | print(status['text'].encode('utf-8')) 35 | # Note: if using Python 3, they are printed as a byte 36 | print("") 37 | -------------------------------------------------------------------------------- /kubernetes/images/staging/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-alpine 2 | 3 | # Env Vars 4 | ENV LANG=en_US.UTF-8 5 | ENV JAVA_TOOL_OPTIONS=-Dfile.encoding=UTF8 6 | 7 | WORKDIR /loklak_server 8 | 9 | RUN apk update && apk add --no-cache git bash && \ 10 | git clone https://github.com/loklak/loklak_server.git /loklak_server && \ 11 | git checkout development && \ 12 | ./gradlew build -x test -x checkstyleTest -x checkstyleMain -x jacocoTestReport && \ 13 | sed -i.bak 's/^\(port.http=\).*/\180/' conf/config.properties && \ 14 | sed -i.bak 's/^\(port.https=\).*/\1443/' conf/config.properties && \ 15 | sed -i.bak 's/^\(upgradeInterval=\).*/\186400000000/' conf/config.properties && \ 16 | sed -i.bak 's/^\(elasticsearch_transport.enabled\).*/\1=true/' conf/config.properties && \ 17 | sed -i.bak 's/^\(elasticsearch_transport.addresses\).*/\1=elasticsearch.elasticsearch:9300/' conf/config.properties && \ 18 | sed -i.bak 's/^\(Xmx\).*/\1=3G/' conf/config.properties && \ 19 | echo "while true; do sleep 10;done" >> bin/start.sh 20 | 21 | 22 | # Start 23 | CMD ["bin/start.sh", "-Idn"] 24 | -------------------------------------------------------------------------------- /kubernetes/yamls/lego/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: kube-lego 5 | namespace: kube-lego 6 | spec: 7 | replicas: 1 8 | template: 9 | metadata: 10 | labels: 11 | app: kube-lego 12 | spec: 13 | containers: 14 | - name: kube-lego 15 | image: jetstack/kube-lego:0.1.4 16 | imagePullPolicy: Always 17 | ports: 18 | - containerPort: 8080 19 | env: 20 | - name: LEGO_EMAIL 21 | valueFrom: 22 | configMapKeyRef: 23 | name: kube-lego 24 | key: lego.email 25 | - name: LEGO_LOG_LEVEL 26 | value: debug 27 | - name: LEGO_URL 28 | valueFrom: 29 | configMapKeyRef: 30 | name: kube-lego 31 | key: lego.url 32 | - name: LEGO_NAMESPACE 33 | valueFrom: 34 | fieldRef: 35 | fieldPath: metadata.namespace 36 | - name: LEGO_POD_IP 37 | valueFrom: 38 | fieldRef: 39 | fieldPath: status.podIP 40 | -------------------------------------------------------------------------------- /bin/generate_test_queries.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | import time 5 | try: 6 | import wikipedia 7 | except ImportError: 8 | print('Queries generation requires wikipedia package. \n' 9 | 'You can install it by running "pip install wikipedia"') 10 | exit() 11 | 12 | # If user in bin directory set correct path to queries 13 | if os.getcwd().endswith('bin'): 14 | save_folder = os.getcwd().rstrip('bin') + 'test/queries' 15 | else: 16 | save_folder = os.getcwd() + '/test/queries' 17 | 18 | if len(sys.argv) != 3: 19 | print('Please run script by format: python bin/generate_test_queries.py en 100') 20 | exit() 21 | 22 | language = sys.argv[1] 23 | queries_num = int(sys.argv[2]) 24 | wikipedia.set_lang(language) 25 | queries = wikipedia.random(queries_num) 26 | 27 | with open('{}/{}_{}.txt'.format(save_folder, language, int(time.time())), 'w') as file: 28 | for query in queries: 29 | if sys.version[0] == '3': 30 | file.write('%s\n' % query) 31 | else: 32 | file.write('%s\n' % query.encode('utf-8')) 33 | 34 | print('Done.') 35 | -------------------------------------------------------------------------------- /test/org/loklak/tools/storage/JsonFileTest.java: -------------------------------------------------------------------------------- 1 | package org.loklak.tools.storage; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import org.junit.After; 6 | import org.junit.Before; 7 | import org.loklak.tools.BufferedRandomAccessFile; 8 | 9 | import junit.framework.TestCase; 10 | 11 | public class JsonFileTest extends TestCase { 12 | 13 | private File testFile; 14 | private JsonFile testJsonFile; 15 | 16 | @Before 17 | public void setUp() throws Exception { 18 | this.testFile = BufferedRandomAccessFile.Test.getTestFile(); 19 | this.testJsonFile = new JsonFile(this.testFile); 20 | } 21 | 22 | @After 23 | public void tearDown() throws Exception { 24 | this.testFile.delete(); 25 | } 26 | 27 | public void test() throws IOException { 28 | 29 | long start = System.currentTimeMillis(); 30 | for (int i = 0; i < 100; i++) { 31 | this.testJsonFile.put("key", i); 32 | } 33 | long stop = System.currentTimeMillis(); 34 | System.out.println("runtime: " + (stop - start) + " milliseconds"); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /conf/logs/log-to-file.properties: -------------------------------------------------------------------------------- 1 | ## log to file 2 | 3 | shutdownHook = disable 4 | 5 | property.logPath = data 6 | appenders = file 7 | 8 | appender.file.type = RollingFile 9 | appender.file.name = LOGFILE 10 | appender.file.fileName = ${logPath}/loklak.log 11 | appender.file.filePattern = ${logPath}/loklak.log-%i.gz 12 | appender.file.layout.type = PatternLayout 13 | appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS}:%-5level:%C{10}:%t: %msg%n 14 | appender.file.bufferedIO = true 15 | appender.file.bufferSize = 8192 16 | appender.file.immediateFlush = false 17 | appender.file.policies.type=Policies 18 | appender.file.policies.size.type=SizeBasedTriggeringPolicy 19 | appender.file.policies.size.size=10MB 20 | appender.file.strategy.type=DefaultRolloverStrategy 21 | appender.file.strategy.max=3 22 | 23 | loggers = onlyerror 24 | logger.onlyerror.name=org.apache.http.client.protocol.ResponseProcessCookies 25 | logger.onlyerror.level = error 26 | logger.onlyerror.appenderRefs = file 27 | logger.onlyerror.appenderRef.stdout.ref = LOGFILE 28 | 29 | rootLogger.level = info 30 | rootLogger.appenderRefs = file 31 | rootLogger.appenderRef.stdout.ref = LOGFILE 32 | -------------------------------------------------------------------------------- /src/org/loklak/objects/ObjectEntry.java: -------------------------------------------------------------------------------- 1 | /** 2 | * ObjectEntry 3 | * Copyright 26.04.2015 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; wo even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.objects; 21 | 22 | import org.json.JSONObject; 23 | 24 | public interface ObjectEntry { 25 | 26 | public String toString(); 27 | 28 | // TODO: convert to elasticsearch internal format directly 29 | 30 | public JSONObject toJSON(); 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/org/loklak/tools/storage/JsonFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * JsonFactory 3 | * Copyright 02.10.2015 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.tools.storage; 21 | 22 | import java.io.IOException; 23 | 24 | import org.json.JSONObject; 25 | 26 | public interface JsonFactory { 27 | 28 | public String getString() throws IOException; 29 | public JSONObject getJSON() throws IOException; 30 | 31 | } 32 | -------------------------------------------------------------------------------- /conf/elasticsearch/mappings/users.json: -------------------------------------------------------------------------------- 1 | { 2 | "_all": {"enabled": false}, 3 | "properties":{ 4 | "timestamp":{ 5 | "type":"date", "format":"dateOptionalTime", "include_in_all":"false", "index":"not_analyzed" 6 | }, 7 | "screen_name":{ 8 | "type":"string", "include_in_all":"false", "index":"not_analyzed" 9 | }, 10 | "name":{ 11 | "type":"string", "include_in_all":"false", "index":"analyzed" 12 | }, 13 | "user_id":{ 14 | "type":"string", "include_in_all":"false", "index":"not_analyzed" 15 | }, 16 | "appearance_first":{ 17 | "type":"date", "format":"dateOptionalTime", "include_in_all":"false", "index":"not_analyzed" 18 | }, 19 | "appearance_latest":{ 20 | "type":"date", "format":"dateOptionalTime", "include_in_all":"false", "index":"not_analyzed" 21 | }, 22 | "profile_image_url_http":{ 23 | "type":"string", "include_in_all":"false", "index":"not_analyzed" 24 | }, 25 | "profile_image_url_https":{ 26 | "type":"string", "include_in_all":"false", "index":"not_analyzed" 27 | }, 28 | "profile_image":{ 29 | "type":"binary", "include_in_all":"false", "index":"no" 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/org/loklak/stream/StreamServlet.java: -------------------------------------------------------------------------------- 1 | package org.loklak.stream; 2 | 3 | import org.eclipse.jetty.servlets.EventSource; 4 | import org.eclipse.jetty.servlets.EventSourceServlet; 5 | 6 | import javax.servlet.ServletException; 7 | import javax.servlet.http.HttpServletRequest; 8 | import javax.servlet.http.HttpServletResponse; 9 | import java.io.IOException; 10 | 11 | public class StreamServlet extends EventSourceServlet { 12 | /** 13 | * 14 | */ 15 | private static final long serialVersionUID = 1224323810947361163L; 16 | 17 | @Override 18 | protected EventSource newEventSource(HttpServletRequest request) { 19 | String channel = request.getParameter("channel"); 20 | if (channel == null) { 21 | return null; 22 | } 23 | if (channel.isEmpty()) { 24 | return null; 25 | } 26 | return new MqttEventSource(channel); 27 | } 28 | 29 | @Override 30 | protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 31 | response.setHeader("Access-Control-Allow-Origin", "*"); 32 | super.doGet(request, response); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /kubernetes/yamls/nginx/deployment.yml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: nginx 5 | namespace: nginx-ingress 6 | spec: 7 | replicas: 1 8 | template: 9 | metadata: 10 | labels: 11 | app: nginx 12 | spec: 13 | containers: 14 | - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3 15 | name: nginx 16 | imagePullPolicy: Always 17 | env: 18 | - name: POD_NAME 19 | valueFrom: 20 | fieldRef: 21 | fieldPath: metadata.name 22 | - name: POD_NAMESPACE 23 | valueFrom: 24 | fieldRef: 25 | fieldPath: metadata.namespace 26 | livenessProbe: 27 | httpGet: 28 | path: /healthz 29 | port: 10254 30 | scheme: HTTP 31 | initialDelaySeconds: 30 32 | periodSeconds: 10 33 | timeoutSeconds: 5 34 | ports: 35 | - containerPort: 80 36 | - containerPort: 443 37 | args: 38 | - /nginx-ingress-controller 39 | - --default-backend-service=nginx-ingress/default-http-backend 40 | - --nginx-configmap=nginx-ingress/nginx 41 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: trusty 2 | language: java 3 | services: docker 4 | jdk: 5 | - oraclejdk8 6 | env: 7 | global: 8 | - secure: DbveaxDMtEP+/Er6ktKCP+P42uDU8xXWRBlVGaqVNU3muaRmmZtj8ngAARxfzY0f9amlJlCavqkEIAumQl9BYKPWIra28ylsLNbzAoCIi8alf9WLgddKwVWsTcZo9+UYocuY6UivJVkofycfFJ1blw/83dWMG0/TiW6s/SrwoDw= 9 | - secure: w5VfIJI0RC27fUcVC/tB63Z1IOIkFCz3khmrrBSMwDjmKFWEplnkm1Yhs379yqohsSHE18SqY4MgLzOu9KJShoP1cur9ozhta+osSjXOZpiES/fQyRLRrCDSQR0ZLX0MBubJIGysFJrlPmMuue9AhcNzOLR6wwYrmZr9mIWMQcc= 10 | addons: 11 | ssh_known_hosts: 104.155.231.103 12 | before_cache: 13 | - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock 14 | - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ 15 | cache: 16 | directories: 17 | - $HOME/.gradle/caches/ 18 | - $HOME/.gradle/wrapper/ 19 | - $HOME/.cache/pip 20 | install: 21 | - pip install --user flake8 22 | script: 23 | - ./gradlew build -x checkstyleTest -x checkstyleMain 24 | - bin/start.sh 25 | - flake8 bin 26 | - docker build -t loklak_server . 27 | - docker images 28 | - docker run loklak_server bash /loklak_server/bin/start.sh -I 29 | after_success: 30 | - bash <(curl -s https://codecov.io/bash) 31 | - gradle clean 32 | - bash .utility/push-docker.sh 33 | - bash kubernetes/bin/update-deployment-travis.sh 34 | -------------------------------------------------------------------------------- /src/org/loklak/objects/Peers.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Peers 3 | * Copyright 07.01.2016 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.objects; 21 | 22 | import java.util.HashMap; 23 | 24 | public class Peers extends HashMap { 25 | 26 | private static final long serialVersionUID = 2767548867468641625L; 27 | 28 | public Peers() { 29 | super(); 30 | } 31 | 32 | public void update(Peer peer) { 33 | 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /kubernetes/images/api/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-alpine 2 | 3 | # Env Vars 4 | ENV LANG=en_US.UTF-8 5 | ENV JAVA_TOOL_OPTIONS=-Dfile.encoding=UTF8 6 | 7 | WORKDIR /loklak_server 8 | 9 | RUN apk update && apk add --no-cache git bash && \ 10 | git clone https://github.com/loklak/loklak_server.git /loklak_server && \ 11 | git checkout origin/master && \ 12 | ./gradlew build -x test -x checkstyleTest -x checkstyleMain -x jacocoTestReport && \ 13 | sed -i.bak 's/^\(port.http=\).*/\180/' conf/config.properties && \ 14 | sed -i.bak 's/^\(port.https=\).*/\1443/' conf/config.properties && \ 15 | sed -i.bak 's/^\(upgradeInterval=\).*/\186400000000/' conf/config.properties && \ 16 | sed -i.bak 's/^\(elasticsearch_transport.enabled\).*/\1=true/' conf/config.properties && \ 17 | sed -i.bak 's/^\(elasticsearch_transport.addresses\).*/\1=elasticsearch.elasticsearch:9300/' conf/config.properties && \ 18 | sed -i.bak 's/^\(dump.write_enabled\).*/\1=false/' conf/config.properties && \ 19 | sed -i.bak 's/^\(backend=\).*/\1http:\/\/root.loklak.org/' conf/config.properties && \ 20 | sed -i.bak 's/^\(Xmx\).*/\1=3G/' conf/config.properties && \ 21 | echo "while true; do sleep 10;done" >> bin/start.sh 22 | 23 | # Start 24 | CMD ["bin/start.sh", "-Idn"] 25 | -------------------------------------------------------------------------------- /.utility/docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Loklak compose setup for test.loklak.org 2 | 3 | # IMPORTANT NOTES: 4 | # - Version is set to 2.1, because setting it to 3 results in the depends_on have to 5 | # be "arrays". Even though the documentation says to do it like below. This is 6 | # probably because the docs isn't updated. If it's updated or fixed, please 7 | # change it to version 3. 8 | 9 | version: '2.1' 10 | services: 11 | elasticsearch: 12 | image: elasticsearch:2 13 | restart: always 14 | volumes: 15 | - ./elastic/config:/usr/share/elasticsearch/config 16 | - ./elastic/data:/usr/share/elasticsearch/data 17 | healthcheck: 18 | test: curl -f localhost:9200 19 | worker: 20 | build: 21 | context: ./loklak/repo 22 | dockerfile: docker/Dockerfile 23 | restart: always 24 | volumes: 25 | - ./loklak/customized_config.properties:/loklak_server/data/settings/customized_config.properties:ro 26 | links: 27 | - elasticsearch 28 | depends_on: 29 | elasticsearch: 30 | condition: service_healthy 31 | healthcheck: 32 | test: wget localhost -O - 33 | web: 34 | image: dockercloud/haproxy 35 | volumes: 36 | - /var/run/docker.sock:/var/run/docker.sock 37 | ports: 38 | - 80:80 39 | links: 40 | - worker 41 | -------------------------------------------------------------------------------- /test/org/loklak/tools/storage/CommonPatternTest.java: -------------------------------------------------------------------------------- 1 | package org.loklak.tools.storage; 2 | 3 | import org.loklak.tools.CommonPattern; 4 | 5 | import org.junit.Test; 6 | import static org.junit.Assert.assertEquals; 7 | /** 8 | * This test file tests org.loklak.tools.CommonPattern.java 9 | */ 10 | public class CommonPatternTest { 11 | 12 | @Test 13 | public void commonPattern() { 14 | 15 | CommonPattern commonPattern = new CommonPattern(); 16 | 17 | assertEquals(" ", commonPattern.SPACE.toString()); 18 | assertEquals(",", commonPattern.COMMA.toString()); 19 | assertEquals(";", commonPattern.SEMICOLON.toString()); 20 | assertEquals(":", commonPattern.DOUBLEPOINT.toString()); 21 | assertEquals("/", commonPattern.SLASH.toString()); 22 | assertEquals("\\|", commonPattern.PIPE.toString()); 23 | assertEquals("\\\\", commonPattern.BACKSLASH.toString()); 24 | assertEquals("\\?", commonPattern.QUESTION.toString()); 25 | assertEquals("&", commonPattern.AMP.toString()); 26 | assertEquals("\\.", commonPattern.DOT.toString()); 27 | assertEquals("\n", commonPattern.NEWLINE.toString()); 28 | assertEquals("_", commonPattern.UNDERSCORE.toString()); 29 | assertEquals("\t", commonPattern.TAB.toString()); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/org/loklak/tools/storage/JsonReader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * JsonReader 3 | * Copyright 04.10.2015 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.tools.storage; 21 | 22 | import org.loklak.tools.storage.JsonRandomAccessFile.JsonHandle; 23 | 24 | public interface JsonReader extends Runnable { 25 | 26 | public final static JsonFactory POISON_JSON_MAP = new JsonHandle(null, -1, -1); 27 | 28 | public int getConcurrency(); 29 | 30 | public JsonFactory take() throws InterruptedException; 31 | 32 | public String getName(); 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/org/loklak/objects/Peer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Peer 3 | * Copyright 07.01.2016 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.objects; 21 | 22 | import org.json.JSONObject; 23 | 24 | public class Peer extends JSONObject { 25 | 26 | public static enum Status { 27 | CANDIDATE, // a new peer which was not tested for an open port or a SENIOR which could not be connected 28 | JUNIOR, // a candidate which was tested for an open port unsuccessfully 29 | SENIOR; // a candidate which was tested for an open port successfully 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/org/loklak/server/APIException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * APIException 3 | * Copyright 19.05.2016 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.server; 21 | 22 | public class APIException extends Exception { 23 | 24 | private static final long serialVersionUID = -6974553774866005875L; 25 | private int statusCode = -1; 26 | 27 | public APIException(int statusCode, String message) { 28 | super(message); 29 | this.statusCode = statusCode; 30 | } 31 | 32 | public int getStatusCode() { 33 | return this.statusCode; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/org/loklak/server/APIHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * APIHandler 3 | * Copyright 17.05.2016 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.server; 21 | 22 | import org.json.JSONObject; 23 | 24 | public interface APIHandler { 25 | 26 | public String[] getServerProtocolHostStub(); 27 | 28 | public BaseUserRole getMinimalBaseUserRole(); 29 | 30 | public JSONObject getDefaultPermissions(BaseUserRole baseUserRole); 31 | 32 | public String getAPIPath(); 33 | 34 | public JSONObject[] service(Query call, Authorization rights) throws APIException; 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/org/loklak/geo/LocationSource.java: -------------------------------------------------------------------------------- 1 | /** 2 | * LocationSource 3 | * Copyright 10.06.2015 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.geo; 21 | 22 | public enum LocationSource { 23 | 24 | USER, // the (loklak) user has set the location, this is a hint that this is a rich tweet. 25 | REPORT, // location came from another source in identical way. This may be a IoT import. 26 | PLACE, // location came from translation of the given place name, which is in the context of messages an invisible meta-information. 27 | ANNOTATION; // location was detected and annotated from visible text content 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/org/loklak/tools/storage/StorageMetadata.java: -------------------------------------------------------------------------------- 1 | /** 2 | * StorageMetadata 3 | * Copyright 21.08.2020 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | 21 | package org.loklak.tools.storage; 22 | 23 | public class StorageMetadata { 24 | 25 | public final String bucketName, objectName, contentType; 26 | public final long createdTime, length; 27 | 28 | public StorageMetadata(String bucketName, String objectName, String contentType, long createdTime, long length) { 29 | this.bucketName = bucketName; 30 | this.objectName = objectName; 31 | this.contentType = contentType; 32 | this.createdTime = createdTime; 33 | this.length = length; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /test/org/json/JSONObjectTest.java: -------------------------------------------------------------------------------- 1 | package org.json; 2 | 3 | import java.io.IOException; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | import junit.framework.TestCase; 8 | 9 | import org.junit.After; 10 | import org.junit.Before; 11 | 12 | public class JSONObjectTest extends TestCase { 13 | 14 | public static JSONObject testJson(boolean ordered) { 15 | JSONObject json = new JSONObject(ordered); 16 | Map map = new HashMap<>(); 17 | map.put("abc", 1); 18 | map.put("def", "Hello World"); 19 | map.put("ghj", new String[]{"Hello", "World"}); 20 | json.putAll(new JSONObject(map)); 21 | json.put("eins", 1); 22 | json.put("zwei", 2); 23 | json.put("drei", 3); 24 | json.put("vier", 4); 25 | json.put("fuenf", 5); 26 | return json; 27 | } 28 | 29 | JSONObject testObject; 30 | 31 | @Before 32 | public void setUp() throws Exception { 33 | this.testObject = testJson(true); 34 | } 35 | 36 | @After 37 | public void tearDown() throws Exception { 38 | } 39 | 40 | public void test() throws IOException { 41 | Object a = this.testObject.get("ghj"); 42 | assertTrue(a instanceof JSONArray); 43 | String t0 = this.testObject.toString(); 44 | JSONObject j0 = new JSONObject(t0); 45 | String t1 = j0.toString(); 46 | assertEquals(t0, t1); 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/org/loklak/ir/UserFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * UserFactory 3 | * Copyright 26.04.2015 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.ir; 21 | 22 | import org.json.JSONObject; 23 | import org.loklak.objects.UserEntry; 24 | 25 | public class UserFactory extends AbstractIndexFactory implements IndexFactory { 26 | 27 | public UserFactory(final ElasticsearchClient elasticsearch_client, final String index_name, final int cacheSize, final int existSize) { 28 | super(elasticsearch_client, index_name, cacheSize, existSize); 29 | } 30 | 31 | @Override 32 | public UserEntry init(JSONObject json) { 33 | return new UserEntry(json); 34 | } 35 | 36 | } -------------------------------------------------------------------------------- /src/org/loklak/ir/QueryFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * QueryFactory 3 | * Copyright 26.04.2015 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.ir; 21 | 22 | import org.json.JSONObject; 23 | import org.loklak.objects.QueryEntry; 24 | 25 | public class QueryFactory extends AbstractIndexFactory implements IndexFactory { 26 | 27 | public QueryFactory(final ElasticsearchClient elasticsearch_client, final String index_name, final int cacheSize, final int existSize) { 28 | super(elasticsearch_client, index_name, cacheSize, existSize); 29 | } 30 | 31 | @Override 32 | public QueryEntry init(JSONObject json) { 33 | return new QueryEntry(json); 34 | } 35 | 36 | } -------------------------------------------------------------------------------- /bin/profile_queries.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import time 4 | import json 5 | try: 6 | import requests 7 | except ImportError: 8 | print('Profiling script requires requests package. \n' 9 | 'You can install it by running "pip install requests"') 10 | exit() 11 | 12 | API_BASE_URL = 'http://127.0.0.1:9000/api/' 13 | 14 | # If user in bin directory set correct path to queries 15 | if os.getcwd().endswith('bin'): 16 | queries_folder = os.getcwd().rstrip('bin') + 'test/queries' 17 | else: 18 | queries_folder = os.getcwd() + '/test/queries' 19 | 20 | results = [] 21 | test_queries = [] 22 | for file_name in os.listdir(queries_folder): 23 | if file_name.endswith('.txt') and file_name != 'README.txt': 24 | for query in open('{}/{}'.format(queries_folder, file_name)).readlines(): 25 | test_queries.append(query.rstrip()) 26 | 27 | print('Start profiling {} queries'.format(len(test_queries))) 28 | for query in test_queries: 29 | time_start = time.time() 30 | 31 | url = API_BASE_URL + 'search.json?source=cache&q={}'.format(query) 32 | result = json.loads(requests.get(url).text) 33 | assert 'statuses' in result 34 | 35 | time_finish = time.time() 36 | results.append(time_finish - time_start) 37 | 38 | print('Profiling finished.') 39 | print('Average time: {} seconds'.format(sum(results) / len(results))) 40 | print('Minimal time: {} seconds'.format(min(results))) 41 | print('Maximal time: {} seconds'.format(max(results))) 42 | -------------------------------------------------------------------------------- /src/org/loklak/objects/ProviderType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * ProviderType 3 | * Copyright 22.02.2015 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; wo even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.objects; 21 | 22 | /** 23 | * The ProviderType objects answers on the question "who provided that content". 24 | * Do not mix up this with the SourceType. 25 | * There can be many providers for the same SourceType. 26 | */ 27 | public enum ProviderType { 28 | 29 | NOONE, // value assigned during instantiation phase 30 | USER, // generated by a user as generic message 31 | SCRAPED, // scraped with this peer from a remote service 32 | IMPORT, // external resource imported with special reader 33 | REMOTE, // pushed as message bulk from a remote peer 34 | 35 | } 36 | -------------------------------------------------------------------------------- /bin/installation.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # If you're looking for the variables, please go to bin/.preload.sh 4 | 5 | # Make sure we're on project root 6 | cd $(dirname $0)/.. 7 | 8 | # Execute preload script 9 | source bin/.preload.sh 10 | 11 | echo "starting loklak installation" 12 | echo "startup" > $STARTUPFILE 13 | 14 | cmdline="$cmdline -server -classpath $CLASSPATH -Dlog4j.configurationFile=$LOGCONFIG org.loklak.LoklakInstallation >> data/loklak.log 2>&1 &"; 15 | 16 | eval $cmdline 17 | PID=$! 18 | echo $PID > $PIDFILE 19 | 20 | while [ -f $STARTUPFILE ] && [ $(ps -p $PID -o pid=) ]; do 21 | if [ $(cat $STARTUPFILE) = 'done' ]; then 22 | break 23 | else 24 | sleep 1 25 | fi 26 | done 27 | 28 | if [ -f $STARTUPFILE ] && [ $(ps -p $PID -o pid=) ]; then 29 | CUSTOMPORT=$(grep -iw 'port.http' conf/config.properties | sed 's/^[^=]*=//' ); 30 | LOCALHOST=$(grep -iw 'shortlink.urlstub' conf/config.properties | sed 's/^[^=]*=//'); 31 | echo "loklak installation started at port $CUSTOMPORT, open your browser at $LOCALHOST" 32 | rm -f $STARTUPFILE 33 | 34 | echo "waiting for installation to finish" 35 | wait "$PID" 36 | if [ $? -eq 0 ]; then 37 | echo "loklak installation finished" 38 | echo 'done' > $INSTALLATIONCONFIG 39 | else 40 | echo "loklak installation aborted" 41 | fi 42 | 43 | exit 0 44 | else 45 | echo "loklak installation failed to start. See data/loklag.log for details. Here are the last logs:" 46 | tail data/loklak.log 47 | rm -f $STARTUPFILE 48 | exit 1 49 | fi 50 | -------------------------------------------------------------------------------- /docs/misc/StreamExample.md: -------------------------------------------------------------------------------- 1 | # Example of Streaming Messages from loklak Server 2 | 3 | ## Enabling Streaming 4 | 5 | To enable streaming in loklak, we first need to enable the stream flag in `conf/config.properties` - 6 | 7 | ```properties 8 | stream.enabled = true 9 | ``` 10 | 11 | ## Deciding the channels 12 | 13 | In order to listen to a stream, we first need to decide the channel which we want to listen on. 14 | 15 | To get an overview of channels available in loklak, take a look at [Stream Channels](StreamChannels.md). 16 | 17 | ## Connecting to Stream Using `EventSource` 18 | 19 | 20 | For this example, we take the peer name as `my.loklak.peer`, i.e. we can access the loklak API page at `http://my.loklak.peer`. It is assumed that this peer has streaming enabled. 21 | 22 | Let us try to access all the Tweets with hashtag `#OpenSource` 23 | 24 | ```javascript 25 | var eventSource = new EventSource('http://my.loklak.peer/api/stream.json?channel=twitter%2Fhashtag%2FOpenSource'); 26 | ``` 27 | 28 | ## Handling Events 29 | ```javascript 30 | eventSource.onopen = function() { 31 | console.log('Streaming Started at channel twitter/hashtag/OpenSource'); 32 | }; 33 | 34 | eventSource.onmessage = function(event) { 35 | console.log('Data received from loklak - ' + event.data); 36 | }; 37 | 38 | eventSource.onerror = function() { 39 | console.log('Eventsource failed!') 40 | } 41 | ``` 42 | 43 | Take a look at live example [here](stream.html). 44 | 45 | > More about [EventSource](https://developer.mozilla.org/en/docs/Web/API/EventSource). 46 | -------------------------------------------------------------------------------- /test/org/loklak/tools/UTF8Test.java: -------------------------------------------------------------------------------- 1 | package org.loklak.tools; 2 | 3 | import org.loklak.tools.UTF8; 4 | import org.junit.Test; 5 | import static org.junit.Assert.assertEquals; 6 | /** 7 | * These unit-tests test org.loklak.tools.UTF8.java 8 | */ 9 | public class UTF8Test{ 10 | 11 | @Test 12 | public void testGetBytes(){ 13 | UTF8 utf8 = new UTF8(); 14 | String str = "loklak-server"; 15 | byte [] returnVal = utf8.getBytes(str); 16 | String s = ""; 17 | for(int i=0; i. 18 | */ 19 | 20 | package org.loklak.ir; 21 | 22 | import org.json.JSONObject; 23 | import org.loklak.harvester.Post; 24 | import org.loklak.harvester.TwitterScraper.TwitterTweet; 25 | 26 | public class MessageFactory extends AbstractIndexFactory implements IndexFactory { 27 | 28 | public MessageFactory(final ElasticsearchClient elasticsearch_client, final String index_name, final int cacheSize, final int existSize) { 29 | super(elasticsearch_client, index_name, cacheSize, existSize); 30 | } 31 | 32 | @Override 33 | public TwitterTweet init(JSONObject json) { 34 | return new TwitterTweet(json); 35 | } 36 | 37 | } 38 | 39 | -------------------------------------------------------------------------------- /src/org/loklak/ir/AccountFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * AccountFactory 3 | * Copyright 27.05.2015 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.ir; 21 | 22 | import java.io.IOException; 23 | 24 | import org.json.JSONObject; 25 | import org.loklak.objects.AccountEntry; 26 | 27 | public class AccountFactory extends AbstractIndexFactory implements IndexFactory { 28 | 29 | public AccountFactory(final ElasticsearchClient elasticsearch_client, final String index_name, final int cacheSize, final int existSize) { 30 | super(elasticsearch_client, index_name, cacheSize, existSize); 31 | } 32 | 33 | @Override 34 | public AccountEntry init(JSONObject map) throws IOException { 35 | return new AccountEntry(map); 36 | } 37 | 38 | } -------------------------------------------------------------------------------- /src/org/loklak/ir/BulkWriteResult.java: -------------------------------------------------------------------------------- 1 | /** 2 | * BulkWriteResult 3 | * Copyright 14.11.2018 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.ir; 21 | 22 | import java.util.LinkedHashMap; 23 | import java.util.LinkedHashSet; 24 | import java.util.Map; 25 | import java.util.Set; 26 | 27 | public class BulkWriteResult { 28 | 29 | private Map errors; 30 | private Set created; 31 | 32 | public BulkWriteResult() { 33 | this.errors = new LinkedHashMap<>(); 34 | this.created = new LinkedHashSet<>(); 35 | } 36 | 37 | public Map getErrors() { 38 | return this.errors; 39 | } 40 | 41 | public Set getCreated() { 42 | return this.created; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/org/loklak/ir/ImportProfileFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * ImportProfileFactory 3 | * Copyright 24.07.2015 by Dang Hai An, @zyzo 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | package org.loklak.ir; 20 | 21 | import org.json.JSONObject; 22 | import org.loklak.objects.ImportProfileEntry; 23 | 24 | import java.io.IOException; 25 | 26 | public class ImportProfileFactory extends AbstractIndexFactory implements IndexFactory { 27 | 28 | public ImportProfileFactory(ElasticsearchClient elasticsearch_client, String index_name, int cacheSize, final int existSize) { 29 | super(elasticsearch_client, index_name, cacheSize, existSize); 30 | } 31 | 32 | @Override 33 | public ImportProfileEntry init(JSONObject map) throws IOException { 34 | return new ImportProfileEntry(map); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | - Issue type: Bug report/Feature request 5 | 6 | ### Short description 7 | 8 | 9 | 12 | ### Environment 13 | 14 | - Operating system: 15 | - Software version: 16 | - Software source: 17 | 18 | ### Steps to reproduce 19 | 20 | 1. 21 | 2. 22 | 3. 23 | 24 | ### Expected behaviour 25 | 26 | 27 | ### Actual behaviour 28 | 29 | 30 | 33 | ### Usecase 34 | 35 | 36 | ### Description 37 | -------------------------------------------------------------------------------- /bin/.preload.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | JARFILE="build/libs/loklak_server-all.jar" 4 | INSTALLATIONCONFIG="data/settings/installation.txt" 5 | PIDFILE="data/loklak.pid" 6 | DFAULTCONFIG="conf/config.properties" 7 | CUSTOMCONFIG="data/settings/customized_config.properties" 8 | LOGCONFIG="conf/logs/log-to-file.properties" 9 | STARTUPFILE="data/startup.tmp" 10 | DFAULTXmx="1024m"; 11 | CUSTOMXmx="" 12 | 13 | mkdir -p data/settings 14 | 15 | #to not allow process to overwrite the already running one. 16 | if [ -f $PIDFILE ]; then 17 | PID=$(cat $PIDFILE 2>/dev/null) 18 | if kill $PID > /dev/null 2>&1; then 19 | echo "Server is already running, please stop it and then start" 20 | exit 1 21 | else 22 | rm $PIDFILE 23 | fi 24 | fi 25 | 26 | if [ -f $DFAULTCONFIG ]; then 27 | j="$(grep Xmx $DFAULTCONFIG | sed 's/^[^=]*=//')"; 28 | if [ -n $j ]; then DFAULTXmx="$j"; fi; 29 | fi 30 | if [ -f $CUSTOMCONFIG ]; then 31 | j="$(grep Xmx $CUSTOMCONFIG | sed 's/^[^=]*=//')"; 32 | if [ -n $j ]; then CUSTOMXmx="$j"; fi; 33 | fi 34 | 35 | if [ -f $JARFILE ]; then 36 | CLASSPATH="$JARFILE" 37 | else 38 | echo "It seems you haven't compile Loklak" 39 | echo "To build Loklak," 40 | echo "$ ./gradlew build" 41 | exit 1 42 | fi 43 | 44 | cmdline="java"; 45 | 46 | if [ -n "$ENVXmx" ] ; then cmdline="$cmdline -Xmx$ENVXmx"; 47 | elif [ -n "$CUSTOMXmx" ]; then cmdline="$cmdline -Xmx$CUSTOMXmx"; 48 | elif [ -n "$DFAULTXmx" ]; then cmdline="$cmdline -Xmx$DFAULTXmx"; 49 | fi 50 | 51 | export INSTALLATIONCONFIG 52 | export LOGCONFIG 53 | export STARTUPFILE 54 | export CLASSPATH 55 | -------------------------------------------------------------------------------- /conf/schema/openwifimap.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema" : "http://json-schema.org/draft-04/schema#", 3 | "type": "object", 4 | "description": "Openwifimap API https://github.com/freifunk/openwifimap-api", 5 | "properties": { 6 | "total_rows": { 7 | "title": "Total rows", 8 | "type": "integer", 9 | "description": "Total number of results" 10 | }, 11 | "offset": { 12 | "title": "Offset", 13 | "type": "integer" 14 | }, 15 | "rows": { 16 | "title": "Rows", 17 | "type": "array", 18 | "uniqueItems": true, 19 | "items": { 20 | "type": "object", 21 | "properties": { 22 | "id": { 23 | "type": "string" 24 | }, 25 | "key": { 26 | "type": "string" 27 | }, 28 | "value": { 29 | "type": "object", 30 | "properties": { 31 | "hostname": { 32 | "type": "string" 33 | }, 34 | "ctime": { 35 | "type": "string" 36 | }, 37 | "mtime": { 38 | "type": "string" 39 | }, 40 | "id": { 41 | "type": "string" 42 | }, 43 | "latlng": { 44 | "type": "array", 45 | "items": { 46 | "type": "number" 47 | }, 48 | "minItems": 2, 49 | "maxItems": 2 50 | } 51 | }, 52 | "required": ["hostname", "ctime", "mtime", "id", "latlng"] 53 | } 54 | } 55 | } 56 | } 57 | } 58 | } -------------------------------------------------------------------------------- /docs/development/eclipseSetup.md: -------------------------------------------------------------------------------- 1 | # How to setup Loklak Server on Eclipse 2 | 3 | If you already have a local copy of `loklak_server` repository 4 | 5 | 1. Download and install [Eclipse](https://eclipse.org/downloads/) 6 | 2. Open Eclipse and choose `Import Project` 7 | 3. In the window that opens up, Choose `git` and press `Next` 8 | 4. Choose a local repository and Add the repository to the search results window. 9 | 5. Press Finish 10 | 6. Then choose the `loklak_server` git repository in the list and proceed in the wizard. 11 | 7. Choose the radio option which says `Import existing Eclipse Projects` 12 | 8. Press Finish to open up the Package Explorer and the IDE should be available with the project opened. 13 | 9. Once the repository is ready using the Package Explorer navigate to `org.loklak.LoklakServer` 14 | 10. Right click on LoklakServer, Choose `Run As > Run Configurations` 15 | 11. Choose `Java Application` and press the `New Configuration` button in the top left corner of this pane. 16 | 12. In the frame panel that opens up, Move to the Arguments Tab and add the VM Arguments as `-Xmx2G -Xms2G -server -ea` 17 | 13. Click `Apply` and then `Close` the window 18 | 19 | You can use Eclipse to download the loklak_server from the `git url` in Step 4, Instead of choosing `Local Repository` 20 | choose `Remote URL` and use the git link `https://github.com/loklak/loklak_server.git` and follow the rest of the instructions 21 | 22 | ## Using the Ant File 23 | 1. Choose `File > New > Project` 24 | 2. Choose `Java > Java Project from existing ant file` 25 | 3. Navigate to the local copy of the repository and use the build file to open up `loklak_server` 26 | -------------------------------------------------------------------------------- /src/org/loklak/data/IndexEntry.java: -------------------------------------------------------------------------------- 1 | /** 2 | * IndexEntry 3 | * Copyright 16.05.2016 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.data; 21 | 22 | import org.loklak.objects.ObjectEntry; 23 | import org.loklak.objects.SourceType; 24 | 25 | public class IndexEntry { 26 | 27 | private final String id; 28 | private final SourceType type; 29 | private final IndexObject obj; 30 | 31 | public IndexEntry(String id, SourceType type, IndexObject obj) { 32 | this.id = id; 33 | this.type = type; 34 | this.obj = obj; 35 | } 36 | 37 | public String getId() { 38 | return this.id; 39 | } 40 | 41 | public SourceType getType() { 42 | return this.type; 43 | } 44 | 45 | public IndexObject getObject() { 46 | return this.obj; 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-alpine AS builder 2 | LABEL maintainer="Michael Peter Christen " 3 | 4 | # build the image with (i.e.) 5 | # docker build -t loklak/loklak_server:latest . 6 | 7 | # start the image with (i.e.) 8 | # docker run -d -p 9000:9000 9 | 10 | # copy the required parts of the source code 11 | ADD src /loklak_server/src/ 12 | ADD gradle /loklak_server/gradle/ 13 | ADD gradlew /loklak_server/ 14 | ADD build.gradle /loklak_server/ 15 | ADD settings.gradle /loklak_server/ 16 | 17 | # compile loklak 18 | RUN cd /loklak_server && ./gradlew build --no-daemon -x checkstyleMain -x checkstyleTest -x jacocoTestReport 19 | 20 | # Second Stage 21 | FROM openjdk:8-jre-alpine 22 | 23 | # install required software 24 | RUN apk update && apk add --no-cache bash 25 | 26 | # Create Volume for persistence 27 | VOLUME ["/loklak_server/data"] 28 | 29 | # loklak start 30 | CMD ["/loklak_server/bin/start.sh", "-Idn"] 31 | 32 | # setup locales 33 | ENV LANG=en_US.UTF-8 34 | 35 | # Expose the web interface ports 36 | EXPOSE 9000 9443 37 | 38 | # copy the required parts of the source code 39 | WORKDIR /loklak_server/ 40 | COPY --from=builder /loklak_server/build/libs/loklak_server-all.jar build/libs/ 41 | ADD bin /loklak_server/bin/ 42 | ADD conf /loklak_server/conf/ 43 | ADD html /loklak_server/html/ 44 | ADD installation /loklak_server/installation/ 45 | ADD ssi /loklak_server/ssi/ 46 | ADD test/queries /loklak_server/test/queries/ 47 | 48 | # change config file 49 | RUN sed -i 's/^\(upgradeInterval=\).*/\186400000000/' /loklak_server/conf/config.properties 50 | 51 | # set current working directory to loklak_server 52 | WORKDIR /loklak_server 53 | -------------------------------------------------------------------------------- /src/org/json/JSONPointerException.java: -------------------------------------------------------------------------------- 1 | package org.json; 2 | 3 | /* 4 | Copyright (c) 2002 JSON.org 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | The Software shall be used for Good, not Evil. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | SOFTWARE. 25 | */ 26 | 27 | /** 28 | * The JSONPointerException is thrown by {@link JSONPointer} if an error occurs 29 | * during evaluating a pointer. 30 | * 31 | * @author JSON.org 32 | * @version 2016-05-13 33 | */ 34 | public class JSONPointerException extends JSONException { 35 | private static final long serialVersionUID = 8872944667561856751L; 36 | 37 | public JSONPointerException(String message) { 38 | super(message); 39 | } 40 | 41 | public JSONPointerException(String message, Throwable cause) { 42 | super(message, cause); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /test/queries/chinese_01.txt: -------------------------------------------------------------------------------- 1 | 奇怪 2 | 你好 3 | 中文 4 | 英文 5 | 巧克力 6 | 情人节 7 | 建军节 8 | ＃圣诞老人 9 | ＃拉斯维加斯 10 | 美国 11 | 阿黛尔 12 | 网络 13 | 开学 14 | 春节 15 | 阅兵 16 | 国庆 17 | 百度 18 | 谷歌 19 | 加拿大 20 | 起床 21 | 太阳 22 | 电视 23 | 邮件 24 | 视频 25 | 打开 26 | 视频 27 | 早上好 28 | 老师 29 | 汉语 30 | 上班 31 | 工作 32 | 教材 33 | 经典 34 | 兴趣 35 | 爆笑 36 | 知识。 37 | 太太 38 | 白富美 39 | 大男子主义 40 | 宅男 41 | 独生子女 42 | 婆媳关系 43 | 中国特色 44 | 枯燥 45 | 生态 46 | 婚恋 47 | 牛人 48 | 另类 49 | 亚马逊 50 | 课本 51 | 网民 52 | 图文并茂的 53 | 图片 54 | 雷倒 55 | 游泳池 56 | 辣妹 57 | 垂涎三尺 58 | 搞笑 59 | 学不到 60 | 正统 61 | 沟通 62 | 娱乐 63 | 快乐 64 | 友谊 65 | 安卓 66 | 苹果 67 | 土豪 68 | 大妈 69 | 习近平 70 | 绿水青 71 | 解放军 72 | 微博 73 | 外媒 74 | 热词 75 | 深圳 76 | 滑坡 77 | 责任人 78 | 垮塌 79 | 滑坡土 80 | 企业 81 | 好莱坞 82 | 海外出口 83 | 迷雾 84 | 大片 85 | 黑龙江 86 | 父亲人 87 | 官员 88 | 落马 89 | 异常 90 | 容貌 91 | 转账 92 | 免费 93 | 央行 94 | 收银 95 | 地陷 96 | 消失 97 | 受审 98 | 现金 99 | 作弊 100 | 北京 101 | 二孩 102 | 元旦 103 | 实施 104 | 生育 105 | 抚养费 106 | 宁夏 107 | 落马 108 | 迷信活动 109 | 安倍 110 | 靖国神社 111 | 时间 112 | 联通 113 | 拥护 114 | 雾霾 115 | 坠亡 116 | 卸任 117 | 药企 118 | 趋势 119 | 变革 120 | 愿景 121 | 局长 122 | A股 123 | 出逃 124 | 中信 125 | 抛售 126 | 狼队 127 | 纪录片 128 | 截图 129 | 自焚 130 | ＃反腐 131 | 调查 132 | 电信 133 | 公证 134 | 房产证 135 | 债务 136 | 整容 137 | 红包 138 | 五百 139 | 出版社 140 | 航空 141 | 欧洲 142 | 警告 143 | 反恐 144 | 威胁 145 | 奥巴马 146 | 陆战队 147 | 肌肉 148 | 南海 149 | 理解 150 | 慰安妇 151 | 英国 152 | 英镑 153 | 土耳其 154 | 士兵 155 | 独角兽 156 | 关系 157 | 女子 158 | 英国 159 | 伴郎 160 | 古巴人 161 | 惊慌 162 | 避难 163 | 叙利亚 164 | 危机 165 | 章子怡 166 | 汪峰 167 | 李冰冰 168 | 王思聪 169 | 特殊对待 170 | 马苏 171 | 胸 172 | 李易峰 173 | 吴昕 174 | 绯闻 175 | 证据 176 | 苏宁 177 | 曼联 178 | 中超 179 | 皇马 180 | 主席 181 | 创业 182 | 谎言 183 | 违纪 184 | 詹姆斯 185 | 腾讯 186 | 360 187 | 结盟 188 | 流量 189 | 硅谷 190 | 创业 191 | 小米 192 | 朝鲜 193 | 农产品 194 | 金融 195 | 互联网 196 | 大片 197 | 签署 198 | 时代 199 | 机器人 200 | 威胁 201 | 人类 202 | 离职 203 | -------------------------------------------------------------------------------- /test/org/loklak/geo/GeoLocationTest.java: -------------------------------------------------------------------------------- 1 | package org.loklak.geo; 2 | 3 | import org.loklak.geo.GeoLocation; 4 | 5 | import org.junit.Test; 6 | import java.util.ArrayList; 7 | import java.util.Collection; 8 | import static org.junit.Assert.assertEquals; 9 | import static org.junit.Assert.assertNotNull; 10 | 11 | /** 12 | * This is a test file for testing methods of org.loklak.geo.GeoLocation.java 13 | */ 14 | public class GeoLocationTest { 15 | 16 | @Test 17 | public void geoLocation() { 18 | 19 | Collection names = new ArrayList(); 20 | names.add("Taj Mahal"); 21 | 22 | final double lat = 27.175015; 23 | final double lon = 78.042155; 24 | final String iso3166cc = "IN-UP"; 25 | 26 | /** Population value is just a testing value. It may differ from actual population. */ 27 | long population = 400; 28 | int kmValue = 3025; 29 | 30 | GeoLocation geoLocation = new GeoLocation(lat, lon, names, iso3166cc); 31 | geoLocation.setPopulation(population); 32 | 33 | Collection returnedNames = geoLocation.getNames(); 34 | String returnedISO3166cc = geoLocation.getISO3166cc(); 35 | long returnedPopulation = geoLocation.getPopulation(); 36 | int returnedKmValue = geoLocation.degreeToKm(27.175015); 37 | String returnedString = geoLocation.toString(); 38 | 39 | assertNotNull(returnedNames); 40 | assertNotNull(returnedISO3166cc); 41 | assertNotNull(returnedPopulation); 42 | assertNotNull(returnedKmValue); 43 | assertNotNull(returnedString); 44 | assertEquals(names, returnedNames); 45 | assertEquals(iso3166cc, returnedISO3166cc); 46 | assertEquals(population, returnedPopulation); 47 | assertEquals(kmValue, returnedKmValue); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /test/org/loklak/tools/storage/AsciiTest.java: -------------------------------------------------------------------------------- 1 | package org.loklak.tools.storage; 2 | 3 | import org.loklak.tools.ASCII; 4 | 5 | import org.junit.Test; 6 | import static org.junit.Assert.assertEquals; 7 | /** 8 | * These unit-tests test org.loklak.tools.ASCII.java 9 | */ 10 | public class AsciiTest { 11 | /** 12 | * This method tests 'compare' method of org.loklak.tools.ASCII.java 13 | */ 14 | @Test 15 | public void testCompare() { 16 | 17 | ASCII ascii = new ASCII(true); 18 | String s0 = "abc"; 19 | String s1 = "xyz"; 20 | int returnedValue1 = ascii.compare(s0, s1); 21 | int returnedValue2 = ascii.compare(s1, s0); 22 | int returnedValue3 = ascii.compare(null, null); 23 | int returnedValue4 = ascii.compare(s0, null); 24 | int returnedValue5 = ascii.compare(null, s1); 25 | assertEquals(-1, returnedValue1); 26 | assertEquals(1, returnedValue2); 27 | assertEquals(0, returnedValue3); 28 | assertEquals(1, returnedValue4); 29 | assertEquals(-1, returnedValue5); 30 | } 31 | 32 | /** 33 | * This method tests 'equals' method of org.loklak.tools.ASCII.java 34 | */ 35 | @Test 36 | public void testEquals() { 37 | 38 | ASCII ascii = new ASCII(true); 39 | String s0 = "abc"; 40 | String s1 = "xyz"; 41 | boolean returnedValue1 = ascii.equals(s0, s1); 42 | boolean returnedValue2 = ascii.equals(s1, s0); 43 | boolean returnedValue3 = ascii.equals(null, null); 44 | boolean returnedValue4 = ascii.equals(s0, null); 45 | boolean returnedValue5 = ascii.equals(null, s1); 46 | assertEquals(false, returnedValue1); 47 | assertEquals(false, returnedValue2); 48 | assertEquals(true, returnedValue3); 49 | assertEquals(false, returnedValue4); 50 | assertEquals(false, returnedValue5); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/org/json/JSONPropertyIgnore.java: -------------------------------------------------------------------------------- 1 | package org.json; 2 | 3 | /* 4 | Copyright (c) 2018 JSON.org 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | The Software shall be used for Good, not Evil. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | SOFTWARE. 25 | */ 26 | 27 | import static java.lang.annotation.ElementType.METHOD; 28 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 29 | 30 | import java.lang.annotation.Documented; 31 | import java.lang.annotation.Retention; 32 | import java.lang.annotation.Target; 33 | 34 | @Documented 35 | @Retention(RUNTIME) 36 | @Target({METHOD}) 37 | /** 38 | * Use this annotation on a getter method to override the Bean name 39 | * parser for Bean -> JSONObject mapping. If this annotation is 40 | * present at any level in the class hierarchy, then the method will 41 | * not be serialized from the bean into the JSONObject. 42 | */ 43 | public @interface JSONPropertyIgnore { } 44 | -------------------------------------------------------------------------------- /test/org/loklak/tools/storage/JsonDatasetTest.java: -------------------------------------------------------------------------------- 1 | package org.loklak.tools.storage; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | 6 | import junit.framework.TestCase; 7 | 8 | import org.json.JSONObject; 9 | import org.json.JSONObjectTest; 10 | import org.junit.After; 11 | import org.junit.Before; 12 | import org.loklak.tools.BufferedRandomAccessFile; 13 | import org.loklak.tools.storage.JsonDataset.Column; 14 | import org.loklak.tools.storage.JsonDataset.JsonFactoryIndex; 15 | 16 | public class JsonDatasetTest extends TestCase { 17 | 18 | private File testFile; 19 | 20 | @Before 21 | public void setUp() throws Exception { 22 | this.testFile = BufferedRandomAccessFile.Test.getTestFile(); 23 | } 24 | 25 | @After 26 | public void tearDown() throws Exception { 27 | this.testFile.delete(); 28 | } 29 | 30 | public void test() throws IOException { 31 | JsonFactoryIndex idx; 32 | for (JsonRepository.Mode mode: JsonRepository.Mode.values()) try { 33 | 34 | // write the file 35 | JsonDataset dtst = new JsonDataset(this.testFile, "idx_", new Column[]{new Column("abc", true), new Column("def", false)}, null, null, mode, false, Integer.MAX_VALUE); 36 | JSONObject json = JSONObjectTest.testJson(true); 37 | dtst.putUnique(json); 38 | dtst.close(); 39 | 40 | // read the file 41 | dtst = new JsonDataset(this.testFile, "idx_", new Column[]{new Column("abc", true), new Column("def", false)}, null, null, mode, false, Integer.MAX_VALUE); 42 | idx = dtst.index.get("abc"); 43 | System.out.println(idx.get(1)); 44 | idx = dtst.index.get("def"); 45 | System.out.println(idx.get("Hello World")); 46 | 47 | } catch (IOException e) { 48 | e.printStackTrace(); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /docs/Dockerfile-learnings.md: -------------------------------------------------------------------------------- 1 | # Massive decrease of Docker Image 2 | 3 | Before: 4 | ``` 5 | REPOSITORY TAG IMAGE ID CREATED SIZE 6 | loklak_server latest 6775047619d8 Less than a second ago 751.6 MB 7 | ubuntu latest 104bec311bcd 9 days ago 129 MB 8 | ``` 9 | [Travis Build](https://travis-ci.org/yukiisbored/loklak_server/builds/186665492) 10 | 11 | After: 12 | ``` 13 | REPOSITORY TAG IMAGE ID CREATED SIZE 14 | loklak_server latest 1559b163a3dc 1 seconds ago 173.5 MB 15 | alpine latest baa5d63471ea 9 weeks ago 4.803 MB 16 | ``` 17 | [Travis Build](https://travis-ci.org/yukiisbored/loklak_server/builds/186675655) 18 | 19 | ## How I did it? 20 | 21 | This is pretty easy, because it uses Java without having weird native libraries 22 | 23 | So to minify it to the extreme, This is how I come up with this: 24 | 25 | 1. Use a smaller distro/distro that has smaller packages/a distro that focuses 26 | on being minimal 27 | 2. DO NOT USE X 28 | 3. Remove development tools because we don't need those 29 | 4. Do not create backup files 30 | 5. Make it into a single `RUN` command for customizability (so we can remove 31 | development tools at the end) 32 | 6. Use packages that doesn't require X (X is heavy and we don't even require X, 33 | We can't even use X without doing fancy stuff like an X via SSH) 34 | 7. Do not cache anything, good thing Alpine doesn't cache by default 35 | 8. Copy only the required files 36 | 37 | ## What didn't work? 38 | 39 | 1. GNU Coreutils specific commands/arguments 40 | 2. Not having `bash` 41 | 42 | ## How I made it work? 43 | 44 | 1. Replace `ps` statements into simple `kill -0` instructions since every 45 | `POSIX`-compliant has this feature 46 | 2. Install `bash` 47 | -------------------------------------------------------------------------------- /conf/elasticsearch/mappings/import_profiles.json: -------------------------------------------------------------------------------- 1 | { 2 | "_all": {"enabled": false}, 3 | "properties":{ 4 | "timestamp":{ 5 | "type":"date", "format":"dateOptionalTime", "include_in_all":"false", "index":"not_analyzed" 6 | }, 7 | "id":{ 8 | "type":"string", "include_in_all":"false", "index":"not_analyzed" 9 | }, 10 | "source_url":{ 11 | "type":"string", "include_in_all":"false", "index":"not_analyzed" 12 | }, 13 | "source_type":{ 14 | "type":"string", "include_in_all":"false", "index":"not_analyzed" 15 | }, 16 | "source_hash":{ 17 | "type":"long", "include_in_all":"false", "index":"not_analyzed" 18 | }, 19 | "harvesting_freq":{ 20 | "type":"long", "include_in_all":"false", "index":"not_analyzed" 21 | }, 22 | "lifetime":{ 23 | "type":"long", "include_in_all":"false", "index":"not_analyzed" 24 | }, 25 | "created_at":{ 26 | "type":"date", "format":"dateOptionalTime", "include_in_all":"false", "index":"not_analyzed" 27 | }, 28 | "last_modified":{ 29 | "type":"date", "format":"dateOptionalTime", "include_in_all":"false", "index":"not_analyzed" 30 | }, 31 | "last_harvested":{ 32 | "type":"date", "format":"dateOptionalTime", "include_in_all":"false", "index":"not_analyzed" 33 | }, 34 | "importer":{ 35 | "type":"string", "include_in_all":"false", "index":"not_analyzed" 36 | }, 37 | "client_host":{ 38 | "type":"string", "include_in_all":"false", "index":"not_analyzed" 39 | }, 40 | "active_status":{ 41 | "type":"string", "include_in_all":"false", "index":"not_analyzed" 42 | }, 43 | "privacy_status":{ 44 | "type":"string", "include_in_all":"false", "index":"not_analyzed" 45 | }, 46 | "imported":{ 47 | "type":"string", "include_in_all":"false", "index":"not_analyzed" 48 | }, 49 | "sharers":{ 50 | "type":"string", "include_in_all":"false", "index":"not_analyzed" 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /test/queries/cantonese_00.txt: -------------------------------------------------------------------------------- 1 | 香港 2 | 溫布頓錦標賽 3 | 世界杯欖球賽 4 | 奧斯卡金像獎頒獎禮 5 | 奧斯卡 6 | 美國網球公開賽 7 | 澳洲網球公開賽 8 | 法國網球公開賽 9 | 板球世界杯 10 | 世界女排大獎賽 11 | 世界特殊奧運會 12 | 環球小姐 13 | 李光耀 14 | 洪秀柱 15 | 彭麗媛 16 | 杜如風 17 | 袁嘉敏 18 | 麥明詩 19 | 司馬燕 20 | 羅霖 21 | 曾江 22 | 蔡思貝 23 | 陳瀅 24 | 中東呼吸綜合症 25 | 天津爆炸 26 | 希臘公投 27 | 伊斯蘭國 28 | 八仙樂園 29 | 尼泊爾大地震 30 | 巴黎恐襲 31 | 藍月亮 32 | 颱風蓮花 33 | 颱風 34 | 香港書展 35 | 書展 36 | 荔園 37 | 歐陸嘉年華 38 | 嘉年華 39 | 電腦通訊節 40 | 動漫電玩節 41 | 動漫 42 | 渣打馬拉松 43 | 馬拉松 44 | 香港巴塞爾藝術展 45 | 香港年宵市場 46 | 年宵 47 | 香港花卉展覽 48 | 花卉 49 | 美食博覽 50 | 美食 51 | 區議會選舉 52 | 區議會 53 | 選舉 54 | 普選政改方案 55 | 普選 56 | 政改 57 | 抗戰勝利 58 | 財政預算案 59 | 中港世界杯外圍賽 60 | 世界杯 61 | 世界杯外圍賽 62 | 白居二計劃 63 | 大時代 64 | 羅生門 65 | 迪士尼 66 | 毛記電視分獎典禮 67 | 毛記電視 68 | 河國榮 69 | 搖滾 70 | 搖滾音樂家 71 | 音樂家 72 | 變色龍 73 | 洗版 74 | 歌曲 75 | 司機 76 | 太陽 77 | 進攻 78 | 中學 79 | 自修課 80 | 班主任 81 | 嘉賓 82 | 頒獎禮 83 | 人氣 84 | 日常生活 85 | 演出 86 | 合唱團 87 | 放題 88 | 討論區 89 | 演藝界 90 | 訪問 91 | 二次創作 92 | 社會 93 | 文字 94 | 畫畫 95 | 意見 96 | 排行榜 97 | 冠軍 98 | 專輯 99 | 目光 100 | 亞太區 101 | 人民幣 102 | 金管局 103 | 銀行 104 | 市場 105 | 風險 106 | 管理 107 | 監察 108 | 司法覆核 109 | 法官 110 | 濫用 111 | 土地 112 | 法制 113 | 大律師公會 114 | 大律師 115 | 風筒 116 | 士巴拿 117 | 抽氣扇 118 | 發泡膠 119 | 拖板 120 | 多士爐 121 | 冷氣 122 | 插蘇 123 | 火牛 124 | 紙皮箱 125 | 電飯煲 126 | 雞毛掃 127 | 番鹼 128 | 地拖 129 | 雪櫃 130 | 插蘇頭 131 | 光管 132 | 燈掣 133 | 雪梨 134 | 布甸 135 | 雪糕 136 | 爆谷 137 | 奇異果 138 | 雞髀 139 | 雙皮奶 140 | 士多啤梨 141 | 燒肉 142 | 提子 143 | 蛋撻 144 | 車厘子 145 | 朱古力 146 | 單車 147 | 的士 148 | 電單車 149 | 救傷車 150 | 巴士 151 | 私家車 152 | 筷子 153 | 學校 154 | 飯 155 | 門 156 | 電影 157 | 屋 158 | 汽水 159 | 手錶 160 | 碗 161 | 電芯 162 | 眼淚 163 | 水 164 | 世界大戰 165 | 人口 166 | 方言 167 | 百花齊放 168 | 語言 169 | 殖民地 170 | 影響 171 | 潮流用語 172 | 網絡 173 | 補習班 174 | 震驚 175 | 熱門程度 176 | 娛樂 177 | 油價 178 | 天氣預報 179 | 智慧型手機 180 | 手機 181 | 規劃行程 182 | 加油站 183 | 優惠 184 | 價格 185 | 實習生 186 | 溫暖 187 | 觀眾 188 | 熱烈關注 189 | 名次 190 | 演員 191 | 劇情 192 | 服裝 193 | 古裝 194 | 韓劇 195 | 台劇 196 | 牛奶 197 | 網路霸凌 198 | 心理 199 | 奧運會 200 | 欖球 201 | -------------------------------------------------------------------------------- /test/api_test.sh: -------------------------------------------------------------------------------- 1 | curl http://localhost:9000/api/status.json 2 | curl http://localhost:9000/api/search.json?q=spacex 3 | curl "http://localhost:9000/api/search.json?q=spacex&source=cache" 4 | curl "http://localhost:9000/api/search.json?q=spacex&source=twitter" 5 | curl "http://localhost:9000/api/search.json?q=spacex&source=cache&count=0&fields=mentions,hashtags&limit=6" 6 | curl "http://localhost:9000/api/suggest.json?q=soj&orderby=query_count" 7 | curl "http://localhost:9000/api/suggest.json?count=20&order=asc" 8 | curl "http://localhost:9000/api/suggest.json?q=&timezoneOffset=-60&count=90&source=query&order=asc&orderby=retrieval_next&until=now&selectby=retrieval_next&random=3&minified=true&port.http=9000&port.https=9443&peername=anonymous" 9 | curl "localhost:9000/api/crawler.json?start=frankfurt&depth=2" 10 | curl http://localhost:9000/api/hello.json 11 | curl 'http://localhost:9000/api/geocode.json?data=%7B%22places%22:[%22Frankfurt%20am%20Main%22,%22New%20York%22,%22Singapore%22]%7D' 12 | curl http://localhost:9000/api/peers.json 13 | curl "http://localhost:9000/api/proxy.png?screen_name=loklak_app&url=https://pbs.twimg.com/profile_images/577512240640733184/fizL4YIn_bigger.png" 14 | curl http://localhost:9000/api/import.json 15 | curl http://localhost:9000/api/settings.json 16 | curl "http://localhost:9000/api/account.json?action=update&data=%7B%22screen_name%22:%22test%22,%22oauth_token%22:%22abc%22,%22oauth_token_secret%22:%22def%22%7D" 17 | curl "http://localhost:9000/api/account.json?screen_name=test" 18 | curl "http://localhost:9000/api/account.json?action=update&data=%7B%22screen_name%22:%22test%22,%22apps%22:%7B%22wall%22:%7B%22type%22:%22horizontal%22%7D%7D%7D" 19 | curl http://localhost:9000/api/user.json?screen_name=loklak_app 20 | curl http://localhost:9000/api/threaddump.txt 21 | curl http://localhost:9000/vis/map.png > /dev/null 22 | curl "http://localhost:9000/vis/markdown.png?text=hello%20world%0Dhello%20universe&color_text=000000&color_background=ffffff&padding=3" > /dev/null 23 | -------------------------------------------------------------------------------- /src/org/json/JSONString.java: -------------------------------------------------------------------------------- 1 | package org.json; 2 | 3 | /* 4 | Copyright (c) 2002 JSON.org 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | The Software shall be used for Good, not Evil. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | SOFTWARE. 25 | */ 26 | 27 | /** 28 | * The JSONString interface allows a toJSONString() 29 | * method so that a class can change the behavior of 30 | * JSONObject.toString(), JSONArray.toString(), 31 | * and JSONWriter.value(Object). The 32 | * toJSONString method will be used instead of the default behavior 33 | * of using the Object's toString() method and quoting the result. 34 | */ 35 | public interface JSONString { 36 | /** 37 | * The toJSONString method allows a class to produce its own JSON 38 | * serialization. 39 | * 40 | * @return A strictly syntactically correct JSON text. 41 | */ 42 | public String toJSONString(); 43 | } 44 | -------------------------------------------------------------------------------- /kubernetes/yamls/api/elasticsearch/es-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: es 5 | namespace: elasticsearch 6 | labels: 7 | component: elasticsearch 8 | spec: 9 | replicas: 1 10 | template: 11 | metadata: 12 | labels: 13 | component: elasticsearch 14 | spec: 15 | serviceAccount: elasticsearch 16 | containers: 17 | - name: es 18 | livenessProbe: 19 | httpGet: 20 | path: /_cluster/health 21 | port: 9200 22 | initialDelaySeconds: 30 23 | timeoutSeconds: 3 24 | readinessProbe: 25 | httpGet: 26 | path: /_cluster/health 27 | port: 9200 28 | initialDelaySeconds: 30 29 | timeoutSeconds: 3 30 | securityContext: 31 | capabilities: 32 | add: 33 | - IPC_LOCK 34 | image: quay.io/pires/docker-elasticsearch-kubernetes:2.0.0 35 | env: 36 | - name: KUBERNETES_CA_CERTIFICATE_FILE 37 | value: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt 38 | - name: NAMESPACE 39 | valueFrom: 40 | fieldRef: 41 | fieldPath: metadata.namespace 42 | - name: "CLUSTER_NAME" 43 | value: "loklakcluster" 44 | - name: "DISCOVERY_SERVICE" 45 | value: "elasticsearch" 46 | - name: NODE_MASTER 47 | value: "true" 48 | - name: NODE_DATA 49 | value: "true" 50 | - name: HTTP_ENABLE 51 | value: "true" 52 | ports: 53 | - containerPort: 9200 54 | name: http 55 | protocol: TCP 56 | - containerPort: 9300 57 | name: transport 58 | protocol: TCP 59 | volumeMounts: 60 | - mountPath: /data 61 | name: storage 62 | volumes: 63 | - name: storage 64 | gcePersistentDisk: 65 | pdName: data-index-disk 66 | fsType: ext4 67 | -------------------------------------------------------------------------------- /kubernetes/yamls/staging/elasticsearch/es-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: es 5 | namespace: elasticsearch 6 | labels: 7 | component: elasticsearch 8 | spec: 9 | replicas: 1 10 | template: 11 | metadata: 12 | labels: 13 | component: elasticsearch 14 | spec: 15 | serviceAccount: elasticsearch 16 | containers: 17 | - name: es 18 | livenessProbe: 19 | httpGet: 20 | path: /_cluster/health 21 | port: 9200 22 | initialDelaySeconds: 30 23 | timeoutSeconds: 3 24 | readinessProbe: 25 | httpGet: 26 | path: /_cluster/health 27 | port: 9200 28 | initialDelaySeconds: 30 29 | timeoutSeconds: 3 30 | securityContext: 31 | capabilities: 32 | add: 33 | - IPC_LOCK 34 | image: quay.io/pires/docker-elasticsearch-kubernetes:2.0.0 35 | env: 36 | - name: KUBERNETES_CA_CERTIFICATE_FILE 37 | value: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt 38 | - name: NAMESPACE 39 | valueFrom: 40 | fieldRef: 41 | fieldPath: metadata.namespace 42 | - name: "CLUSTER_NAME" 43 | value: "loklakcluster" 44 | - name: "DISCOVERY_SERVICE" 45 | value: "elasticsearch" 46 | - name: NODE_MASTER 47 | value: "true" 48 | - name: NODE_DATA 49 | value: "true" 50 | - name: HTTP_ENABLE 51 | value: "true" 52 | ports: 53 | - containerPort: 9200 54 | name: http 55 | protocol: TCP 56 | - containerPort: 9300 57 | name: transport 58 | protocol: TCP 59 | volumeMounts: 60 | - mountPath: /data 61 | name: storage 62 | volumes: 63 | - name: storage 64 | gcePersistentDisk: 65 | pdName: data-index-disk 66 | fsType: ext4 67 | -------------------------------------------------------------------------------- /src/org/loklak/api/cms/TopMenuService.java: -------------------------------------------------------------------------------- 1 | package org.loklak.api.cms; 2 | 3 | import javax.servlet.http.HttpServletResponse; 4 | 5 | import org.json.JSONArray; 6 | import org.json.JSONObject; 7 | import org.loklak.data.DAO; 8 | import org.loklak.server.APIHandler; 9 | import org.loklak.server.AbstractAPIHandler; 10 | import org.loklak.server.Authorization; 11 | import org.loklak.server.BaseUserRole; 12 | import org.loklak.server.Query; 13 | import org.loklak.tools.storage.JSONObjectWithDefault; 14 | 15 | public class TopMenuService extends AbstractAPIHandler implements APIHandler { 16 | 17 | private static final long serialVersionUID = 1839868262296635665L; 18 | 19 | @Override 20 | public BaseUserRole getMinimalBaseUserRole() { 21 | return BaseUserRole.ANONYMOUS; 22 | } 23 | 24 | @Override 25 | public JSONObject getDefaultPermissions(BaseUserRole baseUserRole) { 26 | return null; 27 | } 28 | 29 | @Override 30 | public String getAPIPath() { 31 | return "/cms/topmenu.json"; 32 | } 33 | 34 | @Override 35 | public JSONObject serviceImpl(Query call, HttpServletResponse response, Authorization rights, 36 | final JSONObjectWithDefault permissions) { 37 | 38 | int limited_count = (int) DAO.getConfig("download.limited.count", (long) Integer.MAX_VALUE); 39 | 40 | JSONObject json = new JSONObject(true); 41 | JSONArray topmenu = new JSONArray().put(new JSONObject().put("About", "http://loklak.org/about")) 42 | .put(new JSONObject().put("Search", "http://loklak.org")) 43 | .put(new JSONObject().put("Apps", "http://apps.loklak.org")) 44 | .put(new JSONObject().put("Blog", "http://blog.loklak.net")) 45 | .put(new JSONObject().put("Developers", "http://dev.loklak.org")) 46 | .put(new JSONObject().put("API", "http://api.loklak.org")); 47 | if (limited_count > 0 && DAO.writeDump) 48 | topmenu.put(new JSONObject().put("Dumps", "dump.html")); 49 | json.put("items", topmenu); 50 | 51 | // modify caching 52 | json.put("$EXPIRES", 600); 53 | return json; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/org/json/JSONPropertyName.java: -------------------------------------------------------------------------------- 1 | package org.json; 2 | 3 | /* 4 | Copyright (c) 2018 JSON.org 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | The Software shall be used for Good, not Evil. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | SOFTWARE. 25 | */ 26 | 27 | import static java.lang.annotation.ElementType.METHOD; 28 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 29 | 30 | import java.lang.annotation.Documented; 31 | import java.lang.annotation.Retention; 32 | import java.lang.annotation.Target; 33 | 34 | @Documented 35 | @Retention(RUNTIME) 36 | @Target({METHOD}) 37 | /** 38 | * Use this annotation on a getter method to override the Bean name 39 | * parser for Bean -> JSONObject mapping. A value set to empty string "" 40 | * will have the Bean parser fall back to the default field name processing. 41 | */ 42 | public @interface JSONPropertyName { 43 | /** 44 | * @return The name of the property as to be used in the JSON Object. 45 | */ 46 | String value(); 47 | } 48 | -------------------------------------------------------------------------------- /src/org/loklak/api/iot/FreifunkNodePushServlet.java: -------------------------------------------------------------------------------- 1 | /** 2 | * FreifunkNodePushServlet 3 | * Copyright 16.07.2015 by Dang Hai An, @zyzo 4 | *

5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | *

10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | *

15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | package org.loklak.api.iot; 20 | 21 | import org.json.JSONArray; 22 | import org.json.JSONObject; 23 | import org.loklak.harvester.JsonFieldConverter; 24 | import org.loklak.harvester.JsonValidator; 25 | import org.loklak.objects.SourceType; 26 | 27 | public class FreifunkNodePushServlet extends AbstractPushServlet { 28 | 29 | private static final long serialVersionUID = 563611786137243970L; 30 | 31 | @Override 32 | protected SourceType getSourceType() { 33 | return SourceType.FREIFUNK_NODE; 34 | } 35 | 36 | @Override 37 | protected JsonValidator.JsonSchemaEnum getValidatorSchema() { 38 | return JsonValidator.JsonSchemaEnum.FREIFUNK_NODE; 39 | } 40 | 41 | @Override 42 | protected JsonFieldConverter.JsonConversionSchemaEnum getConversionSchema() { 43 | return JsonFieldConverter.JsonConversionSchemaEnum.FREIFUNK_NODE; 44 | } 45 | 46 | @Override 47 | protected JSONArray extractMessages(JSONObject data) { 48 | return data.getJSONArray("nodes"); 49 | } 50 | 51 | @Override 52 | protected void customProcessing(JSONObject message) {} 53 | } 54 | -------------------------------------------------------------------------------- /src/org/loklak/api/iot/OpenWifiMapPushServlet.java: -------------------------------------------------------------------------------- 1 | /** 2 | * OpenWifiMapPushServlet 3 | * Copyright 16.07.2015 by Dang Hai An, @zyzo 4 | *

5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | *

10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | *

15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.api.iot; 21 | 22 | import org.json.JSONArray; 23 | import org.json.JSONObject; 24 | import org.loklak.harvester.JsonFieldConverter; 25 | import org.loklak.harvester.JsonValidator; 26 | import org.loklak.objects.SourceType; 27 | 28 | public class OpenWifiMapPushServlet extends AbstractPushServlet { 29 | 30 | private static final long serialVersionUID = -5983742254182146642L; 31 | 32 | @Override 33 | protected SourceType getSourceType() { 34 | return SourceType.OPENWIFIMAP; 35 | } 36 | 37 | @Override 38 | protected JsonValidator.JsonSchemaEnum getValidatorSchema() { 39 | return JsonValidator.JsonSchemaEnum.OPENWIFIMAP; 40 | } 41 | 42 | @Override 43 | protected JsonFieldConverter.JsonConversionSchemaEnum getConversionSchema() { 44 | return JsonFieldConverter.JsonConversionSchemaEnum.OPENWIFIMAP; 45 | } 46 | 47 | @Override 48 | protected JSONArray extractMessages(JSONObject data) { 49 | return data.getJSONArray("rows"); 50 | } 51 | 52 | @Override 53 | protected void customProcessing(JSONObject message) {} 54 | } 55 | -------------------------------------------------------------------------------- /src/org/loklak/harvester/HarvestingFrequency.java: -------------------------------------------------------------------------------- 1 | /** 2 | * HarvestingFrequency 3 | * Copyright 01.08.2015 by Dang Hai An, @zyzo 4 | *

5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | *

10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | *

15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.harvester; 21 | 22 | public enum HarvestingFrequency { 23 | THIRTY_MIN(30), // every half-hour 24 | AN_HOUR(60), // every hour 25 | THREE_HOURS(180), // every three hours 26 | SIX_HOURS(360), // every six hours 27 | A_DAY(1440), // every day 28 | NEVER(Integer.MAX_VALUE); // never update 29 | 30 | int frequency; 31 | HarvestingFrequency(int frequency) { 32 | this.frequency = frequency; 33 | } 34 | 35 | /** 36 | * @return int : frequency in minutes 37 | */ 38 | public int getFrequency() { 39 | return frequency; 40 | } 41 | 42 | /** 43 | * @throws IllegalArgumentException when frequency value is not permitted (not declared as an enum) 44 | */ 45 | public static HarvestingFrequency valueOf(int frequency) { 46 | for (HarvestingFrequency f: HarvestingFrequency.values()) { 47 | if (f.getFrequency() == frequency) { 48 | return f; 49 | } 50 | } 51 | throw new IllegalArgumentException(); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/org/loklak/server/Client.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Client 3 | * Copyright 03.06.2016 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.server; 21 | 22 | import org.json.JSONObject; 23 | 24 | public class Client { 25 | 26 | private final static char SEPARATOR = ':'; 27 | 28 | private String id; 29 | private int separatorPos; 30 | 31 | protected Client(String rawIdString) { 32 | this.separatorPos = rawIdString.indexOf(SEPARATOR); 33 | assert this.separatorPos >= 0; 34 | this.id = rawIdString; 35 | } 36 | 37 | protected Client(String typeName, String untypedId) { 38 | this.id = typeName + SEPARATOR + untypedId; 39 | this.separatorPos = typeName.length(); 40 | } 41 | 42 | protected String getKey() { 43 | return id.substring(0, this.separatorPos); 44 | } 45 | 46 | public String getName() { 47 | return this.id.substring(this.separatorPos + 1); 48 | } 49 | 50 | public String toString() { 51 | return this.id; 52 | } 53 | 54 | public JSONObject toJSON() { 55 | JSONObject json = new JSONObject(true); 56 | json.put("type", getKey()); 57 | json.put("name", this.getName()); 58 | return json; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/org/loklak/geo/AbstractGeoPoint.java: -------------------------------------------------------------------------------- 1 | /** 2 | * AbstractGeoPoint 3 | * Copyright 03.06.2015 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.geo; 21 | 22 | public abstract class AbstractGeoPoint implements GeoPoint { 23 | 24 | /** 25 | * compute the distance between two points using the Haversine Algorithm 26 | * https://en.wikipedia.org/wiki/Haversine_formula 27 | * @param othr the other point 28 | * @return the distance of this point and the other point in meter 29 | */ 30 | public double distance(final GeoPoint othr) { 31 | return distance(this.lat(), this.lon(), othr.lat(), othr.lon()); 32 | } 33 | 34 | public static double distance(final double lat1, final double lon1, final double lat2, final double lon2) { 35 | double dlat = (lat2 - lat1) * D2R; double dlon = (lon2 - lon1) * D2R; 36 | double a = Math.pow(Math.sin(dlat / 2.0d), 2.0d) + 37 | Math.cos(lat1 * D2R) * Math.cos(lat2 * D2R) * Math.pow(Math.sin(dlon / 2.0d), 2.0d); 38 | double c = 2.0d * Math.atan2(Math.sqrt(a), Math.sqrt(1.0d - a)); 39 | return EQUATOR_EARTH_RADIUS * c; 40 | } 41 | 42 | static final double EQUATOR_EARTH_RADIUS = 63781370.0d; 43 | static final double D2R = (Math.PI / 180.0d); 44 | 45 | } 46 | -------------------------------------------------------------------------------- /Dockerfile-arm32v6: -------------------------------------------------------------------------------- 1 | FROM arm32v6/openjdk:8-alpine AS builder 2 | LABEL maintainer="Michael Peter Christen " 3 | 4 | # This image is suitable for Raspberry Pi 1, 2, Compute Module 1 and Zero, 5 | # for a RPi 3, 4 and Compute Module 3 please use the arm32v7 image. 6 | # See also: https://en.wikipedia.org/wiki/Raspberry_Pi#Processor 7 | 8 | # create image on a Raspberry Pi with: 9 | # sudo apt-get install openjdk-8-jdk-headless:armhf 10 | # git clone https://github.com/loklak/loklak_server.git 11 | # cd loklak_server 12 | # docker build -t loklak/loklak_server:latest-arm32v6 -f Dockerfile-arm32v6 . 13 | 14 | # start the image with (i.e.) 15 | # docker run -d -p 9000:9000 16 | 17 | # copy the required parts of the source code 18 | ADD src /loklak_server/src/ 19 | ADD gradle /loklak_server/gradle/ 20 | ADD gradlew /loklak_server/ 21 | ADD build.gradle /loklak_server/ 22 | ADD settings.gradle /loklak_server/ 23 | 24 | # compile loklak 25 | RUN cd /loklak_server && ./gradlew build --no-daemon -x checkstyleMain -x checkstyleTest -x jacocoTestReport 26 | 27 | # Second Stage 28 | FROM arm32v6/openjdk:8-jre-alpine 29 | 30 | # install required software 31 | RUN apk update && apk add --no-cache bash 32 | 33 | # Create Volume for persistence 34 | VOLUME ["/loklak_server/data"] 35 | 36 | # loklak start 37 | CMD ["/loklak_server/bin/start.sh", "-Idn"] 38 | 39 | # setup locales 40 | ENV LANG=en_US.UTF-8 41 | 42 | # Expose the web interface ports 43 | EXPOSE 9000 9443 44 | 45 | # copy the required parts of the source code 46 | WORKDIR /loklak_server/ 47 | COPY --from=builder /loklak_server/build/libs/loklak_server-all.jar build/libs/ 48 | ADD bin /loklak_server/bin/ 49 | ADD conf /loklak_server/conf/ 50 | ADD html /loklak_server/html/ 51 | ADD installation /loklak_server/installation/ 52 | ADD ssi /loklak_server/ssi/ 53 | ADD test/queries /loklak_server/test/queries/ 54 | 55 | # change config file 56 | RUN sed -i 's/^\(upgradeInterval=\).*/\186400000000/' /loklak_server/conf/config.properties 57 | 58 | # set current working directory to loklak_server 59 | WORKDIR /loklak_server 60 | -------------------------------------------------------------------------------- /Dockerfile-arm32v7: -------------------------------------------------------------------------------- 1 | FROM arm32v7/openjdk:8-alpine AS builder 2 | LABEL maintainer="Michael Peter Christen " 3 | 4 | # This image is suitable for Raspberry Pi 3, 4 and Compute Module 3, 5 | # for a RPi 1, 2, Compute Module 1 and Zero please use the arm32v6 image. 6 | # See also: https://en.wikipedia.org/wiki/Raspberry_Pi#Processor 7 | 8 | # create image on a Raspberry Pi with: 9 | # sudo apt-get install openjdk-8-jdk-headless:armhf 10 | # git clone https://github.com/loklak/loklak_server.git 11 | # cd loklak_server 12 | # docker build -t loklak/loklak_server:latest-arm32v7 -f Dockerfile-arm32v7 . 13 | 14 | # start the image with (i.e.) 15 | # docker run -d -p 9000:9000 16 | 17 | # copy the required parts of the source code 18 | ADD src /loklak_server/src/ 19 | ADD gradle /loklak_server/gradle/ 20 | ADD gradlew /loklak_server/ 21 | ADD build.gradle /loklak_server/ 22 | ADD settings.gradle /loklak_server/ 23 | 24 | # compile loklak 25 | RUN cd /loklak_server && ./gradlew build --no-daemon -x checkstyleMain -x checkstyleTest -x jacocoTestReport 26 | 27 | # Second Stage 28 | FROM arm32v7/openjdk:8-jre-alpine 29 | 30 | # install required software 31 | RUN apk update && apk add --no-cache bash 32 | 33 | # Create Volume for persistence 34 | VOLUME ["/loklak_server/data"] 35 | 36 | # loklak start 37 | CMD ["/loklak_server/bin/start.sh", "-Idn"] 38 | 39 | # setup locales 40 | ENV LANG=en_US.UTF-8 41 | 42 | # Expose the web interface ports 43 | EXPOSE 9000 9443 44 | 45 | # copy the required parts of the source code 46 | WORKDIR /loklak_server/ 47 | COPY --from=builder /loklak_server/build/libs/loklak_server-all.jar build/libs/ 48 | ADD bin /loklak_server/bin/ 49 | ADD conf /loklak_server/conf/ 50 | ADD html /loklak_server/html/ 51 | ADD installation /loklak_server/installation/ 52 | ADD ssi /loklak_server/ssi/ 53 | ADD test/queries /loklak_server/test/queries/ 54 | 55 | # change config file 56 | RUN sed -i 's/^\(upgradeInterval=\).*/\186400000000/' /loklak_server/conf/config.properties 57 | 58 | # set current working directory to loklak_server 59 | WORKDIR /loklak_server 60 | -------------------------------------------------------------------------------- /test/queries/chinese_00.txt: -------------------------------------------------------------------------------- 1 | ＃中国梦 2 | 开源 3 | 总统 4 | 圣诞 5 | ＃美国 6 | ＃中国 7 | 获得感 8 | 互联网+ 9 | 颜值 10 | 宝宝 11 | 创客 12 | 脑洞大开 13 | 任性 14 | 剁手党 15 | ＃网红 16 | 主要看气质 17 | 北美洲 18 | 伊斯兰教 19 | 英国广播公司 20 | 苏轼 21 | 牟姓 22 | 松树 23 | 穆斯林 24 | 中国古代十大名医 25 | 魏冉 26 | 曲波 27 | 张姓字辈排行 28 | 洛杉矶 29 | 尸体 30 | 绵阳市三台县 31 | 中国古籍全录 32 | 中国十大古书 33 | 荆门市 34 | mc小炜 35 | 成都市 36 | 英语 37 | 大韩民国 38 | 中文百科在线简介 39 | ＃日本 40 | 孙姓 41 | 经典怀旧歌曲 42 | 吸血鬼日记 43 | 藏獒 44 | 活佛 45 | 常德市 46 | 南宁历史 47 | 方婧 48 | 辽阳市 49 | 马永贞 50 | 2010年中国城市人均收入排名 51 | 快乐大本营 52 | 吴奇隆 53 | 木兰辞 54 | 2010年GDP世界排名 55 | kuso 56 | 中华人民共和国学科分类与代码国家标准 57 | 关于大熊猫的知识 58 | 渤海 59 | 青春期 60 | 中国大陆女歌手列表 61 | 人肤蝇 62 | 眠空 63 | 燃料电池 64 | 韩愈 65 | 湖南省 66 | 广东省 67 | 廖姓 68 | 水果释迦 69 | 鬼门关 70 | 南加州大学 71 | 红玉髓 72 | 宁乡 73 | 爱新觉罗·溥仪 74 | 元宁宗 75 | 富贵竹 76 | 叶佳修 77 | 古曼童 78 | 高锰酸钾 79 | 鼻敏感 80 | 彬彬来吃 81 | 洗脑术 82 | 渠江皇家薄片 83 | 安化黑茶起源品牌 84 | 操作系统 85 | 吴姓 86 | 红楼梦 87 | 缅甸联邦 88 | 台湾特有种植物 89 | 翟姓 90 | 谭姓 91 | 古姓 92 | 千山暮雪 93 | 穴位 94 | 海东青 95 | 那一世 96 | 染料 97 | 华胥引 98 | 中国国际广播电台 99 | 谢姓字辈排行 100 | 高长恭 101 | 安以轩 102 | 黑玛瑙 103 | 读者手册 104 | 鞭笞 105 | 冷碗碗 106 | 云南省 107 | 百家姓 108 | 广西壮族自治区 109 | 钟汉良 110 | 古舟子咏 111 | 釜山 112 | 西双版纳傣族自治州 113 | 孙艺珍 114 | 秦昭襄王 115 | 玛丽莲梦露 116 | 火药 117 | 迷羊 118 | 灵石县 119 | 酵母菌 120 | 亚麻籽 121 | 辅警 122 | 皇家湘淳薄片 123 | 杜甫 124 | 唐朝 125 | 萧亚轩 126 | 神马都是浮云 127 | ＃耶稣 128 | 百合花 129 | 贾思勰 130 | 仲夏夜之梦 131 | 沁园春·雪 132 | 欲情课 133 | 李叔同 134 | 送别 135 | 周姓字辈排行 136 | 时有女子 137 | 奥特曼 138 | 白歆惠 139 | 柯蓝 140 | 大麻 141 | 近亲性交 142 | 中华人民共和国 143 | 刘姓字辈排行 144 | 保钓运动 145 | 易经 146 | 唐嫣 147 | 归园田居 148 | 割礼 149 | 一百分妈妈 150 | 柯以柔 151 | 发财树 152 | 罗汉鱼 153 | 符咒制作 154 | 大方臭豆腐 155 | INTJ 156 | 阳泉市 157 | 张一一 158 | 皇家薄片 159 | 美国队长2 160 | 放大镜 161 | 网络硬盘 162 | 湖北省 163 | 新加坡共和国 164 | 陈毅 165 | 不负如来不负卿 166 | 凤飞飞 167 | 济南市 168 | 灯笼花 169 | 兰陵王入阵曲 170 | 李静 171 | 畅销书 172 | 数据 173 | 发达经济体 174 | 富人 175 | 财富 176 | 书 177 | 资本 178 | 回报率 179 | 增速 180 | 问题 181 | 大萧条 182 | 发展中国家 183 | 不平等 184 | 关心 185 | 辩论 186 | 加剧 187 | 镀金时代 188 | 资本论 189 | 日记 190 | 较量 191 | 人人网 192 | 新浪 193 | 微博 194 | 微信 195 | 空间 196 | 脸书 197 | 推特 198 | 知乎 199 | 比特币 200 | 维基 201 | -------------------------------------------------------------------------------- /src/org/loklak/stream/MqttEventSource.java: -------------------------------------------------------------------------------- 1 | package org.loklak.stream; 2 | 3 | import org.eclipse.jetty.servlets.EventSource; 4 | import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken; 5 | import org.eclipse.paho.client.mqttv3.MqttCallback; 6 | import org.eclipse.paho.client.mqttv3.MqttClient; 7 | import org.eclipse.paho.client.mqttv3.MqttException; 8 | import org.eclipse.paho.client.mqttv3.MqttMessage; 9 | import org.loklak.data.DAO; 10 | 11 | import java.io.IOException; 12 | 13 | public class MqttEventSource implements EventSource, MqttCallback { 14 | 15 | private Emitter emitter; 16 | private MqttClient mqttClient; 17 | private String channel; 18 | 19 | MqttEventSource(String channel) { 20 | this.channel = channel; 21 | } 22 | 23 | @Override 24 | public void onOpen(Emitter emitter) throws IOException { 25 | this.emitter = emitter; 26 | try { 27 | this.mqttClient = new MqttClient(DAO.getConfig("stream.mqtt.address", "tcp://127.0.0.1:1883"), "loklak_server_subscriber"); 28 | this.mqttClient.connect(); 29 | this.mqttClient.setCallback(this); 30 | this.mqttClient.subscribe(this.channel); 31 | } catch (MqttException e) { 32 | this.emitter.close(); 33 | } 34 | } 35 | 36 | @Override 37 | public void onClose() { 38 | try { 39 | this.mqttClient.close(); 40 | this.mqttClient.disconnect(); 41 | } catch (MqttException e) { 42 | DAO.severe("Error terminating the stream client for API request", e); 43 | } 44 | } 45 | 46 | @Override 47 | public void connectionLost(Throwable cause) { 48 | DAO.severe("Connection lost for Stream request at channel " + this.channel, cause); 49 | } 50 | 51 | @Override 52 | public void messageArrived(String topic, MqttMessage message) throws Exception { 53 | this.emitter.data(message.toString()); 54 | } 55 | 56 | @Override 57 | public void deliveryComplete(IMqttDeliveryToken token) { 58 | // Cool! nothing to do here. 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /kubernetes/bin/update-deployment-travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | if [ "$TRAVIS_REPO_SLUG" != "loklak/loklak_server" ]; then 6 | echo "Skipping image update for repo $TRAVIS_REPO_SLUG" 7 | exit 0 8 | fi 9 | 10 | if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then 11 | echo "Skipping image update for pull request" 12 | exit 0 13 | fi 14 | 15 | if [ "$TRAVIS_BRANCH" != "master" ] && [ "$TRAVIS_BRANCH" != "development" ]; then 16 | echo "Skipping image update for branch $TRAVIS_BRANCH" 17 | exit 0 18 | fi 19 | 20 | # Temporary, will be removed once we have all the deployments running 21 | if [ "$TRAVIS_BRANCH" == "master" ]; then 22 | echo "No deployments for master branch" 23 | exit 0 24 | fi 25 | 26 | GC_PROJECT_STAGING=loklak-1 27 | GC_PROJECT_API=YET-TO-DEPLOY 28 | 29 | GC_CLUSTER=loklak-cluster 30 | GC_ZONE=us-central1-a 31 | 32 | TAG_STAGING=loklak/loklak_server:kubernetes-staging-$TRAVIS_COMMIT 33 | TAG_API=loklak/loklak_server:kubernetes-api-$TRAVIS_COMMIT 34 | 35 | echo ">>> Decrypting credentials" 36 | openssl aes-256-cbc -K $encrypted_48d01dc243a6_key -iv $encrypted_48d01dc243a6_iv -in kubernetes/gcloud-credentials.json.enc -out kubernetes/gcloud-credentials.json -d 37 | 38 | echo ">>> Installing Google Cloud SDK with Kubernetes" 39 | export CLOUDSDK_CORE_DISABLE_PROMPTS=1 40 | curl https://sdk.cloud.google.com | bash > /dev/null 41 | source ~/google-cloud-sdk/path.bash.inc 42 | gcloud components install kubectl 43 | 44 | echo ">>> Authenticating Google Cloud using decrypted credentials" 45 | gcloud auth activate-service-account --key-file kubernetes/gcloud-credentials.json 46 | 47 | echo ">>> Configuring Google Cloud" 48 | gcloud config set compute/zone $GC_ZONE 49 | export GOOGLE_APPLICATION_CREDENTIALS=kubernetes/gcloud-credentials.json 50 | 51 | 52 | if [ $TRAVIS_BRANCH == "development" ]; then 53 | echo ">>> Updating Kubernetes deployment for staging.loklak.org" 54 | gcloud config set project $GC_PROJECT_STAGING 55 | gcloud container clusters get-credentials $GC_CLUSTER 56 | kubectl set image deployment/server --namespace=web server=$TAG_STAGING 57 | fi 58 | -------------------------------------------------------------------------------- /test/org/loklak/harvester/RedirectUnshortenerTest.java: -------------------------------------------------------------------------------- 1 | package org.loklak.harvester; 2 | 3 | import java.util.LinkedHashMap; 4 | import java.util.Map; 5 | 6 | import org.junit.Test; 7 | import static org.junit.Assert.assertEquals; 8 | import org.loklak.data.DAO; 9 | 10 | import org.loklak.harvester.RedirectUnshortener; 11 | 12 | public class RedirectUnshortenerTest { 13 | 14 | @Test 15 | public void testRedirectUnshortener() { 16 | Map shortlinkMap = new LinkedHashMap<>(); 17 | 18 | shortlinkMap.put("https://goo.gl/r4pNHk", "https://www.youtube.com/watch?v=RRlOCHD-p8Q"); 19 | shortlinkMap.put("http://tinyurl.com/loklak-server", "https://github.com/loklak/loklak_server/wiki"); 20 | shortlinkMap.put("https://t.co/raRRie3ado", "https://medium.com/javascript-scene/angular-2-vs-react-the-ultimate-dance-off-60e7dfbc379c"); 21 | shortlinkMap.put("http://bit.ly/2tvkGa1", "https://medium.com/@keekri17/my-adventures-at-foss-asia-d1a5d462b792"); 22 | shortlinkMap.put("http://ow.ly/P9SX30ddSTI", "https://github.com/loklak/loklak_server/issues/1284"); 23 | shortlinkMap.put("http://bit.do/blog-fossasia", "http://blog.fossasia.org/"); 24 | shortlinkMap.put("http://fb.me/4lcXZsyyO", "https://www.facebook.com/permalink.php?story_fbid=1550262581900846&id=1381813572079082"); 25 | shortlinkMap.put("http://wp.me/sf2B5-shorten", "https://en.blog.wordpress.com/2009/08/14/shorten/"); 26 | shortlinkMap.put("https://is.gd/gyk3VT", "https://github.com/fossasia/"); 27 | shortlinkMap.put("https://is.gd/Lros16", "https://twitter.com/lklknt"); 28 | 29 | try { 30 | for (Map.Entry entry: shortlinkMap.entrySet()) { 31 | String unshortenedURL = RedirectUnshortener.unShorten(entry.getKey()); 32 | DAO.log("expected: " + entry.getValue() + "; computed:" + unshortenedURL); 33 | assertEquals(entry.getValue(), unshortenedURL); 34 | } 35 | } catch(Exception e) { 36 | DAO.log("RedirectUnshortenerTest.testRedirectUnshortener() failed with an Exception"); 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /docs/installation/deploying-development-kubernetes.md: -------------------------------------------------------------------------------- 1 | # Deploying `development` version of loklak on Kubernetes 2 | 3 | ## 1. Background of the Deployment 4 | 5 | ### 1.1. API Server and Elasticsearch 6 | The API server and Elasticsearch would co exist in the `web` namespace. API Server would use `NodeBuilder` to create a Node based Elasticsearch cluster with dump and index at `/loklak_server/data` volume. 7 | 8 | ### 1.2. Persistent Storage for Data Dump and Elasticsearch Index 9 | The data dump and Elasticsearch index would be mounted on external persistent disk so that rolling updates do not wipe out the data. 10 | 11 | ## 2. Steps 12 | 13 | ### 2.1. Create a Kubernetes cluster 14 | 15 | Kubernetes cluster can be created easily using the `gcloud` CLI. More details about this are available [here](https://github.com/loklak/loklak_server/blob/development/docs/installation/installation_google_cloud_kubernetes.md#7-creating-a-container-cluster). 16 | 17 | ### 2.2. Clone the loklak project 18 | 19 | ```bash 20 | git clone https://github.com/loklak/lokklak_server 21 | cd loklak_server 22 | ``` 23 | 24 | Ensure that you are at `development` branch. 25 | 26 | ```bash 27 | git checkout development 28 | ``` 29 | 30 | ### 2.3. Create a Persistent Disk 31 | 32 | ```bash 33 | gcloud compute disks create --size=100GB --zone= data-index-disk 34 | ``` 35 | 36 | ### 2.4. Create Kubernets objects using configuration files 37 | 38 | ```bash 39 | ./kubernetes/bin/deploy-development.sh create 40 | ``` 41 | 42 | ## 3. Modifying deployment 43 | 44 | ### 3.1. Setting a new Docker image 45 | 46 | To update deployment image, we can use the following command - 47 | 48 | ```bash 49 | ./kubernetes/bin/update-development-image.sh # Defaults to loklak/loklak_server:latest-kubernetes-development 50 | ``` 51 | 52 | ### 3.2. Updating configurations 53 | 54 | While updating the configurations, it should be ensured that the `api-service.yml` configuration is not recreated. This would retain previous IP and save us from the trouble of updating DNS. 55 | 56 | ### 3.3. Deleting deployment 57 | 58 | ```bash 59 | ./kubernetes/bin/deploy-development.sh delete 60 | ``` 61 | -------------------------------------------------------------------------------- /.utility/push-docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e # Exit on error 4 | 5 | echo "Welcome to loklak Docker hub push script" 6 | 7 | if [ "$TRAVIS_REPO_SLUG" != "loklak/loklak_server" ]; then 8 | echo "Skipping Docker push for repo $TRAVIS_REPO_SLUG" 9 | exit 0 10 | fi 11 | 12 | if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then 13 | echo "Skipping Docker push for pull request" 14 | exit 0 15 | fi 16 | 17 | if [ "$TRAVIS_BRANCH" != "master" ] && [ "$TRAVIS_BRANCH" != "development" ]; then 18 | echo "Skipping Docker push for branch $TRAVIS_BRANCH" 19 | exit 0 20 | fi 21 | 22 | TAG_BRANCH=loklak/loklak_server:latest-$TRAVIS_BRANCH 23 | TAG_COMMIT=loklak/loklak_server:$TRAVIS_COMMIT 24 | 25 | echo "Logging in to Docker hub" 26 | docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD 27 | 28 | docker tag loklak_server $TAG_COMMIT 29 | docker push $TAG_COMMIT 30 | 31 | docker tag $TAG_COMMIT $TAG_BRANCH 32 | docker push $TAG_BRANCH 33 | 34 | docker tag $TAG_BRANCH loklak_server 35 | 36 | if [ "$TRAVIS_BRANCH" == "development" ]; then 37 | # Build and push staging.loklak.org image 38 | KUBERNETES_BRANCH=loklak/loklak_server:latest-kubernetes-staging 39 | KUBERNETES_COMMIT=loklak/loklak_server:kubernetes-staging-$TRAVIS_COMMIT 40 | docker build -t $KUBERNETES_BRANCH kubernetes/images/staging 41 | docker push $KUBERNETES_BRANCH 42 | docker tag $KUBERNETES_BRANCH $KUBERNETES_COMMIT 43 | docker push $KUBERNETES_COMMIT 44 | elif [ "$TRAVIS_BRANCH" == "master" ]; then 45 | # Build and push api.loklak.org image 46 | KUBERNETES_BRANCH=loklak/loklak_server:latest-kubernetes-api 47 | KUBERNETES_COMMIT=loklak/loklak_server:kubernetes-api-$TRAVIS_COMMIT 48 | docker build -t $KUBERNETES_BRANCH kubernetes/images/api 49 | docker push $KUBERNETES_BRANCH 50 | docker tag $KUBERNETES_BRANCH $KUBERNETES_COMMIT 51 | docker push $KUBERNETES_COMMIT 52 | 53 | # Push latest image 54 | docker tag loklak_server loklak/loklak_server:latest 55 | docker push loklak/loklak_server:latest 56 | docker tag loklak/loklak_server:latest loklak_server 57 | else 58 | echo "Skipping Kubernetes image push for branch $TRAVIS_BRANCH" 59 | fi 60 | -------------------------------------------------------------------------------- /src/org/loklak/api/iot/PushReport.java: -------------------------------------------------------------------------------- 1 | package org.loklak.api.iot; 2 | 3 | import org.loklak.objects.ImportProfileEntry; 4 | 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | public class PushReport { 9 | private int recordCount, newCount, knownCount, errorCount; 10 | private ImportProfileEntry importProfile; 11 | 12 | private List knownMessageIds; 13 | public PushReport() { 14 | recordCount = newCount = knownCount = errorCount = 0; 15 | importProfile = null; 16 | knownMessageIds = new ArrayList<>(); 17 | } 18 | 19 | public int getNewCount() { 20 | return newCount; 21 | } 22 | 23 | public void incrementNewCount() { 24 | this.newCount++; 25 | } 26 | 27 | public int getKnownCount() { 28 | return knownCount; 29 | } 30 | 31 | public void incrementKnownCount(String id) { 32 | knownMessageIds.add(id); 33 | this.knownCount++; 34 | } 35 | 36 | public void incrementErrorCount() { 37 | this.errorCount++; 38 | } 39 | 40 | public int getErrorCount() { 41 | return this.errorCount; 42 | } 43 | 44 | public ImportProfileEntry getImportProfile() { 45 | return importProfile; 46 | } 47 | 48 | public void setImportProfile(ImportProfileEntry importProfile) { 49 | this.importProfile = importProfile; 50 | } 51 | 52 | public List getKnownMessageIds() { 53 | return knownMessageIds; 54 | } 55 | 56 | public int getRecordCount() { 57 | return recordCount; 58 | } 59 | 60 | public void incrementRecordCount() { 61 | this.recordCount++; 62 | } 63 | 64 | public void combine(PushReport that) { 65 | this.recordCount += that.recordCount; 66 | this.newCount += that.newCount; 67 | this.knownCount += that.knownCount; 68 | this.errorCount += that.errorCount; 69 | // prioritize `that` import profile 70 | if (that.importProfile != null) 71 | this.importProfile = that.importProfile; 72 | if (that.knownMessageIds != null) 73 | this.knownMessageIds.addAll(that.knownMessageIds); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/org/loklak/tools/CacheStats.java: -------------------------------------------------------------------------------- 1 | /** 2 | * CacheStats 3 | * Copyright 18.04.2016 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.tools; 21 | 22 | import java.util.concurrent.atomic.AtomicLong; 23 | 24 | import org.json.JSONObject; 25 | 26 | public class CacheStats { 27 | 28 | private AtomicLong update, hit, miss; 29 | 30 | public CacheStats() { 31 | this.update = new AtomicLong(0); 32 | this.hit = new AtomicLong(0); 33 | this.miss = new AtomicLong(0); 34 | } 35 | 36 | public void clear() { 37 | this.update.set(0); 38 | this.hit.set(0); 39 | this.miss.set(0); 40 | } 41 | 42 | public void update() { 43 | this.update.incrementAndGet(); 44 | } 45 | 46 | public void hit() { 47 | this.hit.incrementAndGet(); 48 | } 49 | 50 | public void miss() { 51 | this.miss.incrementAndGet(); 52 | } 53 | 54 | public long getUpdate() { 55 | return this.update.get(); 56 | } 57 | 58 | public long getHit() { 59 | return this.hit.get(); 60 | } 61 | 62 | public long getMiss() { 63 | return this.miss.get(); 64 | } 65 | 66 | public JSONObject getJSON() { 67 | JSONObject json = new JSONObject(true); 68 | json.put("update", getUpdate()); 69 | json.put("hit", getHit()); 70 | json.put("miss", getMiss()); 71 | return json; 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /test/queries/spanish_01.txt: -------------------------------------------------------------------------------- 1 | además 2 | agua 3 | ahora 4 | algo 5 | algunos 6 | ante 7 | antes 8 | año 9 | años 10 | aquí 11 | así 12 | aunque 13 | ayer 14 | bien 15 | cada 16 | casa 17 | casi 18 | caso 19 | ciudad 20 | como 21 | cómo 22 | contra 23 | cosas 24 | cual 25 | cualquier 26 | cuando 27 | cuatro 28 | cuenta 29 | debe 30 | decir 31 | del 32 | dentro 33 | desde 34 | después 35 | día 36 | días 37 | dice 38 | dijo 39 | donde 40 | dos 41 | durante 42 | el 43 | él 44 | ella 45 | ello 46 | ellos 47 | embargo 48 | entonces 49 | entre 50 | era 51 | es 52 | esa 53 | ese 54 | eso 55 | españa 56 | esta 57 | está 58 | estaba 59 | estado 60 | están 61 | estas 62 | este 63 | esto 64 | estos 65 | fin 66 | forma 67 | frente 68 | fue 69 | fueron 70 | general 71 | gobierno 72 | gran 73 | grupo 74 | ha 75 | haber 76 | había 77 | hace 78 | hacer 79 | hacia 80 | han 81 | hasta 82 | hay 83 | he 84 | hecho 85 | historia 86 | hombre 87 | hoy 88 | josé 89 | juan 90 | la 91 | las 92 | le 93 | les 94 | lo 95 | los 96 | luego 97 | lugar 98 | madrid 99 | manera 100 | más 101 | mayor 102 | me 103 | medio 104 | mejor 105 | menos 106 | mi 107 | mientras 108 | millones 109 | misma 110 | mismo 111 | momento 112 | mucho 113 | muchos 114 | mujer 115 | mundo 116 | muy 117 | nacional 118 | nada 119 | ni 120 | no 121 | noche 122 | nos 123 | nuestro 124 | nueva 125 | nuevo 126 | nunca 127 | otra 128 | otras 129 | otro 130 | otros 131 | país 132 | para 133 | parece 134 | parte 135 | partido 136 | pasado 137 | pero 138 | personas 139 | poco 140 | poder 141 | política 142 | por 143 | porque 144 | presidente 145 | primer 146 | primera 147 | puede 148 | pueden 149 | pues 150 | punto 151 | que 152 | qué 153 | quien 154 | se 155 | sea 156 | según 157 | ser 158 | será 159 | si 160 | sí 161 | sido 162 | siempre 163 | sin 164 | sino 165 | sistema 166 | sobre 167 | social 168 | sólo 169 | son 170 | su 171 | sus 172 | tal 173 | también 174 | tan 175 | tanto 176 | te 177 | tener 178 | tenía 179 | tiempo 180 | tiene 181 | tienen 182 | tipo 183 | toda 184 | todas 185 | todo 186 | todos 187 | trabajo 188 | tras 189 | tres 190 | un 191 | una 192 | uno 193 | unos 194 | va 195 | veces 196 | ver 197 | vez 198 | vida 199 | ya 200 | yo 201 | -------------------------------------------------------------------------------- /test/org/loklak/api/search/WordpressCrawlerServiceTest.java: -------------------------------------------------------------------------------- 1 | package org.loklak.api.search; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.IOException; 5 | import org.junit.Test; 6 | import static org.hamcrest.core.Is.is; 7 | import static org.junit.Assert.assertThat; 8 | import static org.junit.Assert.assertTrue; 9 | import static org.junit.Assert.assertEquals; 10 | import org.json.JSONArray; 11 | import org.json.JSONObject; 12 | import org.loklak.api.search.WordpressCrawlerService; 13 | import org.loklak.data.DAO; 14 | import org.loklak.http.ClientConnection; 15 | 16 | /** 17 | * These unit-tests test org.loklak.api.search.WordpressCrawlerService 18 | */ 19 | public class WordpressCrawlerServiceTest { 20 | 21 | @Test 22 | public void apiPathTest() { 23 | WordpressCrawlerService wordpressCrawler = new WordpressCrawlerService(); 24 | assertEquals("/api/wordpresscrawler.json", wordpressCrawler.getAPIPath()); 25 | } 26 | 27 | @Test 28 | public void wordpressCrawlerServiceTest() throws NullPointerException { 29 | BufferedReader br = null; 30 | String url = "http://blog.fossasia.org/author/saptaks/"; 31 | String author = "saptaks"; 32 | WordpressCrawlerService wordpressCrawler = new WordpressCrawlerService(url); 33 | try { 34 | ClientConnection connection = new ClientConnection(url, ""); 35 | //Check Network issue 36 | assertThat(connection.getStatusCode(), is(200)); 37 | 38 | br = wordpressCrawler.getHtml(connection); 39 | } catch (IOException e) { 40 | DAO.log("WordpressCrawlerServiceTest.WordpressCrawlerServiceTest() failed to connect to network. url:" + url); 41 | } 42 | 43 | JSONArray blogs = new JSONArray(); 44 | try { 45 | blogs = wordpressCrawler.crawlWordpress(url, br).toArray(); 46 | } catch (NullPointerException e) { 47 | DAO.log("WordpressCrawlerServiceTest.wordpressCrawlerServiceTest() failed with a NullPointerException"); 48 | } 49 | 50 | for(int i = 0;i < blogs.length(); i++) { 51 | JSONObject blog = (JSONObject)blogs.get(i); 52 | assertTrue(blog.has("blog_url")); 53 | assertTrue(blog.has("title")); 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /kubernetes/bin/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export DIR=kubernetes 3 | 4 | USAGE="USAGE: ./kubernetes/bin/deploy.sh api|staging create|create all|delete|delete all" 5 | 6 | if [[ "$1" = "api" ]] 7 | then 8 | BRANCH=api 9 | elif [[ "$1" = "staging" ]] 10 | then 11 | BRANCH=staging 12 | else 13 | echo "Please input correct arguments" 14 | echo $USAGE 15 | exit 16 | fi 17 | 18 | if [[ "$2" = "create" ]] 19 | then 20 | echo "Starting '$BRANCH' branch deployment" 21 | echo "Creating objects from configurations." 22 | echo "Make sure that persistent disk is already created." 23 | echo "" 24 | echo "Creating Elasticsearch Deployment" 25 | kubectl create -R -f ${DIR}/yamls/${BRANCH}/elasticsearch/ 26 | # For mqtt 27 | kubectl create -R -f ${DIR}/yamls/mosquitto/ 28 | if [[ "$3" = "all" ]]; 29 | then 30 | # Start KubeLego deployment for TLS certificates 31 | kubectl create -R -f ${DIR}/yamls/lego/ 32 | echo "Start nginx deployment, ingress & service" 33 | kubectl create -R -f ${DIR}/yamls/nginx/ 34 | fi 35 | # Create web namespace 36 | kubectl create -R -f ${DIR}/yamls/${BRANCH}/web/ 37 | # Wait for some time to prevent any chance of API-Server to access incomplete deployments 38 | sleep 20 39 | echo "Creating Loklak Server deployment" 40 | # Create API server deployment and service for development branch 41 | kubectl create -R -f ${DIR}/yamls/${BRANCH}/api-server/ 42 | echo "Deployed Loklak Server branch '$BRANCH' on Kubernetes" 43 | echo "Trying to fetch public IP. ~50s." 44 | sleep 50 45 | if [[ "$3" = "all" ]] 46 | then 47 | kubectl get services --namespace=nginx-ingress 48 | else 49 | kubectl get services --namespace=web 50 | fi 51 | elif [[ "$2" = "delete" ]] 52 | then 53 | echo "Clearing the cluster." 54 | if [ "$3" = "all" ]; then 55 | kubectl delete -f ${DIR}/yamls/lego/00-namespace.yml 56 | kubectl delete -f ${DIR}/yamls/nginx/00-namespace.yml 57 | fi 58 | kubectl delete -R -f ${DIR}/yamls/${BRANCH}/ 59 | kubectl delete -f ${DIR}/yamls/mosquitto/00-namespace.yaml 60 | echo "Deleted loklak project from Kubernetes" 61 | else 62 | echo "Please arguments provided!" 63 | echo $USAGE 64 | fi 65 | -------------------------------------------------------------------------------- /conf/elasticsearch/mappings/queries.json: -------------------------------------------------------------------------------- 1 | { 2 | "_all": {"enabled": false}, 3 | "properties":{ 4 | "timestamp":{ 5 | "type":"date", "format":"dateOptionalTime", "include_in_all":"false", "index":"not_analyzed" 6 | }, 7 | "query":{ 8 | "type":"string", "include_in_all":"false" 9 | }, 10 | "query_length":{ 11 | "type":"long", "include_in_all":"false", "index":"not_analyzed" 12 | }, 13 | "since":{ 14 | "type":"date", "format":"dateOptionalTime", "include_in_all":"false", "index":"not_analyzed" 15 | }, 16 | "until":{ 17 | "type":"date", "format":"dateOptionalTime", "include_in_all":"false", "index":"not_analyzed" 18 | }, 19 | "source_type":{ 20 | "type":"string", "include_in_all":"false", "index":"not_analyzed" 21 | }, 22 | "timezoneOffset":{ 23 | "type":"long", "include_in_all":"false", "index":"not_analyzed" 24 | }, 25 | "query_first":{ 26 | "type":"date", "format":"dateOptionalTime", "include_in_all":"false", "index":"not_analyzed" 27 | }, 28 | "query_last":{ 29 | "type":"date", "format":"dateOptionalTime", "include_in_all":"false", "index":"not_analyzed" 30 | }, 31 | "retrieval_last":{ 32 | "type":"date", "format":"dateOptionalTime", "include_in_all":"false", "index":"not_analyzed" 33 | }, 34 | "retrieval_next":{ 35 | "type":"date", "format":"dateOptionalTime", "include_in_all":"false", "index":"not_analyzed" 36 | }, 37 | "expected_next":{ 38 | "type":"date", "format":"dateOptionalTime", "include_in_all":"false", "index":"not_analyzed" 39 | }, 40 | "query_count":{ 41 | "type":"long", "include_in_all":"false", "index":"not_analyzed" 42 | }, 43 | "retrieval_count":{ 44 | "type":"long", "include_in_all":"false", "index":"not_analyzed" 45 | }, 46 | "message_period":{ 47 | "type":"long", "include_in_all":"false", "index":"not_analyzed" 48 | }, 49 | "messages_per_day":{ 50 | "type":"long", "include_in_all":"false", "index":"not_analyzed" 51 | }, 52 | "score_retrieval":{ 53 | "type":"long", "include_in_all":"false", "index":"not_analyzed" 54 | }, 55 | "score_suggest":{ 56 | "type":"long", "include_in_all":"false", "index":"not_analyzed" 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /test/queries/german_00.txt: -------------------------------------------------------------------------------- 1 | der 2 | und 3 | sein 4 | in 5 | zu 6 | haben 7 | ich 8 | werden 9 | sie 10 | von 11 | nicht 12 | mit 13 | es 14 | sich 15 | auch 16 | auf 17 | für 18 | an 19 | so 20 | dass 21 | können 22 | dies 23 | als 24 | ihr 25 | ja 26 | wie 27 | bei 28 | oder 29 | wir 30 | aber 31 | dann 32 | man 33 | da 34 | sein 35 | noch 36 | nach 37 | was 38 | also 39 | aus 40 | all 41 | wenn 42 | nur 43 | müssen 44 | sagen 45 | um 46 | machen 47 | kein 48 | Jahr 49 | du 50 | mein 51 | schon 52 | vor 53 | durch 54 | geben 55 | mehr 56 | andere 57 | viel 58 | kommen 59 | jetzt 60 | sollen 61 | mir 62 | wollen 63 | ganz 64 | mich 65 | immer 66 | gehen 67 | sehr 68 | hier 69 | doch 70 | bis 71 | groß 72 | wieder 73 | Mal 74 | zwei 75 | gut 76 | wissen 77 | neu 78 | sehen 79 | lassen 80 | uns 81 | weil 82 | unter 83 | denn 84 | stehen 85 | jed 86 | Beispiel 87 | Zeit 88 | erste 89 | ihm 90 | ihn 91 | wo 92 | lang 93 | eigentlich 94 | damit 95 | selbst 96 | unser 97 | oben 98 | drei 99 | wenig 100 | Frau 101 | Mensch 102 | deutsch 103 | Kind 104 | etwas 105 | Tag 106 | nun 107 | finden 108 | nichts 109 | bleiben 110 | sondern 111 | klein 112 | zwischen 113 | alt 114 | gegen 115 | liegen 116 | ohne 117 | heute 118 | weit 119 | heißen 120 | denken 121 | eben 122 | erst 123 | natürlich 124 | ob 125 | hoch 126 | beide 127 | Mann 128 | einfach 129 | vielleicht 130 | dort 131 | dabei 132 | einmal 133 | ihnen 134 | welch 135 | nehmen 136 | tun 137 | seit 138 | dürfen 139 | glauben 140 | halten 141 | nennen 142 | Land 143 | letzt 144 | gleich 145 | solch 146 | dazu 147 | mögen 148 | Frage 149 | gar 150 | zeigen 151 | führen 152 | möglich 153 | sprechen 154 | während 155 | Haus 156 | Fall 157 | eigen 158 | bringen 159 | Leute 160 | schön 161 | einige 162 | bereits 163 | Arbeit 164 | leben 165 | fahren 166 | meinen 167 | spät 168 | etwa 169 | wer 170 | Prozent 171 | fragen 172 | gerade 173 | wichtig 174 | zwar 175 | Hand 176 | wirklich 177 | kennen 178 | weitere 179 | genau 180 | jung 181 | gelten 182 | Stadt 183 | Herr 184 | Teil 185 | Problem 186 | Welt 187 | jedoch 188 | stellen 189 | darauf 190 | bisschen 191 | vier 192 | nie 193 | spielen 194 | denen 195 | Recht 196 | arbeiten 197 | brauchen 198 | folgen 199 | lernen 200 | Ende -------------------------------------------------------------------------------- /src/org/loklak/objects/ResultList.java: -------------------------------------------------------------------------------- 1 | /** 2 | * ResultList 3 | * Copyright 12.11.2015 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; wo even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package org.loklak.objects; 21 | 22 | import java.util.ArrayList; 23 | 24 | import org.json.JSONArray; 25 | import org.json.JSONException; 26 | import org.loklak.susi.SusiThought; 27 | 28 | public class ResultList extends ArrayList { 29 | 30 | private static final long serialVersionUID = -982453065951290203L; 31 | 32 | private long hits = -1; 33 | 34 | public ResultList() { 35 | super(); 36 | } 37 | 38 | /** 39 | * set the number of total hits this list has 40 | * @param totalHits 41 | */ 42 | public void setHits(long totalHits) { 43 | this.hits = totalHits; 44 | } 45 | 46 | /** 47 | * get the number of total hits 48 | * @return 49 | */ 50 | public long getHits() { 51 | return this.hits == -1 ? this.size() : this.hits; 52 | } 53 | 54 | public void clear() { 55 | super.clear(); 56 | this.hits = -1; 57 | } 58 | 59 | public SusiThought toSusi() throws JSONException { 60 | SusiThought json = new SusiThought().setHits(this.size()); 61 | JSONArray statuses = new JSONArray(); 62 | for (E t: this) { 63 | if (t instanceof QueryEntry) { 64 | QueryEntry qe = (QueryEntry) t; 65 | statuses.put(qe.toJSON()); 66 | } 67 | } 68 | json.setData(statuses); 69 | return json; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /test/org/loklak/data/ElasticsearchClientTest.java: -------------------------------------------------------------------------------- 1 | package org.loklak.data; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | import junit.framework.TestCase; 9 | 10 | import org.elasticsearch.common.settings.Settings; 11 | import org.json.JSONObject; 12 | import org.json.JSONObjectTest; 13 | import org.junit.After; 14 | import org.junit.Before; 15 | import org.loklak.ir.BulkWriteResult; 16 | import org.loklak.ir.ElasticsearchClient; 17 | import org.loklak.ir.BulkWriteEntry; 18 | import org.loklak.tools.BufferedRandomAccessFile; 19 | 20 | public class ElasticsearchClientTest extends TestCase { 21 | 22 | private File testFile; 23 | 24 | @Before 25 | public void setUp() throws Exception { 26 | this.testFile = BufferedRandomAccessFile.Test.getTestFile(); 27 | } 28 | 29 | @After 30 | public void tearDown() throws Exception { 31 | this.testFile.delete(); 32 | } 33 | 34 | private List testBulk(int from, int to) { 35 | List bulk = new ArrayList<>(); 36 | for (int i = from; i < to; i++) { 37 | JSONObject testJson = JSONObjectTest.testJson(true); 38 | BulkWriteEntry entry = new BulkWriteEntry("id_" + i, "testtype", null, null, testJson.toMap()); 39 | bulk.add(entry); 40 | } 41 | return bulk; 42 | } 43 | 44 | public void test() throws IOException { 45 | Settings.Builder settings = Settings.builder(); 46 | settings.put("path.home", this.testFile.getAbsolutePath()); 47 | settings.put("path.data", this.testFile.getAbsolutePath()); 48 | settings.build(); 49 | ElasticsearchClient client = new ElasticsearchClient(settings); 50 | String indexName = "test"; 51 | List bulk = testBulk(0, 1100); 52 | BulkWriteResult result = client.writeMapBulk(indexName, bulk); 53 | assertTrue(result.getCreated().size() == 1100); 54 | assertTrue(result.getErrors().size() == 0); 55 | bulk = testBulk(1000, 1200); 56 | result = client.writeMapBulk(indexName, bulk); 57 | assertTrue(result.getCreated().size() == 100); 58 | assertTrue(result.getErrors().size() == 0); 59 | client.close(); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /html/js/angular-modules/ngStorage.min.js: -------------------------------------------------------------------------------- 1 | /*! ngstorage 0.3.10 | Copyright (c) 2015 Gias Kay Lee | MIT License */!function(a,b){"use strict";"function"==typeof define&&define.amd?define(["angular"],b):a.hasOwnProperty("angular")?b(a.angular):"object"==typeof exports&&(module.exports=b(require("angular")))}(this,function(a){"use strict";function b(b){return function(){var c="ngStorage-";this.setKeyPrefix=function(a){if("string"!=typeof a)throw new TypeError("[ngStorage] - "+b+"Provider.setKeyPrefix() expects a String.");c=a};var d=a.toJson,e=a.fromJson;this.setSerializer=function(a){if("function"!=typeof a)throw new TypeError("[ngStorage] - "+b+"Provider.setSerializer expects a function.");d=a},this.setDeserializer=function(a){if("function"!=typeof a)throw new TypeError("[ngStorage] - "+b+"Provider.setDeserializer expects a function.");e=a},this.get=function(a){return e(window[b].getItem(c+a))},this.set=function(a,e){return window[b].setItem(c+a,d(e))},this.$get=["$rootScope","$window","$log","$timeout","$document",function(f,g,h,i,j){function k(a){var b;try{b=g[a]}catch(c){b=!1}if(b&&"localStorage"===a){var d="__"+Math.round(1e7*Math.random());try{localStorage.setItem(d,d),localStorage.removeItem(d)}catch(c){b=!1}}return b}var l,m,n=c.length,o=k(b)||(h.warn("This browser does not support Web Storage!"),{setItem:a.noop,getItem:a.noop,removeItem:a.noop}),p={$default:function(b){for(var c in b)a.isDefined(p[c])||(p[c]=a.copy(b[c]));return p.$sync(),p},$reset:function(a){for(var b in p)"$"===b[0]||delete p[b]&&o.removeItem(c+b);return p.$default(a)},$sync:function(){for(var a,b=0,d=o.length;d>b;b++)(a=o.key(b))&&c===a.slice(0,n)&&(p[a.slice(n)]=e(o.getItem(a)))},$apply:function(){var b;if(m=null,!a.equals(p,l)){b=a.copy(l),a.forEach(p,function(e,f){a.isDefined(e)&&"$"!==f[0]&&(o.setItem(c+f,d(e)),delete b[f])});for(var e in b)o.removeItem(c+e);l=a.copy(p)}}};return p.$sync(),l=a.copy(p),f.$watch(function(){m||(m=i(p.$apply,100,!1))}),g.addEventListener&&g.addEventListener("storage",function(b){if(b.key){var d=j[0];d.hasFocus&&d.hasFocus()||c!==b.key.slice(0,n)||(b.newValue?p[b.key.slice(n)]=e(b.newValue):delete p[b.key.slice(n)],l=a.copy(p),f.$apply())}}),g.addEventListener&&g.addEventListener("beforeunload",function(){p.$apply()}),p}]}}return a=a&&a.module?a:window.angular,a.module("ngStorage",[]).provider("$localStorage",b("localStorage")).provider("$sessionStorage",b("sessionStorage"))}); -------------------------------------------------------------------------------- /test/queries/english_03.txt: -------------------------------------------------------------------------------- 1 | Minecraft 2 | strike 3 | beetel 4 | catch 5 | mount 6 | wish 7 | sky 8 | board 9 | joy 10 | winter 11 | sat 12 | written 13 | wild 14 | instrument 15 | kept 16 | glass 17 | grass 18 | cow 19 | job 20 | edge 21 | sign 22 | visit 23 | past 24 | soft 25 | fun 26 | bright 27 | gas 28 | weather 29 | month 30 | million 31 | bear 32 | finish 33 | happy 34 | hope 35 | flower 36 | clothe 37 | strange 38 | gone 39 | jump 40 | baby 41 | eight 42 | village 43 | meet 44 | root 45 | buy 46 | raise 47 | solve 48 | metal 49 | whether 50 | push 51 | seven 52 | paragraph 53 | third 54 | shall 55 | held 56 | hair 57 | describe 58 | cook 59 | floor 60 | either 61 | result 62 | burn 63 | hill 64 | safe 65 | cat 66 | century 67 | consider 68 | type 69 | law 70 | bit 71 | coast 72 | copy 73 | phrase 74 | silent 75 | tall 76 | sand 77 | soil 78 | roll 79 | temperature 80 | finger 81 | industry 82 | value 83 | fight 84 | lie 85 | beat 86 | excite 87 | natural 88 | view 89 | sense 90 | ear 91 | else 92 | quite 93 | broke 94 | case 95 | middle 96 | kill 97 | son 98 | lake 99 | moment 100 | scale 101 | loud 102 | spring 103 | observe 104 | child 105 | straight 106 | consonant 107 | nation 108 | dictionary 109 | milk 110 | speed 111 | method 112 | organ 113 | pay 114 | age 115 | section 116 | dress 117 | cloud 118 | surprise 119 | quiet 120 | stone 121 | tiny 122 | climb 123 | cool 124 | design 125 | poor 126 | lot 127 | experiment 128 | bottom 129 | key 130 | iron 131 | single 132 | stick 133 | flat 134 | twenty 135 | skin 136 | smile 137 | crease 138 | hole 139 | trade 140 | melody 141 | trip 142 | office 143 | receive 144 | row 145 | mouth 146 | exact 147 | symbol 148 | die 149 | least 150 | trouble 151 | shout 152 | except 153 | wrote 154 | seed 155 | tone 156 | join 157 | suggest 158 | clean 159 | break 160 | lady 161 | yard 162 | rise 163 | bad 164 | blow 165 | oil 166 | blood 167 | touch 168 | grew 169 | cent 170 | mix 171 | team 172 | wire 173 | cost 174 | lost 175 | brown 176 | wear 177 | garden 178 | equal 179 | sent 180 | choose 181 | fell 182 | fit 183 | flow 184 | fair 185 | bank 186 | collect 187 | save 188 | control 189 | decimal 190 | gentle 191 | woman 192 | captain 193 | practice 194 | separate 195 | difficult 196 | doctor 197 | please 198 | protect 199 | noon 200 | hand 201 | -------------------------------------------------------------------------------- /html/js/angular-modules/angular-momentjs.min.js: -------------------------------------------------------------------------------- 1 | /* 2 | angular-momentjs - v0.2.2 3 | 2015-07-10 4 | */ 5 | 6 | !function(a,b){"use strict";b.module("angular-moment",["gdi2290.moment"]),b.module("angular-momentjs",["gdi2290.moment"]),b.module("ngMoment",["gdi2290.moment"]),b.module("ngMomentjs",["gdi2290.moment"]),b.module("ngMomentJS",["gdi2290.moment"]),b.module("gdi2290.amTimeAgo",[]),b.module("gdi2290.amDateFormat",[]),b.module("gdi2290.moment-service",[]),b.module("gdi2290.moment",["gdi2290.moment-service","gdi2290.amDateFormat","gdi2290.amTimeAgo"]),b.module("gdi2290.amDateFormat").filter("amDateFormat",["$moment",function(a){return function(b,c){return"undefined"==typeof b||null===b?"":(!isNaN(parseFloat(b))&&isFinite(b)&&(b=new Date(parseInt(b,10))),a.then?void a().then(function(a){return a(b).format(c)}):a(b).format(c))}}]),b.module("gdi2290.amTimeAgo").directive("amTimeAgo",["$moment","$timeout",function(a,c){function d(a){return"undefined"==typeof a||null===a||""===a}return function(e,f,g){function h(){m&&(c.cancel(m),m=null)}function i(b){f.text(b.fromNow());var d;a.then?a().then(function(a){d=a().diff(b,"minute")}):d=a().diff(b,"minute");var e=3600;1>d?e=1:60>d?e=30:180>d&&(e=300),m=c(function(){i(b)},1e3*e,!1)}function j(){h(),a().then?a.then(function(a){i(a(k,l))}):i(a(k,l))}var k,l,m=null;e.$watch(g.amTimeAgo,function(a){return d(a)?(h(),void(k&&(f.text(""),k=null))):(b.isNumber(a)&&(a=new Date(a)),k=a,void j())}),g.$observe("amFormat",function(a){l=a,k&&j()}),e.$on("$destroy",function(){h()})}}]),b.module("gdi2290.moment-service").provider("$moment",function(){function a(a){if(document){var b=document.createElement("script");b.type="text/javascript",b.async=!0,b.src=d,b.onreadystatechange=function(){"complete"===this.readyState&&a()},b.onload=a;var c=document.getElementsByTagName("head")[0];c.appendChild(b)}}var b,c=!1,d="//cdnjs.cloudflare.com/ajax/libs/moment.js/2.10.3/moment.js",e={};this.asyncLoading=function(a){return c=a||c,this},this.scriptUrl=function(a){return d=a||d,this},this.locale=function(a,c){return b=a||b,e=c||e,this},this.$get=["$timeout","$q","$window",function(d,f,g){var h=f.defer(),i=g.moment;if(c){var j=function(){d(function(){b&&(g.moment.lang?g.moment.lang(b,e):g.moment.locale(b,e)),h.resolve(g.moment)})};a(j)}else b&&(g.moment.lang?g.moment.lang(b,e):g.moment.locale(b,e));return c?h.promise:i}]})}(this,this.angular,void 0); 7 | //# sourceMappingURL=angular-momentjs.min.js.map -------------------------------------------------------------------------------- /conf/schema/freifunk-node.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-04/schema#", 3 | "description": "freifunk node schema", 4 | "type": "object", 5 | "properties": { 6 | "meta": { 7 | "type": "object" 8 | }, 9 | "nodes": { 10 | "type": "array", 11 | "minItems": 1, 12 | "uniqueItems": false, 13 | "items": { 14 | "type": "object", 15 | "properties": { 16 | "id": { 17 | "type": "string" 18 | }, 19 | "name": { 20 | "type": "string" 21 | }, 22 | "macs": { 23 | "type": "string" 24 | }, 25 | "uptime": { 26 | "type": "number" 27 | }, 28 | "model": { 29 | "type": "string" 30 | }, 31 | "firmware": { 32 | "type": "string" 33 | }, 34 | "geo": { 35 | "type": "array", 36 | "minItems": 2, 37 | "maxItems": 2 38 | }, 39 | "clientcount": { 40 | "type": "integer" 41 | }, 42 | "flags": { 43 | "type": "object", 44 | "properties": { 45 | "online": { 46 | "type": "boolean" 47 | }, 48 | "client": { 49 | "type": "boolean" 50 | }, 51 | "gateway": { 52 | "type": "boolean" 53 | }, 54 | "legacy": { 55 | "type": "boolean" 56 | } 57 | } 58 | } 59 | } 60 | } 61 | }, 62 | "links": { 63 | "type": "object", 64 | "properties": { 65 | "people": { 66 | "type": "array", 67 | "minItems": 1, 68 | "uniqueItems": false, 69 | "items": { 70 | "type": "object", 71 | "properties": { 72 | "id": { 73 | "type": "integer" 74 | }, 75 | "source": { 76 | "type": "integer" 77 | }, 78 | "quality": { 79 | "type": "string" 80 | }, 81 | "type": { 82 | "type": "string" 83 | }, 84 | "target": { 85 | "type": "integer" 86 | } 87 | } 88 | } 89 | } 90 | } 91 | } 92 | } 93 | } -------------------------------------------------------------------------------- /html/settings.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Loklak Server Configuration 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 |

37 | 38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 | 51 |
52 |
53 |
54 | 55 | 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /docs/installation/tutorials.md: -------------------------------------------------------------------------------- 1 | # Tutorials for loklak 2 | 3 | Installation of loklak is very easy. Just watch a video tutorial! 4 | 5 | - [loklak on linux](#learn-how-to-install-loklak-on-linux) 6 | - [loklak on ubuntu](#set-up-loklak-on-ubuntu) 7 | - [loklak on arch linux](#how-to-install-loklak-on-arch-linux) 8 | - [loklak on windows](#how-to-build-and-run-loklak-server-on-windows) 9 | - [loklak with docker on google cloud](#how-to-install-loklak-server-on-google-cloud-using-docker) 10 | - [loklak with docker on bluemix](#loklak-setup-guide-on-bluemix-with-docker) 11 | - [loklak on bluemix](#how-to-run-loklak-server-on-ibm-bluemix-cloud-platform) 12 | - [loklak on OS X](#install-loklak-server-on-os-x) 13 | 14 | ### Learn how to install loklak on linux 15 | 16 | [![asciicast](https://asciinema.org/a/5c3c0x9n9uutt8pyoynheppov.png)](https://asciinema.org/a/5c3c0x9n9uutt8pyoynheppov) 17 | 18 | ### Set up loklak on Ubuntu 19 | 20 | [![asciicast](https://asciinema.org/a/34229.png)](https://asciinema.org/a/34229) 21 | 22 | ### How to Install loklak on Arch Linux 23 | 24 | [![asciicast](https://asciinema.org/a/34119.png)](https://asciinema.org/a/34119) 25 | 26 | ### How to build and run loklak server on Windows 27 | 28 | [![How to build and run loklak server on Windows](http://img.youtube.com/vi/G1OZGcCQwYM/0.jpg)](http://www.youtube.com/watch?v=G1OZGcCQwYM "How to build and run loklak server on Windows") 29 | 30 | ### How to install Loklak server on Google Cloud using Docker 31 | 32 | [![How to install Loklak server on Google Cloud using Docker](http://img.youtube.com/vi/nwX5AcynSTA/0.jpg)](http://www.youtube.com/watch?v=nwX5AcynSTA "How to install Loklak server on Google Cloud using Docker") 33 | 34 | ### Loklak setup guide on Bluemix with Docker 35 | 36 | [![Loklak setup guide on Bluemix with Docker](http://img.youtube.com/vi/fOgY6eoZBQ0/0.jpg)](http://www.youtube.com/watch?v=fOgY6eoZBQ0 "Loklak setup guide on Bluemix with Docker") 37 | 38 | ### How to run loklak server on IBM Bluemix cloud platform 39 | 40 | [![How to run loklak server on IBM Bluemix cloud platform](http://img.youtube.com/vi/DgRY61HrqWc/0.jpg)](http://www.youtube.com/watch?v=DgRY61HrqWc "How to run loklak server on IBM Bluemix cloud platform") 41 | 42 | ### Install Loklak Server on OS X 43 | 44 | [![Install Loklak Server on OS X](http://img.youtube.com/vi/yhVzVh-N120/0.jpg)](http://www.youtube.com/watch?v=yhVzVh-N120 "Install Loklak Server on OS X") 45 | 46 | 47 | -------------------------------------------------------------------------------- /src/org/loklak/api/cms/SettingsManagementService.java: -------------------------------------------------------------------------------- 1 | package org.loklak.api.cms; 2 | 3 | import org.json.JSONObject; 4 | import org.loklak.data.DAO; 5 | import org.loklak.server.APIHandler; 6 | import org.loklak.server.AbstractAPIHandler; 7 | import org.loklak.server.BaseUserRole; 8 | import org.loklak.server.Query; 9 | import org.loklak.server.Authorization; 10 | import org.loklak.server.APIException; 11 | import org.loklak.tools.storage.JSONObjectWithDefault; 12 | 13 | import javax.servlet.http.HttpServletResponse; 14 | import java.util.HashMap; 15 | import java.util.Map; 16 | 17 | public class SettingsManagementService extends AbstractAPIHandler implements APIHandler { 18 | 19 | /** 20 | * 21 | */ 22 | private static final long serialVersionUID = 2803794331847463124L; 23 | 24 | @Override 25 | public String getAPIPath() { 26 | return "/api/settings-management.json"; 27 | } 28 | 29 | @Override 30 | public BaseUserRole getMinimalBaseUserRole() { 31 | return BaseUserRole.ADMIN; 32 | } 33 | 34 | @Override 35 | public JSONObject getDefaultPermissions(BaseUserRole baseUserRole) { 36 | JSONObject result = new JSONObject(); 37 | 38 | boolean answer = false; 39 | 40 | // Only ADMINs are allowed to modify the settings 41 | if (baseUserRole == BaseUserRole.ADMIN) 42 | answer = true; 43 | 44 | result.put("allow_modifications_of_settings", answer); 45 | 46 | return result; 47 | } 48 | 49 | @Override 50 | public JSONObject serviceImpl(Query call, HttpServletResponse response, Authorization rights, 51 | JSONObjectWithDefault permissions) throws APIException { 52 | JSONObject result = new JSONObject(); 53 | 54 | if (!DAO.getConfigKeys().containsAll(call.getKeys())) { 55 | throw new APIException(503, "Invalid configuration key"); 56 | } 57 | 58 | Map appliedSettings = new HashMap<>(); 59 | 60 | // Apply settings 61 | for (String key : call.getKeys()) { 62 | String value = call.get(key, ""); 63 | //DAO.setConfig(key, value); 64 | appliedSettings.put(key, value); 65 | } 66 | 67 | result.put("settings", appliedSettings); 68 | 69 | result.put("message", "New settings are successfully applied"); 70 | 71 | return result; 72 | } 73 | } 74 | --------------------------------------------------------------------------------