├── .github └── FUNDING.yml ├── conf └── config.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── bin ├── stop_rabbitmq.sh ├── clear_parser_queues.sh ├── index_size.sh ├── clear_indexer_queues.sh ├── clear_all_queues.sh ├── install_hq.sh ├── distribute_config.sh ├── prepare_index.sh ├── delete_index.sh ├── stop_elasticsearch.py ├── stop.sh ├── update_all.sh ├── clear_crawler_queues.sh ├── stop_all.py ├── start_rabbitmq.py ├── start_elasticsearch.py ├── clear_loader_queues.sh └── run_all.py ├── .idea ├── vcs.xml ├── modules.xml └── yacy_grid_mcp.iml ├── .gitignore ├── .pydevproject ├── .settings ├── org.eclipse.buildship.core.prefs └── org.eclipse.jdt.core.prefs ├── src └── main │ ├── resources │ └── log4j.properties │ └── java │ ├── org │ └── json │ │ ├── JSONString.java │ │ ├── JSONException.java │ │ ├── JSONPointerException.java │ │ ├── JSONPropertyIgnore.java │ │ ├── JSONPropertyName.java │ │ └── XMLParserConfiguration.java │ └── net │ └── yacy │ └── grid │ ├── json │ ├── JsonFactory.java │ ├── JsonObjectWithDefault.java │ ├── JsonList.java │ ├── JsonRandomAccessFile.java │ ├── JsonLDNode.java │ └── JsonTray.java │ ├── http │ ├── APIException.java │ ├── APIHandler.java │ ├── Log.java │ ├── ServiceResponse.java │ ├── ObjectAPIHandler.java │ └── AbstractAPIHandler.java │ ├── geo │ ├── LocationSource.java │ ├── AbstractGeoPoint.java │ ├── GeoPoint.java │ └── IntegerGeoPoint.java │ ├── tools │ ├── GitTool.java │ ├── SpaceExceededException.java │ ├── MapTools.java │ ├── ClassProvider.java │ ├── CommonPattern.java │ ├── IO.java │ ├── Compression.java │ ├── CacheMap.java │ ├── HashARC.java │ ├── ARC.java │ ├── ComparableARC.java │ ├── KeyList.java │ ├── TimeoutMatcher.java │ ├── MapUtil.java │ ├── Memory.java │ ├── Swagger.java │ ├── AnchorURL.java │ └── Punycode.java │ ├── core │ ├── api │ │ ├── info │ │ │ ├── LogService.java │ │ │ └── StatusService.java │ │ └── vis │ │ │ └── MapService.java │ └── Core.java │ └── graphics │ ├── AnimationPlotter.java │ ├── CircleTool.java │ └── PrintTool.java ├── .project ├── .classpath ├── .travis.yml ├── README.md ├── gradlew.bat └── gradlew /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: orbiter 2 | patreon: 0rb1t3r 3 | -------------------------------------------------------------------------------- /conf/config.properties: -------------------------------------------------------------------------------- 1 | # YaCy Grid Peer configuration 2 | 3 | # the default port of this peer 4 | port = 8010 5 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yacy/yacy_grid_core/master/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /bin/stop_rabbitmq.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | ps aux | grep rabbitmq | grep -v 'grep' |awk '{print $2}' | cut -d/ -f 1 | xargs kill 3 | -------------------------------------------------------------------------------- /bin/clear_parser_queues.sh: -------------------------------------------------------------------------------- 1 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=parser&queueName=yacyparser_00' 2 | -------------------------------------------------------------------------------- /bin/index_size.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | cd "`dirname $0`" 3 | 4 | curl http://elastic:changeme@localhost:9200/web/_count 5 | echo 6 | -------------------------------------------------------------------------------- /bin/clear_indexer_queues.sh: -------------------------------------------------------------------------------- 1 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=indexer&queueName=elasticsearch_00' 2 | -------------------------------------------------------------------------------- /bin/clear_all_queues.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | cd "`dirname $0`" 3 | 4 | ./clear_crawler_queues.sh & 5 | ./clear_parser_queues.sh & 6 | ./clear_indexer_queues.sh & 7 | ./clear_loader_queues.sh 8 | 9 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | classes/ 2 | target/ 3 | data/ 4 | /class/ 5 | /.gradle/ 6 | /build/ 7 | .DS_Store 8 | .settings 9 | .idea/ 10 | yacy_grid_mcp.eml 11 | yacy_grid_mcp.iml 12 | bin/ai/ 13 | bin/log4j.properties 14 | bin/net/ 15 | bin/org/ 16 | 17 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /bin/install_hq.sh: -------------------------------------------------------------------------------- 1 | cd ../data/apps/ 2 | wget https://github.com/ElasticHQ/elasticsearch-HQ/archive/v3.5.0.tar.gz 3 | tar xfz v3.5.0.tar.gz 4 | cd elasticsearch-HQ-3.5.0/ 5 | pip3 install -r requirements.txt 6 | pip3 install flask_socketio 7 | pip3 install apscheduler 8 | pip3 install eventlet 9 | python3 application.py 10 | -------------------------------------------------------------------------------- /.pydevproject: -------------------------------------------------------------------------------- 1 | 2 | 3 | Default 4 | python 2.7 5 | 6 | -------------------------------------------------------------------------------- /bin/distribute_config.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | cd "`dirname $0`" 3 | cd .. 4 | cp data/mcp-8100/conf/config.properties ../yacy_grid_crawler/data/crawler-8300/conf/ 5 | cp data/mcp-8100/conf/config.properties ../yacy_grid_loader/data/loader-8200/conf/ 6 | cp data/mcp-8100/conf/config.properties ../yacy_grid_parser/data/parser-8500/conf/ 7 | -------------------------------------------------------------------------------- /bin/prepare_index.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | cd "`dirname $0`" 3 | cd ../conf/mappings 4 | curl -XPUT http://elastic:changeme@localhost:9200/web -H 'Content-Type: application/json' --data-binary "@web.json" 5 | curl -XPUT http://elastic:changeme@localhost:9200/crawler -H 'Content-Type: application/json' --data-binary "@crawler.json" 6 | echo 7 | -------------------------------------------------------------------------------- /.settings/org.eclipse.buildship.core.prefs: -------------------------------------------------------------------------------- 1 | build.commands=org.eclipse.jdt.core.javabuilder 2 | connection.arguments= 3 | connection.java.home=null 4 | connection.jvm.arguments= 5 | connection.project.dir= 6 | derived.resources=.gradle,build 7 | eclipse.preferences.version=1 8 | natures=org.eclipse.jdt.core.javanature 9 | project.path=\: 10 | -------------------------------------------------------------------------------- /bin/delete_index.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | cd "`dirname $0`" 3 | 4 | # the following command is not good, because afterwards the index must be prepared with the mapping again 5 | #curl -XDELETE 'http://elastic:changeme@localhost:9200/web' 6 | 7 | curl -XPOST 'http://elastic:changeme@localhost:9200/web/_delete_by_query' -d '{ 8 | "query" : { 9 | "match_all" : {} 10 | } 11 | }' 12 | 13 | echo 14 | -------------------------------------------------------------------------------- /bin/stop_elasticsearch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | 3 | import subprocess 4 | 5 | p = subprocess.Popen("ps aux | grep 'org.elasticsearch.bootstrap.Elasticsearch' | grep -v grep | awk '{print $2}' | cut -d/ -f 1", shell=True, stdout=subprocess.PIPE) 6 | pid = p.stdout.read() 7 | 8 | if pid: 9 | print('killing elasticsearch, pid = ' + pid) 10 | subprocess.call('kill ' + pid, shell=True) 11 | else: 12 | print('elasticsearch was not running') 13 | -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Set root logger level to DEBUG and its only appender to A1. 2 | log4j.rootLogger=A1 3 | 4 | # A1 is set to be a ConsoleAppender. 5 | log4j.appender.A1=org.apache.log4j.ConsoleAppender 6 | 7 | # A1 uses PatternLayout. 8 | log4j.appender.A1.layout=org.apache.log4j.PatternLayout 9 | log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n 10 | 11 | log4j.logger.org.eclipse.jetty = INFO 12 | log4j.logger.org.apache.http = INFO -------------------------------------------------------------------------------- /.idea/yacy_grid_mcp.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /bin/stop.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | cd "`dirname $0`" 3 | cd ../data 4 | KILLFILE="mcp-8100.kill" 5 | PIDFILE="mcp-8100.pid" 6 | 7 | # first method to terminate the process 8 | if [ -f "$KILLFILE" ]; 9 | then 10 | rm $KILLFILE 11 | echo "termination requested, waiting.." 12 | # this can take 10 seconds.. 13 | sleep 10 14 | fi 15 | 16 | # second method to terminate the process 17 | if [ -f "$PIDFILE" ]; 18 | then 19 | fuser -k $PIDFILE 20 | fi 21 | 22 | # check if file does not exist any more which would be a sign that this has terminated 23 | if [ ! -f "$PIDFILE" ]; 24 | then 25 | echo "process terminated" 26 | fi 27 | 28 | -------------------------------------------------------------------------------- /.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | # 2 | #Tue Jan 10 22:46:12 CET 2017 3 | org.eclipse.jdt.core.compiler.debug.localVariable=generate 4 | org.eclipse.jdt.core.compiler.compliance=1.8 5 | org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve 6 | org.eclipse.jdt.core.compiler.debug.sourceFile=generate 7 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 8 | org.eclipse.jdt.core.compiler.problem.enumIdentifier=error 9 | org.eclipse.jdt.core.compiler.debug.lineNumber=generate 10 | eclipse.preferences.version=1 11 | org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled 12 | org.eclipse.jdt.core.compiler.source=1.8 13 | org.eclipse.jdt.core.compiler.problem.assertIdentifier=error 14 | -------------------------------------------------------------------------------- /bin/update_all.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | cd "`dirname $0`" 3 | 4 | cd .. 5 | echo updating `pwd` 6 | git pull origin master 7 | ./gradlew assemble 8 | 9 | cd ../yacy_grid_crawler 10 | echo updating `pwd` 11 | git submodule foreach git pull origin master 12 | git pull origin master 13 | ./gradlew assemble 14 | 15 | cd ../yacy_grid_loader 16 | echo updating `pwd` 17 | git submodule foreach git pull origin master 18 | git pull origin master 19 | ./gradlew assemble 20 | 21 | cd ../yacy_grid_parser 22 | echo updating `pwd` 23 | git submodule foreach git pull origin master 24 | git pull origin master 25 | ./gradlew assemble 26 | 27 | cd ../yacy_grid_search 28 | echo updating `pwd` 29 | git submodule foreach git pull origin master 30 | git pull origin master 31 | ./gradlew assemble 32 | -------------------------------------------------------------------------------- /src/main/java/org/json/JSONString.java: -------------------------------------------------------------------------------- 1 | package org.json; 2 | /** 3 | * The JSONString interface allows a toJSONString() 4 | * method so that a class can change the behavior of 5 | * JSONObject.toString(), JSONArray.toString(), 6 | * and JSONWriter.value(Object). The 7 | * toJSONString method will be used instead of the default behavior 8 | * of using the Object's toString() method and quoting the result. 9 | */ 10 | public interface JSONString { 11 | /** 12 | * The toJSONString method allows a class to produce its own JSON 13 | * serialization. 14 | * 15 | * @return A strictly syntactically correct JSON text. 16 | */ 17 | public String toJSONString(); 18 | } -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | yacy_grid_core 4 | YaCy Grid Core 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.buildship.core.gradleprojectbuilder 15 | 16 | 17 | 18 | 19 | org.python.pydev.PyDevBuilder 20 | 21 | 22 | 23 | 24 | 25 | org.eclipse.jdt.core.javanature 26 | org.eclipse.buildship.core.gradleprojectnature 27 | org.python.pydev.pythonNature 28 | 29 | 30 | -------------------------------------------------------------------------------- /.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /bin/clear_crawler_queues.sh: -------------------------------------------------------------------------------- 1 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=crawler&queueName=webcrawler_00' 2 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=crawler&queueName=webcrawler_01' 3 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=crawler&queueName=webcrawler_02' 4 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=crawler&queueName=webcrawler_03' 5 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=crawler&queueName=webcrawler_04' 6 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=crawler&queueName=webcrawler_05' 7 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=crawler&queueName=webcrawler_06' 8 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=crawler&queueName=webcrawler_07' 9 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | # Use node_js environnement 3 | language: java 4 | 5 | # Cache Gcloud SDK between commands 6 | 7 | cache: 8 | directories: 9 | - "$HOME/google-cloud-sdk/" 10 | 11 | # Install services 12 | services: 13 | - docker 14 | 15 | # Set env vars 16 | env: 17 | global: 18 | - GOOGLE_APPLICATION_CREDENTIALS=~/gcloud-service-key.json 19 | - PROJECT_NAME_STG=yacygrid 20 | - CLUSTER_NAME_STG=cluster-1 21 | - CLOUDSDK_COMPUTE_ZONE=us-central1-a 22 | - DOCKER_IMAGE_NAME=yacygridmcp 23 | - KUBE_DEPLOYMENT_NAME=yacygridmcp 24 | - KUBE_DEPLOYMENT_CONTAINER_NAME=yacygridmcp 25 | - NODE_ENV=CI 26 | 27 | 28 | script: 29 | - if [ ! -d "$HOME/google-cloud-sdk/bin" ]; then rm -rf $HOME/google-cloud-sdk; export CLOUDSDK_CORE_DISABLE_PROMPTS=1; curl https://sdk.cloud.google.com | bash; fi 30 | - source /home/travis/google-cloud-sdk/path.bash.inc 31 | - gcloud --quiet version 32 | - gcloud --quiet components update 33 | - gcloud --quiet components update kubectl 34 | 35 | branches: 36 | only: 37 | - master 38 | - development 39 | -------------------------------------------------------------------------------- /bin/stop_all.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | 3 | import os 4 | import socket 5 | import subprocess 6 | import signal 7 | 8 | path_apphome = os.path.dirname(os.path.abspath(__file__)) + '/..' 9 | os.chdir(path_apphome) 10 | # os.system('ls') 11 | 12 | def checkportopen(port): 13 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 14 | return sock.connect_ex(('127.0.0.1', port)) == 0 15 | 16 | 17 | if checkportopen(9200): 18 | print "killing elasticsearch" 19 | pid = subprocess.check_output(['fuser', path_apphome + '/data/mcp-8100/apps/elasticsearch/bin/nohup.out']) 20 | print 'pid is ', pid 21 | os.kill(int(pid), signal.SIGTERM) 22 | 23 | 24 | if checkportopen(2121): 25 | print "ftp server is running" 26 | 27 | if checkportopen(15672): 28 | print "rabbitmq is running" 29 | 30 | if checkportopen(8100): 31 | print "yacy_grid_mcp is running" 32 | 33 | if checkportopen(8200): 34 | print "yacy_grid_loader is running" 35 | 36 | if checkportopen(8300): 37 | print "yacy_grid_crawler is running" 38 | 39 | if checkportopen(8500): 40 | print "yacy_grid_parser is running" 41 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/json/JsonFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * JsonFactory 3 | * Copyright 02.10.2015 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.json; 21 | 22 | import java.io.IOException; 23 | 24 | import org.json.JSONObject; 25 | 26 | public interface JsonFactory { 27 | 28 | public JSONObject getJSON() throws IOException; 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/json/JSONException.java: -------------------------------------------------------------------------------- 1 | package org.json; 2 | 3 | /** 4 | * The JSONException is thrown by the JSON.org classes when things are amiss. 5 | * 6 | * @author JSON.org 7 | * @version 2015-12-09 8 | */ 9 | public class JSONException extends RuntimeException { 10 | /** Serialization ID */ 11 | private static final long serialVersionUID = 0; 12 | 13 | /** 14 | * Constructs a JSONException with an explanatory message. 15 | * 16 | * @param message 17 | * Detail about the reason for the exception. 18 | */ 19 | public JSONException(final String message) { 20 | super(message); 21 | } 22 | 23 | /** 24 | * Constructs a JSONException with an explanatory message and cause. 25 | * 26 | * @param message 27 | * Detail about the reason for the exception. 28 | * @param cause 29 | * The cause. 30 | */ 31 | public JSONException(final String message, final Throwable cause) { 32 | super(message, cause); 33 | } 34 | 35 | /** 36 | * Constructs a new JSONException with the specified cause. 37 | * 38 | * @param cause 39 | * The cause. 40 | */ 41 | public JSONException(final Throwable cause) { 42 | super(cause.getMessage(), cause); 43 | } 44 | 45 | } -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/http/APIException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * APIException 3 | * Copyright 19.05.2016 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.http; 21 | 22 | public class APIException extends Exception { 23 | 24 | private static final long serialVersionUID = -6974553774866005875L; 25 | private int statusCode = -1; 26 | 27 | public APIException(int statusCode, String message) { 28 | super(message); 29 | this.statusCode = statusCode; 30 | } 31 | 32 | public int getStatusCode() { 33 | return this.statusCode; 34 | } 35 | } -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/geo/LocationSource.java: -------------------------------------------------------------------------------- 1 | /** 2 | * LocationSource 3 | * Copyright 10.06.2015 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.geo; 21 | 22 | public enum LocationSource { 23 | 24 | USER, // the (loklak) user has set the location, this is a hint that this is a rich tweet. 25 | REPORT, // location came from another source in identical way. This may be a IoT import. 26 | PLACE, // location came from translation of the given place name, which is in the context of messages an invisible meta-information. 27 | ANNOTATION; // location was detected and annotated from visible text content 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/org/json/JSONPointerException.java: -------------------------------------------------------------------------------- 1 | package org.json; 2 | 3 | /* 4 | Copyright (c) 2002 JSON.org 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | The Software shall be used for Good, not Evil. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | SOFTWARE. 25 | */ 26 | 27 | /** 28 | * The JSONPointerException is thrown by {@link JSONPointer} if an error occurs 29 | * during evaluating a pointer. 30 | * 31 | * @author JSON.org 32 | * @version 2016-05-13 33 | */ 34 | public class JSONPointerException extends JSONException { 35 | private static final long serialVersionUID = 8872944667561856751L; 36 | 37 | public JSONPointerException(String message) { 38 | super(message); 39 | } 40 | 41 | public JSONPointerException(String message, Throwable cause) { 42 | super(message, cause); 43 | } 44 | 45 | } -------------------------------------------------------------------------------- /src/main/java/org/json/JSONPropertyIgnore.java: -------------------------------------------------------------------------------- 1 | package org.json; 2 | 3 | /* 4 | Copyright (c) 2018 JSON.org 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | The Software shall be used for Good, not Evil. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | SOFTWARE. 25 | */ 26 | 27 | import static java.lang.annotation.ElementType.METHOD; 28 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 29 | 30 | import java.lang.annotation.Documented; 31 | import java.lang.annotation.Retention; 32 | import java.lang.annotation.Target; 33 | 34 | @Documented 35 | @Retention(RUNTIME) 36 | @Target({METHOD}) 37 | /** 38 | * Use this annotation on a getter method to override the Bean name 39 | * parser for Bean -> JSONObject mapping. If this annotation is 40 | * present at any level in the class hierarchy, then the method will 41 | * not be serialized from the bean into the JSONObject. 42 | */ 43 | public @interface JSONPropertyIgnore { } -------------------------------------------------------------------------------- /src/main/java/org/json/JSONPropertyName.java: -------------------------------------------------------------------------------- 1 | package org.json; 2 | 3 | /* 4 | Copyright (c) 2018 JSON.org 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | The Software shall be used for Good, not Evil. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | SOFTWARE. 25 | */ 26 | 27 | import static java.lang.annotation.ElementType.METHOD; 28 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 29 | 30 | import java.lang.annotation.Documented; 31 | import java.lang.annotation.Retention; 32 | import java.lang.annotation.Target; 33 | 34 | @Documented 35 | @Retention(RUNTIME) 36 | @Target({METHOD}) 37 | /** 38 | * Use this annotation on a getter method to override the Bean name 39 | * parser for Bean -> JSONObject mapping. A value set to empty string "" 40 | * will have the Bean parser fall back to the default field name processing. 41 | */ 42 | public @interface JSONPropertyName { 43 | /** 44 | * @return The name of the property as to be used in the JSON Object. 45 | */ 46 | String value(); 47 | } -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/geo/AbstractGeoPoint.java: -------------------------------------------------------------------------------- 1 | /** 2 | * AbstractGeoPoint 3 | * Copyright 03.06.2015 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.geo; 21 | 22 | public abstract class AbstractGeoPoint implements GeoPoint { 23 | 24 | /** 25 | * compute the distance between two points using the Haversine Algorithm 26 | * https://en.wikipedia.org/wiki/Haversine_formula 27 | * @param othr the other point 28 | * @return the distance of this point and the other point in meter 29 | */ 30 | public double distance(final GeoPoint othr) { 31 | return distance(this.lat(), this.lon(), othr.lat(), othr.lon()); 32 | } 33 | 34 | public static double distance(final double lat1, final double lon1, final double lat2, final double lon2) { 35 | double dlat = (lat2 - lat1) * D2R; double dlon = (lon2 - lon1) * D2R; 36 | double a = Math.pow(Math.sin(dlat / 2.0d), 2.0d) + 37 | Math.cos(lat1 * D2R) * Math.cos(lat2 * D2R) * Math.pow(Math.sin(dlon / 2.0d), 2.0d); 38 | double c = 2.0d * Math.atan2(Math.sqrt(a), Math.sqrt(1.0d - a)); 39 | return EQUATOR_EARTH_RADIUS * c; 40 | } 41 | 42 | static final double EQUATOR_EARTH_RADIUS = 6378137.0d; 43 | static final double D2R = (Math.PI / 180.0d); 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/tools/GitTool.java: -------------------------------------------------------------------------------- 1 | /** 2 | * GitTool 3 | * (C) 2018 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany 4 | * first published 09.01.2018 on http://yacy.net 5 | * 6 | * This library is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU Lesser General Public 8 | * License as published by the Free Software Foundation; either 9 | * version 2.1 of the License, or (at your option) any later version. 10 | * 11 | * This library is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Lesser General Public License 17 | * along with this program in the file lgpl21.txt 18 | * If not, see . 19 | */ 20 | 21 | package net.yacy.grid.tools; 22 | 23 | import java.io.File; 24 | 25 | import org.eclipse.jgit.api.Git; 26 | import org.eclipse.jgit.lib.Repository; 27 | import org.eclipse.jgit.revwalk.RevCommit; 28 | 29 | public class GitTool { 30 | 31 | public String name, message, branch; 32 | 33 | public GitTool() { 34 | File gitWorkDir = new File("."); 35 | try { 36 | Git git = Git.open(gitWorkDir); 37 | Iterable commits = git.log().all().call(); 38 | Repository repo = git.getRepository(); 39 | branch = repo.getBranch(); 40 | RevCommit latestCommit = commits.iterator().next(); 41 | name = latestCommit.getName(); 42 | message = latestCommit.getFullMessage(); 43 | } catch (Throwable e) { 44 | name = ""; 45 | message = ""; 46 | branch = ""; 47 | } 48 | } 49 | 50 | public String toString() { 51 | return "branch " + branch + ": commit " + name + " " + message; 52 | } 53 | 54 | public static void main(String[] args) { 55 | System.out.println(new GitTool().toString()); 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/core/api/info/LogService.java: -------------------------------------------------------------------------------- 1 | /** 2 | * LogService 3 | * Copyright 02.01.2018 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.core.api.info; 21 | 22 | import java.util.List; 23 | 24 | import javax.servlet.http.HttpServletResponse; 25 | 26 | import net.yacy.grid.http.APIHandler; 27 | import net.yacy.grid.http.Log; 28 | import net.yacy.grid.http.ObjectAPIHandler; 29 | import net.yacy.grid.http.Query; 30 | import net.yacy.grid.http.ServiceResponse; 31 | 32 | /** 33 | * The Log Service 34 | * call http://localhost:8010/yacy/grid/core/info/log.txt 35 | */ 36 | public class LogService extends ObjectAPIHandler implements APIHandler { 37 | 38 | private static final long serialVersionUID = -7095346222464124199L; 39 | public static final String NAME = "log"; 40 | 41 | @Override 42 | public String getAPIPath() { 43 | return "/yacy/grid/core/info/" + NAME + ".txt"; 44 | } 45 | 46 | @Override 47 | public ServiceResponse serviceImpl(Query post, HttpServletResponse response) { 48 | int count = post.get("count", 10000); 49 | final StringBuilder buffer = new StringBuilder(100000); 50 | List lines = Log.logAppender.getLines(count); 51 | for (int i = 0; i < lines.size(); i++) { 52 | buffer.append(lines.get(i)); // lines are stored with \n at end 53 | } 54 | return new ServiceResponse(buffer.toString()); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # YaCy Grid Component: Core 2 | 3 | ## What is this? 4 | The YaCy Grid Core is a Leightweight Microservice Operating Environment. 5 | It includes: 6 | * a jetty-based web server 7 | * a very lightweight JSON library using the JSON-Java reference implementation 8 | * an infrastructure to implement servlets using these JSON classes 9 | * libraries for network communication and http clients 10 | * file-based storage classes for JSON objects 11 | * image-generating classes 12 | * a framework for time-series data including image graph generation 13 | 14 | This combination of a lightweight web server together with JSON, time series 15 | and graphics was arranged because most of API-implementing servers need the 16 | ability to easily implement a JSON API, handle network communication, 17 | be able to statify without the need of a database connection and also to 18 | support a telemetry service which makes it possible to monitor a large number 19 | of instances within a scalable microservice environment in an easy way. 20 | 21 | ## How to use this library? 22 | Right now we do not intend to publish the library as a jar file, instead we 23 | recommend to add this repository as a git submodule. 24 | If you have an existing project, do the following to add yacy_grid_core as 25 | as submodule: 26 | ``` 27 | # open a shell and navigate to your project root path 28 | mkdir submodules 29 | cd submodules 30 | git submodule add https://github.com/yacy_yacy_grid_core 31 | ``` 32 | 33 | Your project `` then can be checked out with 34 | ``` 35 | git clone --recursive https://github.com/.git 36 | ``` 37 | 38 | and an update to an already checked out project can be made with 39 | ``` 40 | git pull origin master 41 | git submodule foreach git pull origin master 42 | ``` 43 | 44 | ## How to compile and run yacy_grid_core 45 | Because this is a libaray and not an application there is not really a use 46 | running the program at all. However for demonstration purpose you can actually 47 | start the server without any usefull service. 48 | 49 | To build and run, do: 50 | ``` 51 | gradle assemble 52 | gradle run 53 | ``` 54 | 55 | 56 | ## What is the software license? 57 | LGPL 2.1 (C) by Michael Peter Christen 58 | 59 | Have fun! 60 | @0rb1t3r 61 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/tools/SpaceExceededException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * SpaceExceededException 3 | * Copyright 14.01.2017 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.tools; 21 | 22 | import java.util.Date; 23 | 24 | public class SpaceExceededException extends Exception { 25 | 26 | private static final long serialVersionUID = 9059516027929222151L; 27 | 28 | private final String forUsage; 29 | private final long neededRAM, availableRAM, time; 30 | 31 | public SpaceExceededException(final long neededRAM, final String forUsage) { 32 | super(Long.toString(neededRAM) + " bytes needed for " + forUsage + ": " + Memory.available() + " free at " + (new Date()).toString()); 33 | this.time = System.currentTimeMillis(); 34 | this.availableRAM = Memory.available(); 35 | this.neededRAM = neededRAM; 36 | this.forUsage = forUsage; 37 | } 38 | 39 | public SpaceExceededException(final long neededRAM, final String forUsage, final Throwable t) { 40 | super(Long.toString(neededRAM) + " bytes needed for " + forUsage + ": " + Memory.available() + " free at " + (new Date()).toString(), t); 41 | this.time = System.currentTimeMillis(); 42 | this.availableRAM = Memory.available(); 43 | this.neededRAM = neededRAM; 44 | this.forUsage = forUsage; 45 | } 46 | 47 | public String getUsage() { 48 | return this.forUsage; 49 | } 50 | 51 | public long getNeededRAM() { 52 | return this.neededRAM; 53 | } 54 | 55 | public long getAvailableRAM() { 56 | return this.availableRAM; 57 | } 58 | 59 | public long getTime() { 60 | return this.time; 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /bin/start_rabbitmq.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | 3 | # to run this script, erlang must be installed 4 | # Mac OS: brew install erlang 5 | # debian/ubuntu: apt-get install erlang 6 | 7 | import os 8 | import time 9 | import socket 10 | import urllib 11 | import subprocess 12 | 13 | #https://github.com/rabbitmq/rabbitmq-server/releases/download/v3.7.14/rabbitmq-server-generic-unix-3.7.14.tar.xz 14 | 15 | rabbitversion = 'rabbitmq-server-generic-unix-3.7.14' 16 | rabbitfilename = 'rabbitmq_server-3.7.14' 17 | 18 | path_apphome = os.path.dirname(os.path.abspath(__file__)) + '/..' 19 | os.chdir(path_apphome) 20 | # os.system('ls') 21 | 22 | def checkportopen(port): 23 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 24 | return sock.connect_ex(('127.0.0.1', port)) == 0 25 | 26 | def mkapps(): 27 | if not os.path.isdir(path_apphome + '/data'): os.makedirs(path_apphome + '/data') 28 | if not os.path.isdir(path_apphome + '/data/apps'): os.makedirs(path_apphome + '/data/apps') 29 | 30 | if not checkportopen(5672): 31 | print('rabbitmq is not running') 32 | mkapps() 33 | if not os.path.isfile(path_apphome + '/data/apps/' + rabbitversion + '.tar.xz'): 34 | print('downloading ' + rabbitversion) 35 | downloadurl = 'https://github.com/rabbitmq/rabbitmq-server/releases/download/v3.7.14/' + rabbitversion + '.tar.xz' 36 | downloadpath = path_apphome + '/data/apps/' + rabbitversion + '.tar.xz' 37 | print('Download URL is: ' + downloadurl + ', Path is ' + downloadpath) 38 | urllib.urlretrieve(downloadurl, downloadpath) 39 | rabbitpath = path_apphome + '/data/apps/rabbitmq' 40 | if not os.path.isdir(rabbitpath): 41 | print('decompressing ' + rabbitversion) 42 | os.system('tar xJf ' + path_apphome + '/data/apps/' + rabbitversion + '.tar.xz -C ' + path_apphome + '/data/apps/') 43 | os.rename(path_apphome + '/data/apps/' + rabbitfilename, rabbitpath) 44 | # run rabbitmq 45 | print('running rabbitmq') 46 | os.chdir(rabbitpath + '/sbin') 47 | subprocess.call('./rabbitmq-server &', shell=True) 48 | time.sleep(5) 49 | subprocess.call('./rabbitmq-plugins enable rabbitmq_management', shell=True) 50 | subprocess.call('./rabbitmqctl add_user anonymous yacy', shell=True) 51 | subprocess.call('./rabbitmqctl set_user_tags anonymous administrator', shell=True) 52 | subprocess.call('./rabbitmqctl set_permissions -p / anonymous ".*" ".*" ".*"', shell=True) 53 | 54 | print('to view the administration pages, open http://127.0.0.1:15672/') 55 | print('log in with anonymous:yacy') 56 | -------------------------------------------------------------------------------- /bin/start_elasticsearch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | 3 | import os 4 | import time 5 | import socket 6 | import urllib 7 | import subprocess 8 | 9 | elasticversion = 'elasticsearch-6.8.1' 10 | 11 | path_apphome = os.path.dirname(os.path.abspath(__file__)) + '/..' 12 | os.chdir(path_apphome) 13 | # os.system('ls') 14 | 15 | def checkportopen(port): 16 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 17 | return sock.connect_ex(('127.0.0.1', port)) == 0 18 | 19 | def mkapps(): 20 | if not os.path.isdir(path_apphome + '/data'): os.makedirs(path_apphome + '/data') 21 | if not os.path.isdir(path_apphome + '/data/apps'): os.makedirs(path_apphome + '/data/apps') 22 | 23 | if checkportopen(9200): 24 | print('elasticsearch is already running!') 25 | else: 26 | print('elasticsearch is not running') 27 | mkapps() 28 | if not os.path.isfile(path_apphome + '/data/apps/' + elasticversion + '.tar.gz'): 29 | print('downloading ' + elasticversion) 30 | urllib.urlretrieve ('https://artifacts.elastic.co/downloads/elasticsearch/' + elasticversion + '.tar.gz', path_apphome + '/data/apps/' + elasticversion + '.tar.gz') 31 | elasticpath = path_apphome + '/data/apps/elasticsearch' 32 | firstrun = False 33 | if not os.path.isdir(elasticpath): 34 | print('decompressing' + elasticversion) 35 | os.system('tar xfz ' + path_apphome + '/data/apps/' + elasticversion + '.tar.gz -C ' + path_apphome + '/data/apps/') 36 | os.rename(path_apphome + '/data/apps/' + elasticversion, elasticpath) 37 | os.system('sed -i "s/#network.host: 192.168.0.1/network.host: 0.0.0.0/g" ' + elasticpath + '/config/elasticsearch.yml') 38 | os.system('sed -i "s/#cluster.name: my-application/cluster.name: elasticsearch/g" ' + elasticpath + '/config/elasticsearch.yml') 39 | firstrun = True 40 | # run elasticsearch 41 | print('starting elasticsearch...') 42 | os.chdir(elasticpath + '/bin') 43 | if not os.path.isdir(elasticpath + '/data'): os.makedirs(elasticpath + '/data') 44 | logpath = elasticpath + '/log' 45 | if not os.path.isfile(logpath): os.system('touch ' + logpath) 46 | subprocess.call('./elasticsearch >> ' + logpath + ' &', shell=True) 47 | if firstrun: 48 | while not checkportopen(9200): 49 | print('waiting until elasticsearch is running...') 50 | time.sleep(3) 51 | print('elasticsearch is running at http://127.0.0.1:9200/') 52 | #print('preparing index...') 53 | #os.chdir(path_apphome) 54 | #subprocess.call('bin/prepare_index.sh', shell=True) 55 | 56 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/tools/MapTools.java: -------------------------------------------------------------------------------- 1 | /** 2 | * MapTools 3 | * Copyright 23.03.2016 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.tools; 21 | 22 | import java.util.Collections; 23 | import java.util.HashMap; 24 | import java.util.LinkedHashMap; 25 | import java.util.Map; 26 | import java.util.concurrent.atomic.AtomicInteger; 27 | import java.util.stream.Collectors; 28 | 29 | public class MapTools { 30 | 31 | /** 32 | * sort a given map by the value 33 | * @param map 34 | * @return a map with the same keys where the key with the highest value is first 35 | */ 36 | public static > LinkedHashMap sortByValue(Map map) { 37 | return map 38 | .entrySet() 39 | .stream() 40 | .sorted(Map.Entry.comparingByValue(Collections.reverseOrder())) 41 | .collect(Collectors.toMap( 42 | Map.Entry::getKey, 43 | Map.Entry::getValue, 44 | (e1, e2) -> e1, 45 | LinkedHashMap::new 46 | )); 47 | } 48 | 49 | public static Map deatomize(Map map) { 50 | final Map a = new HashMap<>(); 51 | map.forEach((k, v) -> a.put(k, v.intValue())); 52 | return a; 53 | } 54 | 55 | public static void incCounter(Map map, K key) { 56 | incCounter(map, key, 1); 57 | } 58 | 59 | public static void incCounter(Map map, K key, int inc) { 60 | AtomicInteger c = map.get(key); 61 | if (c == null) { 62 | c = new AtomicInteger(0); 63 | map.put(key, c); 64 | } 65 | c.addAndGet(inc); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/tools/ClassProvider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * ClassProvider 3 | * Copyright 201 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany 4 | * First released 13.12.2011 at http://yacy.net 5 | * 6 | * This library is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU Lesser General Public 8 | * License as published by the Free Software Foundation; either 9 | * version 2.1 of the License, or (at your option) any later version. 10 | * 11 | * This library is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Lesser General Public License 17 | * along with this program in the file lgpl21.txt 18 | * If not, see . 19 | */ 20 | 21 | package net.yacy.grid.tools; 22 | 23 | import java.io.File; 24 | import java.lang.reflect.Method; 25 | import java.net.MalformedURLException; 26 | import java.net.URL; 27 | import java.net.URLClassLoader; 28 | 29 | public class ClassProvider { 30 | 31 | public static Class load(final String classname, final File jarfile) { 32 | Class c; 33 | try { 34 | c = Class.forName(classname); 35 | } catch (final ClassNotFoundException e) { 36 | c = null; 37 | } 38 | if (c == null) { 39 | // load jar 40 | String path = jarfile.getAbsolutePath(); 41 | if (File.separatorChar != '/') path = path.replace(File.separatorChar, '/'); 42 | if (!path.startsWith("/")) path = "/" + path; 43 | URL[] urls; 44 | try { 45 | urls = new URL[]{new URL("file", "", path)}; 46 | @SuppressWarnings("resource") 47 | final ClassLoader cl = new URLClassLoader(urls); 48 | c = cl.loadClass(classname); 49 | } catch (final MalformedURLException e) { 50 | } catch (final ClassNotFoundException e) { 51 | } 52 | } 53 | return c; 54 | } 55 | 56 | public static Method getStaticMethod(final Class c, final String methodName, final Class[] args) { 57 | if (c == null) return null; 58 | try { 59 | return c.getMethod(methodName, args); 60 | } catch (final SecurityException e) { 61 | return null; 62 | } catch (final NoSuchMethodException e) { 63 | return null; 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/http/APIHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * APIHandler 3 | * Copyright 17.05.2016 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.http; 21 | 22 | import java.io.IOException; 23 | import java.util.Map; 24 | 25 | import javax.servlet.http.HttpServletResponse; 26 | 27 | import org.json.JSONObject; 28 | 29 | /** 30 | * Interface for all servlets 31 | */ 32 | public interface APIHandler { 33 | 34 | /** 35 | * get the path to the servlet 36 | * @return the url path of the servlet 37 | */ 38 | public String getAPIPath(); 39 | 40 | /** 41 | * get a name of the servlet (which can be used in the Action object to use servlets for contract actions) 42 | * @return the name element from getAPIPath() 43 | */ 44 | public String getAPIName(); 45 | 46 | /** 47 | * call the servlet with a query locally without a network connection 48 | * @param call a query object 49 | * @param response a http response object 50 | * @return a Service Response 51 | * @throws APIException 52 | */ 53 | public ServiceResponse serviceImpl(Query call, HttpServletResponse response) throws APIException; 54 | 55 | /** 56 | * call a remote servlet with given params 57 | * @param protocolhostportstub the url stub string 58 | * @param params a json object with a set of key/values where each value is of type String 59 | * @return a Service Response 60 | * @throws IOException 61 | */ 62 | public ServiceResponse serviceImpl(final String protocolhostportstub, JSONObject params) throws IOException; 63 | 64 | /** 65 | * call a remote servlet with given params 66 | * @param protocolhostportstub the url stub string 67 | * @param params a map object with a set of key/values where each value is of type byte[] 68 | * @return a Service Response 69 | * @throws IOException 70 | */ 71 | public ServiceResponse serviceImpl(final String protocolhostportstub, Map params) throws IOException; 72 | 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/geo/GeoPoint.java: -------------------------------------------------------------------------------- 1 | /** 2 | * GeoPoint 3 | * Copyright 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany 4 | * first published 08.10.2009 on http://yacy.net 5 | * 6 | * This file is part of YaCy Content Integration 7 | * 8 | * This library is free software; you can redistribute it and/or 9 | * modify it under the terms of the GNU Lesser General Public 10 | * License as published by the Free Software Foundation; either 11 | * version 2.1 of the License, or (at your option) any later version. 12 | * 13 | * This library is distributed in the hope that it will be useful, 14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 | * Lesser General Public License for more details. 17 | * 18 | * You should have received a copy of the GNU Lesser General Public License 19 | * along with this program in the file lgpl21.txt 20 | * If not, see . 21 | */ 22 | 23 | package net.yacy.grid.geo; 24 | 25 | 26 | /** 27 | * Geolocation storage may vary using different data structures for the points. 28 | * The reason to have different implementation is to save memory for the point storage. 29 | * With each version of a point storage comes a accuracy level which can be returned by the object. 30 | */ 31 | public interface GeoPoint { 32 | 33 | public static final double meter = 90.0d / 1.0e7d; // this is actually the definition of 'meter': 10 million meter shall be the distance from the equator to the pole 34 | 35 | /** 36 | * get the latitude of the point 37 | * @return 38 | */ 39 | public double lat(); 40 | 41 | /** 42 | * get the longitude of the point 43 | * @return 44 | */ 45 | public double lon(); 46 | 47 | /** 48 | * get the implementation-dependent accuracy of the latitude 49 | * @return 50 | */ 51 | public double accuracyLat(); 52 | 53 | /** 54 | * get the implementation-dependent accuracy of the longitude 55 | * @return 56 | */ 57 | public double accuracyLon(); 58 | 59 | /** 60 | * compute the hash code of a coordinate 61 | * this produces identical hash codes for locations that are close to each other 62 | */ 63 | @Override 64 | public int hashCode(); 65 | 66 | /** 67 | * equality test that is needed to use the class inside HashMap/HashSet 68 | */ 69 | @Override 70 | public boolean equals(final Object o); 71 | 72 | /** 73 | * compute the distance between two points using the Haversine Algorithm 74 | * https://en.wikipedia.org/wiki/Haversine_formula 75 | * @param othr the other point 76 | * @return the distance of this point and the other point in meter 77 | */ 78 | public double distance(final GeoPoint othr); 79 | 80 | /** 81 | * printout format of the point 82 | * @return 83 | */ 84 | @Override 85 | public String toString(); 86 | 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/tools/CommonPattern.java: -------------------------------------------------------------------------------- 1 | /** 2 | * CommonPattern 3 | * Copyright 2012 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany 4 | * First published 26.11.2012 on http://yacy.net 5 | * 6 | * This library is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU Lesser General Public 8 | * License as published by the Free Software Foundation; either 9 | * version 2.1 of the License, or (at your option) any later version. 10 | * 11 | * This library is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Lesser General Public License 17 | * along with this program in the file lgpl21.txt 18 | * If not, see . 19 | */ 20 | 21 | package net.yacy.grid.tools; 22 | 23 | import java.util.regex.Pattern; 24 | 25 | /** 26 | * This class provides Pattern constants to be used 27 | * to replace a regex in s.split(regex) method calls. 28 | * Because s.split(regex) causes an execution of 29 | * Pattern.compile(regex).split(s, 0), it is wise to pre-compile 30 | * all regex to a pattern p. 31 | * Therefore do the following: transform your code into 32 | * Pattern p = Pattern.compile(regex); p.split(s); 33 | * The compilation of a specific pattern should be done only once. 34 | * Therefore this class provides Pattern objects for the most common regex Strings. 35 | * 36 | * The same applies to s.replaceall(regex, replacement) which is equal to 37 | * Pattern.compile(regex).matcher(s).replaceAll(replacement); 38 | */ 39 | public class CommonPattern { 40 | 41 | public final static Pattern SPACE = Pattern.compile(" "); 42 | public final static Pattern SPACES = Pattern.compile(" +"); // pattern for one or multiple spaces 43 | public final static Pattern COMMA = Pattern.compile(","); 44 | public final static Pattern SEMICOLON = Pattern.compile(";"); 45 | public final static Pattern DOUBLEPOINT = Pattern.compile(":"); 46 | public final static Pattern SLASH = Pattern.compile("/"); 47 | public final static Pattern PIPE = Pattern.compile("\\|"); 48 | public final static Pattern BACKSLASH = Pattern.compile("\\\\"); 49 | public final static Pattern QUESTION = Pattern.compile("\\?"); 50 | public final static Pattern AMP = Pattern.compile("&"); 51 | public final static Pattern PLUS = Pattern.compile(Pattern.quote("+")); 52 | public final static Pattern DOT = Pattern.compile("\\."); 53 | public final static Pattern NEWLINE = Pattern.compile("\n"); 54 | public final static Pattern VERTICALBAR = Pattern.compile(Pattern.quote("|")); 55 | public final static Pattern UNDERSCORE = Pattern.compile("_"); 56 | public final static Pattern TAB = Pattern.compile("\t"); 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/core/api/info/StatusService.java: -------------------------------------------------------------------------------- 1 | /** 2 | * StatusService 3 | * Copyright 27.02.2015 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.core.api.info; 21 | 22 | import javax.servlet.http.HttpServletResponse; 23 | 24 | import org.json.JSONObject; 25 | 26 | import net.yacy.grid.http.APIHandler; 27 | import net.yacy.grid.http.APIServer; 28 | import net.yacy.grid.http.ObjectAPIHandler; 29 | import net.yacy.grid.http.Query; 30 | import net.yacy.grid.http.ServiceResponse; 31 | import net.yacy.grid.tools.OS; 32 | 33 | // test: http://localhost:8010/yacy/grid/core/info/status.json 34 | public class StatusService extends ObjectAPIHandler implements APIHandler { 35 | 36 | private static final long serialVersionUID = 8578478303032749479L; 37 | public static final String NAME = "status"; 38 | 39 | @Override 40 | public String getAPIPath() { 41 | return "/yacy/grid/core/info/" + NAME + ".json"; 42 | } 43 | 44 | @Override 45 | public ServiceResponse serviceImpl(Query call, HttpServletResponse response) { 46 | 47 | // generate json 48 | Runtime runtime = Runtime.getRuntime(); 49 | JSONObject json = new JSONObject(true); 50 | JSONObject system = new JSONObject(true); 51 | system.put("service", APIServer.getName()); 52 | system.put("assigned_memory", runtime.maxMemory()); 53 | system.put("used_memory", runtime.totalMemory() - runtime.freeMemory()); 54 | system.put("available_memory", runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()); 55 | system.put("cores", runtime.availableProcessors()); 56 | system.put("threads", Thread.activeCount()); 57 | system.put("load_system_average", OS.getSystemLoadAverage()); 58 | //system.put("load_system_cpu", OS.getSystemCpuLoad()); 59 | system.put("load_process_cpu", OS.getProcessCpuLoad()); 60 | system.put("server_threads", APIServer.getServerThreads()); 61 | 62 | JSONObject client_info = new JSONObject(true); 63 | JSONObject request_header = new JSONObject(true); 64 | client_info.put("request_header", request_header); 65 | 66 | json.put("system", system); 67 | json.put("client_info", client_info); 68 | 69 | return new ServiceResponse(json); 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/tools/IO.java: -------------------------------------------------------------------------------- 1 | package net.yacy.grid.tools; 2 | 3 | import java.io.IOException; 4 | import java.nio.file.Files; 5 | import java.nio.file.Path; 6 | import java.security.*; 7 | import java.security.spec.InvalidKeySpecException; 8 | import java.security.spec.X509EncodedKeySpec; 9 | import java.util.Base64; 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | 13 | import net.yacy.grid.http.Log; 14 | 15 | public final class IO { 16 | 17 | private static Map map; 18 | private static boolean initialized = false; 19 | 20 | public static String readFile(Path path) throws IOException { 21 | byte[] encoded = Files.readAllBytes(path); 22 | return new String(encoded); 23 | } 24 | 25 | public static String readFileCached(Path path) throws IOException 26 | { 27 | Path absPath = path.toAbsolutePath(); 28 | if(!initialized) init(); 29 | if(map.containsKey(absPath)){ 30 | return map.get(absPath); 31 | } 32 | else{ 33 | String result = readFile(absPath); 34 | map.put(absPath, result); 35 | return result; 36 | } 37 | } 38 | 39 | private static void init(){ 40 | map = new HashMap(); 41 | initialized = true; 42 | } 43 | 44 | /** 45 | * Create hash for a key 46 | * @param pubkey 47 | * @param algorithm 48 | * @return String hash 49 | */ 50 | public static String getKeyHash(PublicKey pubkey, String algorithm){ 51 | try { 52 | MessageDigest md = MessageDigest.getInstance(algorithm); 53 | md.update(pubkey.getEncoded()); 54 | return Base64.getEncoder().encodeToString(md.digest()); 55 | } catch (NoSuchAlgorithmException e) { 56 | Log.logger.error("", e); 57 | } 58 | return null; 59 | } 60 | 61 | /** 62 | * Create hash for a key, use default algorithm SHA-256 63 | * @param pubkey 64 | * @return String hash 65 | */ 66 | public static String getKeyHash(PublicKey pubkey){ 67 | return getKeyHash(pubkey, "SHA-256"); 68 | } 69 | 70 | /** 71 | * Get String representation of a key 72 | * @param key 73 | * @return String representation of a key 74 | */ 75 | public static String getKeyAsString(Key key){ 76 | return Base64.getEncoder().encodeToString(key.getEncoded()); 77 | } 78 | 79 | /** 80 | * Create PublicKey from String representation 81 | * @param encodedKey 82 | * @param algorithm 83 | * @return PublicKey public_key 84 | */ 85 | public synchronized static PublicKey decodePublicKey(String encodedKey, String algorithm){ 86 | try{ 87 | X509EncodedKeySpec keySpec = new X509EncodedKeySpec(Base64.getDecoder().decode(encodedKey)); 88 | PublicKey pub = KeyFactory.getInstance(algorithm).generatePublic(keySpec); 89 | return pub; 90 | } 91 | catch(NoSuchAlgorithmException | InvalidKeySpecException e) { 92 | Log.logger.error("", e); 93 | } 94 | return null; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/core/Core.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Base 3 | * Copyright 14.01.2017 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.core; 21 | 22 | import java.io.IOException; 23 | import java.util.ArrayList; 24 | import java.util.Arrays; 25 | import java.util.List; 26 | 27 | import javax.servlet.Servlet; 28 | 29 | import net.yacy.grid.core.api.info.LogService; 30 | import net.yacy.grid.core.api.info.StatusService; 31 | import net.yacy.grid.core.api.info.ThreaddumpService; 32 | import net.yacy.grid.core.api.vis.MapService; 33 | import net.yacy.grid.core.api.vis.MarkdownService; 34 | import net.yacy.grid.http.APIServer; 35 | import net.yacy.grid.http.Log; 36 | import net.yacy.grid.tools.GitTool; 37 | 38 | public class Core { 39 | 40 | public final static String DATA_PATH = "data"; 41 | 42 | // define services 43 | @SuppressWarnings("unchecked") 44 | public final static Class[] BASE_SERVICES = new Class[]{ 45 | // information services 46 | StatusService.class, 47 | ThreaddumpService.class, 48 | LogService.class, 49 | 50 | // graphic services 51 | MapService.class, 52 | MarkdownService.class 53 | }; 54 | 55 | public static void main(String[] args) { 56 | // initialize environment variables 57 | System.setProperty("java.awt.headless", "true"); // no awt used here so we can switch off that stuff 58 | 59 | // start server 60 | List> services = new ArrayList<>(); 61 | services.addAll(Arrays.asList(BASE_SERVICES)); 62 | APIServer.initEnvironment("base", 8888, services, DATA_PATH, true); 63 | try { 64 | APIServer.open(null, false); 65 | 66 | // start server 67 | Log.logger.info("started Base!"); 68 | Log.logger.info("read status of base peer:"); 69 | Log.logger.info("curl http://127.0.0.1:8010/yacy/grid/core/info/status.json"); 70 | Log.logger.info("curl http://127.0.0.1:8010/yacy/grid/core/info/log.txt"); 71 | Log.logger.info("curl http://127.0.0.1:8010/yacy/grid/core/info/threaddump.txt"); 72 | Log.logger.info(new GitTool().toString()); 73 | APIServer.runService(null); 74 | } catch (IOException e) { 75 | Log.logger.info("cannot start Base"); 76 | e.printStackTrace(); 77 | } 78 | 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /bin/clear_loader_queues.sh: -------------------------------------------------------------------------------- 1 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_00' 2 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_01' 3 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_02' 4 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_03' 5 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_04' 6 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_05' 7 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_06' 8 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_07' 9 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_08' 10 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_09' 11 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_10' 12 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_11' 13 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_12' 14 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_13' 15 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_14' 16 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_15' 17 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_16' 18 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_17' 19 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_18' 20 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_19' 21 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_20' 22 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_21' 23 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_22' 24 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_23' 25 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_24' 26 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_25' 27 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_26' 28 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_27' 29 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_28' 30 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_29' 31 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_30' 32 | curl 'http://127.0.0.1:8100/yacy/grid/mcp/messages/clear.json?serviceName=loader&queueName=webloader_31' 33 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 33 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 34 | 35 | @rem Find java.exe 36 | if defined JAVA_HOME goto findJavaFromJavaHome 37 | 38 | set JAVA_EXE=java.exe 39 | %JAVA_EXE% -version >NUL 2>&1 40 | if "%ERRORLEVEL%" == "0" goto init 41 | 42 | echo. 43 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 44 | echo. 45 | echo Please set the JAVA_HOME variable in your environment to match the 46 | echo location of your Java installation. 47 | 48 | goto fail 49 | 50 | :findJavaFromJavaHome 51 | set JAVA_HOME=%JAVA_HOME:"=% 52 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 53 | 54 | if exist "%JAVA_EXE%" goto init 55 | 56 | echo. 57 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 58 | echo. 59 | echo Please set the JAVA_HOME variable in your environment to match the 60 | echo location of your Java installation. 61 | 62 | goto fail 63 | 64 | :init 65 | @rem Get command-line arguments, handling Windows variants 66 | 67 | if not "%OS%" == "Windows_NT" goto win9xME_args 68 | 69 | :win9xME_args 70 | @rem Slurp the command line arguments. 71 | set CMD_LINE_ARGS= 72 | set _SKIP=2 73 | 74 | :win9xME_args_slurp 75 | if "x%~1" == "x" goto execute 76 | 77 | set CMD_LINE_ARGS=%* 78 | 79 | :execute 80 | @rem Setup the command line 81 | 82 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 83 | 84 | @rem Execute Gradle 85 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 86 | 87 | :end 88 | @rem End local scope for the variables with windows NT shell 89 | if "%ERRORLEVEL%"=="0" goto mainEnd 90 | 91 | :fail 92 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 93 | rem the _cmd.exe /c_ return code! 94 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 95 | exit /b 1 96 | 97 | :mainEnd 98 | if "%OS%"=="Windows_NT" endlocal 99 | 100 | :omega 101 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/json/JsonObjectWithDefault.java: -------------------------------------------------------------------------------- 1 | /** 2 | * UserManagement 3 | * Copyright 23.06.2015 by Robert Mader, @treba13 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.json; 21 | 22 | import org.json.JSONArray; 23 | import org.json.JSONObject; 24 | 25 | /** 26 | * This class extends JSONObjects with additional get-methods that accept default values. It catches all possible errors and exceptions, 27 | * thus always returning a valid output. It's intention is to provide a saver way to acquire values in security/stability sensitive environments 28 | */ 29 | public class JsonObjectWithDefault extends JSONObject { 30 | 31 | public JsonObjectWithDefault(){ 32 | super(); 33 | } 34 | 35 | public JsonObjectWithDefault(JSONObject src){ 36 | this(); 37 | if(src != null) putAll(src); 38 | } 39 | 40 | public boolean getBoolean(String key, boolean dftval){ 41 | try{ 42 | return getBoolean(key); 43 | } 44 | catch (Throwable e){ 45 | return dftval; 46 | } 47 | } 48 | 49 | public double getDouble(String key, double dftval){ 50 | try{ 51 | return getDouble(key); 52 | } 53 | catch (Throwable e){ 54 | return dftval; 55 | } 56 | } 57 | 58 | public int getInt(String key, int dftval){ 59 | try{ 60 | return getInt(key); 61 | } 62 | catch (Throwable e){ 63 | return dftval; 64 | } 65 | } 66 | public JSONArray getJSONArray(String key, JSONArray dftval){ 67 | try{ 68 | return getJSONArray(key); 69 | } 70 | catch (Throwable e){ 71 | return dftval; 72 | } 73 | } 74 | public JSONObject getJSONObject(String key, JSONObject dftval){ 75 | try{ 76 | return getJSONObject(key); 77 | } 78 | catch (Throwable e){ 79 | return dftval; 80 | } 81 | } 82 | 83 | public JsonObjectWithDefault getJSONObjectWithDefault(String key, JsonObjectWithDefault dftval){ 84 | try{ 85 | return new JsonObjectWithDefault(getJSONObject(key)); 86 | } 87 | catch (Throwable e){ 88 | return dftval; 89 | } 90 | } 91 | public long getLong(String key, long dftval){ 92 | try{ 93 | return getLong(key); 94 | } 95 | catch (Throwable e){ 96 | return dftval; 97 | } 98 | } 99 | public String getString(String key, String dftval){ 100 | try{ 101 | return getString(key); 102 | } 103 | catch (Throwable e){ 104 | return dftval; 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/graphics/AnimationPlotter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * AnimationPlotter 3 | * Copyright 2013 by Michael Christen 4 | * First released 9.9.2010 at http://yacy.net 5 | * 6 | * This library is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU Lesser General Public 8 | * License as published by the Free Software Foundation; either 9 | * version 2.1 of the License, or (at your option) any later version. 10 | * 11 | * This library is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Lesser General Public License 17 | * along with this program in the file lgpl21.txt 18 | * If not, see . 19 | */ 20 | 21 | package net.yacy.grid.graphics; 22 | 23 | import java.awt.image.BufferedImage; 24 | import java.io.File; 25 | import java.io.FileOutputStream; 26 | import java.io.IOException; 27 | import java.util.ArrayList; 28 | import java.util.List; 29 | 30 | import javax.imageio.ImageIO; 31 | import javax.swing.ImageIcon; 32 | import javax.swing.JFrame; 33 | import javax.swing.JLabel; 34 | 35 | public class AnimationPlotter { 36 | 37 | public static class Frame { 38 | BufferedImage image; 39 | int delayMillis; 40 | public Frame(BufferedImage image, int delayMillis) { 41 | this.image = image; 42 | this.delayMillis = delayMillis; 43 | } 44 | } 45 | 46 | private final List frames; 47 | 48 | 49 | public AnimationPlotter() { 50 | this.frames = new ArrayList(); 51 | } 52 | 53 | public void addFrame(final BufferedImage image, final int delayMillis) { 54 | this.frames.add(new Frame(image, delayMillis)); 55 | } 56 | 57 | 58 | public void save(final File path, final String filestub, final String type) throws IOException { 59 | assert path.isDirectory(); 60 | for (int i = 0; i < this.frames.size(); i++) { 61 | Frame frame = this.frames.get(i); 62 | File file = new File(path, filestub + "_" + intformat(i) + '.' + type); 63 | final FileOutputStream fos = new FileOutputStream(file); 64 | ImageIO.write(frame.image, type, fos); 65 | fos.close(); 66 | } 67 | } 68 | 69 | private String intformat(final int i) { 70 | String n = Integer.toString(i); 71 | while (n.length() < 6) n = '0' + n; 72 | return n; 73 | } 74 | 75 | /** 76 | * show the images as stream of JFrame on desktop 77 | */ 78 | public void show() { 79 | JFrame f = new JFrame(); 80 | f.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); 81 | f.setVisible(true); 82 | JLabel label = null; 83 | while (true) { 84 | for (int i = 0; i < this.frames.size(); i++) { 85 | Frame frame = this.frames.get(i); 86 | if (label == null) { 87 | label = new JLabel(new ImageIcon(frame.image)); 88 | f.getContentPane().add(label); 89 | f.pack(); 90 | } else { 91 | label.getGraphics().drawImage(frame.image,0,0, label); 92 | } 93 | try {Thread.sleep(frame.delayMillis);} catch (InterruptedException e) {} 94 | } 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/tools/Compression.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Compression 3 | * Copyright 12.06.2015 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.tools; 21 | 22 | import java.io.BufferedInputStream; 23 | import java.io.BufferedOutputStream; 24 | import java.io.ByteArrayInputStream; 25 | import java.io.ByteArrayOutputStream; 26 | import java.io.File; 27 | import java.io.FileInputStream; 28 | import java.io.FileOutputStream; 29 | import java.io.IOException; 30 | import java.util.zip.Deflater; 31 | import java.util.zip.GZIPInputStream; 32 | import java.util.zip.GZIPOutputStream; 33 | 34 | public class Compression { 35 | 36 | public static void gzip(File source, File dest, boolean deleteSource) throws IOException { 37 | byte[] buffer = new byte[2^20]; 38 | GZIPOutputStream out = new GZIPOutputStream(new BufferedOutputStream(new FileOutputStream(dest), 65536)){{def.setLevel(Deflater.BEST_COMPRESSION);}}; 39 | FileInputStream in = new FileInputStream(source); 40 | int l; while ((l = in.read(buffer)) > 0) out.write(buffer, 0, l); 41 | in.close(); out.finish(); out.close(); 42 | if (deleteSource && dest.exists()) source.delete(); 43 | } 44 | 45 | public static byte[] gzip(byte[] b) { 46 | try { 47 | ByteArrayOutputStream baos = new ByteArrayOutputStream(b.length); 48 | GZIPOutputStream out = new GZIPOutputStream(baos, Math.min(65536, b.length)){{def.setLevel(Deflater.BEST_COMPRESSION);}}; 49 | out.write(b, 0, b.length); 50 | out.finish(); 51 | out.close(); 52 | return baos.toByteArray(); 53 | } catch (IOException e) {} 54 | return null; 55 | } 56 | 57 | public static void gunzip(File source, File dest, boolean deleteSource) throws IOException { 58 | byte[] buffer = new byte[2^20]; 59 | FileOutputStream out = new FileOutputStream(dest); 60 | GZIPInputStream in = new GZIPInputStream(new BufferedInputStream(new FileInputStream(source))); 61 | int l; while ((l = in.read(buffer)) > 0) out.write(buffer, 0, l); 62 | in.close(); out.close(); 63 | if (deleteSource && dest.exists()) source.delete(); 64 | } 65 | 66 | public static byte[] gunzip(byte[] b) { 67 | byte[] buffer = new byte[Math.min(2^20, b.length)]; 68 | try { 69 | ByteArrayOutputStream baos = new ByteArrayOutputStream(b.length * 2); 70 | GZIPInputStream in = new GZIPInputStream(new ByteArrayInputStream(b), Math.min(65536, b.length)); 71 | int l; while ((l = in.read(buffer)) > 0) baos.write(buffer, 0, l); 72 | in.close(); 73 | baos.close(); 74 | return baos.toByteArray(); 75 | } catch (IOException e) {} 76 | return null; 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/tools/CacheMap.java: -------------------------------------------------------------------------------- 1 | /** 2 | * CacheMap 3 | * Copyright 22.02.2015 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.tools; 21 | 22 | import java.util.Iterator; 23 | import java.util.LinkedHashMap; 24 | 25 | public class CacheMap { 26 | 27 | private int maxSize; 28 | private LinkedHashMap map; 29 | 30 | public CacheMap(int maxSize) { 31 | this.maxSize = maxSize; 32 | this.map = new LinkedHashMap(); 33 | } 34 | 35 | public int size() { 36 | return this.map.size(); 37 | } 38 | 39 | public int maxSize() { 40 | return this.maxSize; 41 | } 42 | 43 | public void clear() { 44 | this.map.clear(); 45 | } 46 | 47 | private void checkSize() { 48 | if (this.map.size() >= this.maxSize) { 49 | Iterator i = this.map.keySet().iterator(); 50 | while (i.hasNext() && this.map.size() > this.maxSize) this.map.remove(i.next()); 51 | } 52 | } 53 | 54 | public boolean full() { 55 | return this.map.size() >= this.maxSize; 56 | } 57 | 58 | public V put(K key, V value) { 59 | V oldval; 60 | synchronized (this.map) { 61 | // make room; this may remove entries from the beginning of the list 62 | checkSize(); 63 | 64 | // we remove the value first to ensure that the value gets at the end of the list 65 | oldval = this.map.remove(key); 66 | 67 | // the new value gets to the end of the list 68 | this.map.put(key, value); 69 | } 70 | return oldval; 71 | } 72 | 73 | public V get(K key) { 74 | V value; 75 | synchronized (this.map) { 76 | // we remove the value to add it again at the end of the list 77 | value = this.map.remove(key); 78 | 79 | // in case that the entry does not exist we are ready here 80 | if (value == null) { 81 | return null; 82 | } 83 | 84 | // the old value gets to the end of the list 85 | this.map.put(key, value); 86 | } 87 | return value; 88 | } 89 | 90 | public V remove(K key) { 91 | synchronized (this.map) { 92 | return this.map.remove(key); 93 | } 94 | } 95 | 96 | public boolean exist(K key) { 97 | boolean exist = false; 98 | synchronized (this.map) { 99 | exist = this.map.containsKey(key); 100 | } 101 | return exist; 102 | } 103 | 104 | public LinkedHashMap getMap(){ 105 | return map; 106 | } 107 | 108 | } 109 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/tools/HashARC.java: -------------------------------------------------------------------------------- 1 | /** 2 | * HashARC 3 | * an Adaptive Replacement Cache for objects that can be compared using hashing 4 | * Copyright 2009 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany 5 | * First released 17.04.2009 at http://yacy.net 6 | * 7 | * This library is free software; you can redistribute it and/or 8 | * modify it under the terms of the GNU Lesser General Public 9 | * License as published by the Free Software Foundation; either 10 | * version 2.1 of the License, or (at your option) any later version. 11 | * 12 | * This library is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | * Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program in the file lgpl21.txt 19 | * If not, see . 20 | */ 21 | 22 | package net.yacy.grid.tools; 23 | 24 | import java.util.Collections; 25 | import java.util.HashMap; 26 | import java.util.LinkedHashMap; 27 | import java.util.Map; 28 | import java.util.Random; 29 | 30 | public final class HashARC extends SimpleARC implements Map, Iterable>, ARC { 31 | 32 | public final static boolean accessOrder = false; // if false, then a insertion-order is used 33 | 34 | public HashARC(final int cacheSize) { 35 | this.cacheSize = cacheSize / 2; 36 | super.levelA = Collections.synchronizedMap(new LinkedHashMap(1, 0.1f, accessOrder) { 37 | private static final long serialVersionUID = 1L; 38 | @Override protected boolean removeEldestEntry(final Map.Entry eldest) { 39 | return size() > HashARC.this.cacheSize; 40 | } 41 | }); 42 | this.levelB = Collections.synchronizedMap(new LinkedHashMap(1, 0.1f, accessOrder) { 43 | private static final long serialVersionUID = 1L; 44 | @Override protected boolean removeEldestEntry(final Map.Entry eldest) { 45 | return size() > HashARC.this.cacheSize; 46 | } 47 | }); 48 | } 49 | 50 | public static void main(final String[] args) { 51 | final Random r = new Random(); 52 | final int testsize = 10000; 53 | final ARC a = new HashARC(testsize * 2); 54 | final Map b = new HashMap(); 55 | String key, value; 56 | for (int i = 0; i < testsize; i++) { 57 | key = "k" + r.nextInt(); 58 | value = "v" + r.nextInt(); 59 | a.insertIfAbsent(key, value); 60 | b.put(key, value); 61 | } 62 | 63 | // now put half of the entries AGAIN into the ARC 64 | int h = testsize / 2; 65 | for (final Map.Entry entry: b.entrySet()) { 66 | a.put(entry.getKey(), entry.getValue()); 67 | if (h-- <= 0) break; 68 | } 69 | 70 | // test correctness 71 | for (final Map.Entry entry: b.entrySet()) { 72 | if (!a.containsKey(entry.getKey())) { 73 | System.out.println("missing: " + entry.getKey()); 74 | continue; 75 | } 76 | if (!a.get(entry.getKey()).equals(entry.getValue())) { 77 | System.out.println("wrong: a = " + entry.getKey() + "," + a.get(entry.getKey()) + "; v = " + entry.getValue()); 78 | } 79 | } 80 | System.out.println("finished test!"); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/json/JsonList.java: -------------------------------------------------------------------------------- 1 | /** 2 | * JSONList 3 | * (C) 3.7.2017 by Michael Peter Christen; mc@yacy.net, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.json; 21 | 22 | import java.io.BufferedReader; 23 | import java.io.ByteArrayInputStream; 24 | import java.io.IOException; 25 | import java.io.InputStream; 26 | import java.io.InputStreamReader; 27 | import java.nio.charset.StandardCharsets; 28 | import java.util.Iterator; 29 | 30 | import org.json.JSONArray; 31 | import org.json.JSONException; 32 | import org.json.JSONObject; 33 | import org.json.JSONTokener; 34 | 35 | /** 36 | * A JSONList is an object which represents a list of json objects: that is 37 | * a popular file format which is used as elasticsearch bulk index format. 38 | * A jsonlist file is a text file where every line is a json object. 39 | * To bring a JSONArray into jsonlist format you have to do 40 | * - make sure that every element in the JSONArray is JSONObject 41 | * - print out every json in the list without indentation 42 | */ 43 | public class JsonList implements Iterable { 44 | 45 | private JSONArray array; 46 | 47 | public JsonList() { 48 | this.array = new JSONArray(); 49 | } 50 | 51 | public JsonList(InputStream sourceStream) throws IOException { 52 | this(); 53 | BufferedReader br = new BufferedReader(new InputStreamReader(sourceStream, StandardCharsets.UTF_8)); 54 | String line; 55 | try { 56 | while ((line = br.readLine()) != null) { 57 | line = line.trim(); 58 | if (line.length() == 0) continue; 59 | JSONObject json = new JSONObject(new JSONTokener(line)); 60 | this.add(json); 61 | } 62 | } catch (JSONException e) { 63 | throw new IOException(e); 64 | } 65 | } 66 | 67 | public JsonList(JSONArray a) throws IOException { 68 | for (int i = 0; i < a.length(); i++) { 69 | if (!(a.get(i) instanceof JSONObject)) throw new IOException("all objects in JSONArray must be JSONObject"); 70 | }; 71 | this.array = a; 72 | } 73 | 74 | public JsonList(byte[] b) throws IOException { 75 | this(new ByteArrayInputStream(b)); 76 | } 77 | 78 | public JsonList(String jsonlist) throws IOException { 79 | this(jsonlist.getBytes(StandardCharsets.UTF_8)); 80 | } 81 | 82 | public JsonList add(JSONObject object) { 83 | this.array.put(object); 84 | return this; 85 | } 86 | 87 | public JSONObject get(int i) { 88 | return this.array.getJSONObject(i); 89 | } 90 | 91 | public int length() { 92 | return this.array.length(); 93 | } 94 | 95 | public String toString() { 96 | StringBuffer sb = new StringBuffer(); 97 | this.array.forEach(entry -> sb.append(entry.toString()).append("\n")); 98 | return sb.toString(); 99 | } 100 | 101 | public JSONArray toArray() { 102 | return this.array; 103 | } 104 | 105 | @Override 106 | public Iterator iterator() { 107 | return this.array.iterator(); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /bin/run_all.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import socket 5 | import urllib 6 | import subprocess 7 | from multiprocessing import Process 8 | 9 | 10 | path_apphome = os.path.dirname(os.path.abspath(__file__)) + '/..' 11 | os.chdir(path_apphome) 12 | # os.system('ls') 13 | 14 | def checkportopen(port): 15 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 16 | return sock.connect_ex(('127.0.0.1', port)) == 0 17 | 18 | def mkapps(): 19 | if not os.path.isdir(path_apphome + '/data'): os.makedirs(path_apphome + '/data') 20 | if not os.path.isdir(path_apphome + '/data/mcp-8100'): os.makedirs(path_apphome + '/data/mcp-8100') 21 | if not os.path.isdir(path_apphome + '/data/mcp-8100/apps'): os.makedirs(path_apphome + '/data/mcp-8100/apps') 22 | 23 | def run_mcp(): 24 | os.system('gradle run') 25 | 26 | def run_loader(): 27 | os.system('cd ../yacy_grid_loader') 28 | os.system('gradle run') 29 | 30 | def run_crawler(): 31 | os.system('cd ../yacy_grid_crawler') 32 | os.system('gradle run') 33 | 34 | def run_parser(): 35 | os.system('cd ../yacy_grid_parser') 36 | os.system('gradle run') 37 | 38 | 39 | if not checkportopen(9200): 40 | print "Elasticsearch is not running" 41 | mkapps() 42 | elasticversion = 'elasticsearch-5.6.5' 43 | if not os.path.isfile(path_apphome + '/data/mcp-8100/apps/' + elasticversion + '.tar.gz'): 44 | print('Downloading ' + elasticversion) 45 | urllib.urlretrieve ('https://artifacts.elastic.co/downloads/elasticsearch/' + elasticversion + '.tar.gz', path_apphome + '/data/mcp-8100/apps/' + elasticversion + '.tar.gz') 46 | if not os.path.isdir(path_apphome + '/data/mcp-8100/apps/elasticsearch'): 47 | print('Decompressing' + elasticversion) 48 | os.system('tar xfz ' + path_apphome + '/data/mcp-8100/apps/' + elasticversion + '.tar.gz -C ' + path_apphome + '/data/mcp-8100/apps/') 49 | os.rename(path_apphome + '/data/mcp-8100/apps/' + elasticversion, path_apphome + '/data/mcp-8100/apps/elasticsearch') 50 | # run elasticsearch 51 | print('Running Elasticsearch') 52 | os.chdir(path_apphome + '/data/mcp-8100/apps/elasticsearch/bin') 53 | os.system('nohup ./elasticsearch &') 54 | 55 | os.chdir(path_apphome) 56 | 57 | if checkportopen(15672): 58 | print "RabbitMQ is Running" 59 | print "If you have configured it according to YaCy setup press N" 60 | print "If you have not configured it according to YaCy setup or Do not know what to do press Y" 61 | n=raw_input() 62 | if(n=='Y' or n=='y'): 63 | os.system('service rabbitmq-server stop') 64 | 65 | if not checkportopen(15672): 66 | print "rabbitmq is not running" 67 | os.system('python bin/start_rabbitmq.py') 68 | 69 | #subprocess.call('bin/update_all.sh') 70 | 71 | if not checkportopen(2121): 72 | print "ftp server is not Running" 73 | 74 | 75 | p1=None 76 | p2=None 77 | p3=None 78 | p4=None 79 | if not checkportopen(8100): 80 | print "yacy_grid_mcp is not running,running yacy_grid_mcp" 81 | p1 = Process(target=run_mcp) 82 | p1.start() 83 | 84 | 85 | if not checkportopen(8200): 86 | print "yacy_grid_loader is not running,running yacy_grid_loader" 87 | p2 = Process(target=run_loader) 88 | p2.start() 89 | 90 | 91 | if not checkportopen(8300): 92 | print "yacy_grid_crawler is not running,running yacy_grid_crawler" 93 | p3 = Process(target=run_crawler) 94 | p3.start() 95 | 96 | 97 | if not checkportopen(8500): 98 | print "yacy_grid_parser is not running,running yacy_grid_parser" 99 | p4 = Process(target=run_parser) 100 | p4.start() 101 | 102 | if p1 : 103 | p1.join 104 | if p2 : 105 | p2.join 106 | if p3 : 107 | p3.join 108 | if p4 : 109 | p4.join 110 | 111 | 112 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/geo/IntegerGeoPoint.java: -------------------------------------------------------------------------------- 1 | /** 2 | * IntegerGeoPoint 3 | * Copyright 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany 4 | * first published 08.10.2009 on http://yacy.net 5 | * 6 | * This file is part of YaCy Content Integration 7 | * 8 | * This library is free software; you can redistribute it and/or 9 | * modify it under the terms of the GNU Lesser General Public 10 | * License as published by the Free Software Foundation; either 11 | * version 2.1 of the License, or (at your option) any later version. 12 | * 13 | * This library is distributed in the hope that it will be useful, 14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 | * Lesser General Public License for more details. 17 | * 18 | * You should have received a copy of the GNU Lesser General Public License 19 | * along with this program in the file lgpl21.txt 20 | * If not, see . 21 | */ 22 | 23 | package net.yacy.grid.geo; 24 | 25 | /** 26 | * GeoPoint implementation with Integer accuracy 27 | */ 28 | public class IntegerGeoPoint extends AbstractGeoPoint implements GeoPoint { 29 | private final long latlon; // using one variable for the coordinate pair saves some space 30 | 31 | /** 32 | * Generates an IntegerGeoPoint from doubles 33 | * @param lat The latitude e.g. 54.123 34 | * @param lon The longitude e.g. 13.123 35 | */ 36 | public IntegerGeoPoint(double lat, double lon) { 37 | this.latlon = (((long) coord2int(lat)) << 32) | (coord2int(lon)); 38 | } 39 | 40 | /** 41 | * Generates an IntegerGeoPoint from integers 42 | * @param lat Eight digit latitude, e.g. (Brandenburger Tor, Berlin) 52516389 43 | * @param lon Eight digit longitude, e.g. (Brandenburger Tor, Berlin) 13377778 44 | */ 45 | public IntegerGeoPoint(int lat, int lon) { 46 | this.latlon = (((long) coord2int(lat / 1e6d)) << 32) | (coord2int(lon / 1e6d)); 47 | } 48 | 49 | /** 50 | * Returns the latitude as a double 51 | */ 52 | @Override 53 | public double lat() { 54 | return int2coord((int) (this.latlon >>> 32)); 55 | } 56 | 57 | 58 | @Override 59 | public double lon() { 60 | return int2coord((int) (this.latlon & (Integer.MAX_VALUE))); 61 | } 62 | 63 | /** 64 | * get the implementation-dependent accuracy of the latitude 65 | * @return 66 | */ 67 | @Override 68 | public double accuracyLat() { 69 | return Math.abs(int2coord(1) - int2coord(2)); 70 | } 71 | 72 | /** 73 | * get the implementation-dependent accuracy of the longitude 74 | * @return 75 | */ 76 | @Override 77 | public double accuracyLon() { 78 | return Math.abs(int2coord(1) - int2coord(2)); 79 | } 80 | 81 | private static final double maxint = new Double(Integer.MAX_VALUE).doubleValue(); 82 | private static final double upscale = maxint / 360.0d; 83 | 84 | private static final int coord2int(double coord) { 85 | return (int) ((coord + 180.0d) * upscale); 86 | } 87 | 88 | private static final double int2coord(int z) { 89 | return (z / upscale) - 180.0d; 90 | } 91 | 92 | /** 93 | * compute the hash code of a coordinate 94 | * this produces identical hash codes for locations that are close to each other 95 | */ 96 | @Override 97 | public int hashCode() { 98 | return (int) ((this.latlon & Integer.MAX_VALUE) >> 1) + (int) (this.latlon >> 33); 99 | } 100 | 101 | /** 102 | * equality test that is needed to use the class inside HashMap/HashSet 103 | */ 104 | @Override 105 | public boolean equals(final Object o) { 106 | if (!(o instanceof IntegerGeoPoint)) return false; 107 | IntegerGeoPoint oo = (IntegerGeoPoint) o; 108 | return (this.latlon == oo.latlon); 109 | } 110 | 111 | @Override 112 | public String toString() { 113 | return "[" + this.lat() + "," + this.lon() + "]"; 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/tools/ARC.java: -------------------------------------------------------------------------------- 1 | /** 2 | * ARC 3 | * an interface for Adaptive Replacement Caches 4 | * Copyright 2009 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany 5 | * First released 29.08.2009 at http://yacy.net 6 | * 7 | * This library is free software; you can redistribute it and/or 8 | * modify it under the terms of the GNU Lesser General Public 9 | * License as published by the Free Software Foundation; either 10 | * version 2.1 of the License, or (at your option) any later version. 11 | * 12 | * This library is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | * Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program in the file lgpl21.txt 19 | * If not, see . 20 | */ 21 | 22 | package net.yacy.grid.tools; 23 | 24 | import java.util.Collection; 25 | import java.util.Iterator; 26 | import java.util.Map; 27 | import java.util.Set; 28 | 29 | /** 30 | * This is a simple cache using two generations of hashtables to store the content with a LFU strategy. 31 | * The Algorithm is described in a slightly more complex version as Adaptive Replacement Cache, "ARC". 32 | * For details see http://www.almaden.ibm.com/cs/people/dmodha/ARC.pdf 33 | * or http://en.wikipedia.org/wiki/Adaptive_Replacement_Cache 34 | * This version omits the ghost entry handling which is described in ARC, and keeps both cache levels 35 | * at the same size. 36 | */ 37 | public interface ARC extends Iterable> { 38 | 39 | /** 40 | * get the size of the ARC. this returns the sum of main and ghost cache 41 | * @return the complete number of entries in the ARC cache 42 | */ 43 | public int size(); 44 | 45 | /** 46 | * put a value to the cache. 47 | * do not return a previous content value 48 | * @param s 49 | * @param v 50 | */ 51 | public void insert(K s, V v); 52 | 53 | /** 54 | * put a value to the cache if there was not an entry before 55 | * do not return a previous content value 56 | * @param s 57 | * @param v 58 | */ 59 | public void insertIfAbsent(K s, V v); 60 | 61 | /** 62 | * put a value to the cache if there was not an entry before 63 | * return a previous content value 64 | * @param s 65 | * @param v 66 | * @return the value before inserting the new value 67 | */ 68 | public V putIfAbsent(K s, V v); 69 | 70 | /** 71 | * put a value to the cache. 72 | * @param s 73 | * @param v 74 | */ 75 | public V put(K s, V v); 76 | 77 | 78 | /** 79 | * get a value from the cache. 80 | * @param s 81 | * @return the value 82 | */ 83 | public V get(K s); 84 | 85 | /** 86 | * check if the map contains the value 87 | * @param value 88 | * @return the keys that have the given value 89 | */ 90 | public Collection getKeys(V value); 91 | 92 | /** 93 | * check if the map contains the key 94 | * @param key 95 | * @return true if the map contains the key 96 | */ 97 | public boolean containsKey(K key); 98 | 99 | /** 100 | * remove an entry from the cache 101 | * @param s 102 | * @return the old value 103 | */ 104 | public V remove(K s); 105 | 106 | /** 107 | * clear the cache 108 | */ 109 | public void clear(); 110 | 111 | /** 112 | * iterator implements the Iterable interface 113 | * the method can easily be implemented using the entrySet method 114 | */ 115 | @Override 116 | public Iterator> iterator(); 117 | 118 | /** 119 | * Return a Set view of the mappings contained in this map. 120 | * This method is the basis for all methods that are implemented 121 | * by a AbstractMap implementation 122 | * 123 | * @return a set view of the mappings contained in this map 124 | */ 125 | public Set> entrySet(); 126 | 127 | /** 128 | * a hash code for this ARC 129 | * @return a hash code 130 | */ 131 | @Override 132 | int hashCode(); 133 | } 134 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/http/Log.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Data 3 | * Copyright 14.01.2017 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.http; 21 | 22 | import java.net.InetAddress; 23 | import java.net.UnknownHostException; 24 | import java.util.ArrayList; 25 | import java.util.concurrent.ConcurrentLinkedQueue; 26 | import java.util.concurrent.atomic.AtomicInteger; 27 | 28 | import org.apache.log4j.AppenderSkeleton; 29 | import org.apache.log4j.ConsoleAppender; 30 | import org.apache.log4j.Layout; 31 | import org.apache.log4j.Logger; 32 | import org.apache.log4j.PatternLayout; 33 | import org.apache.log4j.spi.LoggingEvent; 34 | 35 | public class Log { 36 | 37 | public static Logger logger; 38 | public static LogAppender logAppender; 39 | 40 | public static void init() { 41 | PatternLayout layout = new PatternLayout("%d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %p %c %x - %m%n"); 42 | logger = Logger.getRootLogger(); 43 | logger.removeAllAppenders(); 44 | logAppender = new LogAppender(layout, 100000); 45 | logger.addAppender(logAppender); 46 | ConsoleAppender ca = new ConsoleAppender(layout); 47 | ca.setImmediateFlush(false); 48 | logger.addAppender(ca); 49 | 50 | // check network situation 51 | try { 52 | Log.logger.info("Local Host Address: " + InetAddress.getLocalHost().getHostAddress()); 53 | } catch (UnknownHostException e1) { 54 | e1.printStackTrace(); 55 | } 56 | } 57 | 58 | public static void clearCaches() { 59 | // should i.e. be called in case of short memory status 60 | logAppender.clean(5000); 61 | } 62 | 63 | public static void close() { 64 | 65 | } 66 | 67 | public static class LogAppender extends AppenderSkeleton { 68 | 69 | private int maxlines; 70 | private ConcurrentLinkedQueue lines; 71 | private AtomicInteger a; 72 | 73 | public LogAppender(Layout layout, int maxlines) { 74 | this.layout = layout; 75 | this.maxlines = maxlines; 76 | this.lines = new ConcurrentLinkedQueue<>(); 77 | String line = layout.getHeader(); 78 | if (line != null) this.lines.add(line); 79 | this.a = new AtomicInteger(0); 80 | } 81 | 82 | @Override 83 | public void append(LoggingEvent event) { 84 | if (event == null) return; 85 | String line = this.layout.format(event); 86 | if (line != null) this.lines.add(line); 87 | if (event.getThrowableInformation() != null) { 88 | for (String t: event.getThrowableStrRep()) if (t != null) this.lines.add(t + "\n"); 89 | } 90 | if (this.a.incrementAndGet() % 100 == 0) { 91 | clean(this.maxlines); 92 | this.a.set(0); 93 | } 94 | } 95 | 96 | @Override 97 | public void close() { 98 | lines.clear(); 99 | lines = null; 100 | } 101 | 102 | @Override 103 | public boolean requiresLayout() { 104 | return true; 105 | } 106 | 107 | public ArrayList getLines(int max) { 108 | Object[] a = this.lines.toArray(); 109 | ArrayList l = new ArrayList<>(); 110 | int start = Math.max(0, a.length - max); 111 | for (int i = start; i < a.length; i++) l.add((String) a[i]); 112 | return l; 113 | } 114 | 115 | public void clean(int remaining) { 116 | int c = this.lines.size() - remaining; 117 | while (c-- > 0) this.lines.poll(); 118 | } 119 | 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/tools/ComparableARC.java: -------------------------------------------------------------------------------- 1 | /** 2 | * ComparableARC 3 | * an Adaptive Replacement Cache for comparable objects 4 | * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany 5 | * First released 24.08.2010 at http://yacy.net 6 | * 7 | * 8 | * This library is free software; you can redistribute it and/or 9 | * modify it under the terms of the GNU Lesser General Public 10 | * License as published by the Free Software Foundation; either 11 | * version 2.1 of the License, or (at your option) any later version. 12 | * 13 | * This library is distributed in the hope that it will be useful, 14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 | * Lesser General Public License for more details. 17 | * 18 | * You should have received a copy of the GNU Lesser General Public License 19 | * along with this program in the file lgpl21.txt 20 | * If not, see . 21 | */ 22 | 23 | package net.yacy.grid.tools; 24 | 25 | import java.util.Comparator; 26 | import java.util.Iterator; 27 | import java.util.LinkedList; 28 | import java.util.Map; 29 | import java.util.TreeMap; 30 | 31 | public final class ComparableARC extends SimpleARC implements Map, Iterable>, ARC { 32 | 33 | 34 | public ComparableARC(final int cacheSize, Comparator comparator) { 35 | super.cacheSize = cacheSize / 2; 36 | super.levelA = new LimitedTreeMap(this.cacheSize, comparator); 37 | super.levelB = new LimitedTreeMap(this.cacheSize, comparator); 38 | } 39 | 40 | private static class LimitedTreeMap extends TreeMap { 41 | private static final long serialVersionUID = -2276429187676080820L; 42 | int limit; 43 | LinkedList keys; 44 | public LimitedTreeMap(final int cacheSize, Comparator comparator) { 45 | super(comparator); 46 | this.limit = cacheSize; 47 | this.keys = new LinkedList(); 48 | } 49 | @Override 50 | public synchronized V get(Object k) { 51 | return super.get(k); 52 | } 53 | @Override 54 | public synchronized V put(K k, V v) { 55 | V r = super.put(k, v); 56 | if (r == null) keys.add(k); 57 | if (keys.size() > this.limit) { 58 | K w = this.keys.removeFirst(); 59 | assert w != null; 60 | V t = super.remove(w); 61 | assert t != null : "keys.size() = " + keys.size() + ", limit = " + this.limit; 62 | } 63 | return r; 64 | } 65 | @Override 66 | public void putAll(Map map) { 67 | for (Map.Entry entry: map.entrySet()) put(entry.getKey(), entry.getValue()); 68 | } 69 | @Override 70 | public synchronized V remove(Object k) { 71 | V r = super.remove(k); 72 | if (r == null) return null; 73 | @SuppressWarnings("unchecked") 74 | boolean removed = removeFromKeys((K) k); 75 | assert removed; 76 | return r; 77 | } 78 | @Override 79 | public synchronized Map.Entry pollFirstEntry() { 80 | Map.Entry entry = super.pollFirstEntry(); 81 | boolean removed = removeFromKeys(entry.getKey()); 82 | assert removed; 83 | return entry; 84 | } 85 | @Override 86 | public synchronized Map.Entry pollLastEntry() { 87 | Map.Entry entry = super.pollLastEntry(); 88 | boolean removed = removeFromKeys(entry.getKey()); 89 | assert removed; 90 | return entry; 91 | } 92 | @Override 93 | public synchronized void clear() { 94 | super.clear(); 95 | this.keys.clear(); 96 | } 97 | private boolean removeFromKeys(K k) { 98 | assert k != null; 99 | Iterator i = keys.iterator(); 100 | K x; 101 | while (i.hasNext()) { 102 | x = i.next(); 103 | if (super.comparator().compare(k, x) == 0) { 104 | i.remove(); 105 | return true; 106 | } 107 | } 108 | return false; 109 | } 110 | } 111 | 112 | } 113 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/tools/KeyList.java: -------------------------------------------------------------------------------- 1 | /** 2 | * KeyList 3 | * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany 4 | * First released 18.4.2011 at http://yacy.net 5 | * 6 | * This library is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU Lesser General Public 8 | * License as published by the Free Software Foundation; either 9 | * version 2.1 of the License, or (at your option) any later version. 10 | * 11 | * This library is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Lesser General Public License 17 | * along with this program in the file lgpl21.txt 18 | * If not, see . 19 | */ 20 | 21 | package net.yacy.grid.tools; 22 | 23 | import java.io.BufferedReader; 24 | import java.io.File; 25 | import java.io.FileInputStream; 26 | import java.io.IOException; 27 | import java.io.InputStream; 28 | import java.io.InputStreamReader; 29 | import java.io.RandomAccessFile; 30 | import java.nio.charset.StandardCharsets; 31 | import java.util.Iterator; 32 | import java.util.Map; 33 | import java.util.concurrent.ConcurrentHashMap; 34 | import java.util.zip.GZIPInputStream; 35 | 36 | import net.yacy.grid.http.Log; 37 | 38 | /** 39 | * a key list is a file which contains a list of key words; each line one word 40 | * The key list is stored into a java set object and the list can be extended on the fly 41 | * which is done by extending the file with just another line. 42 | * When is key list file is initialized, all lines are read and pushed into a java set 43 | */ 44 | public class KeyList implements Iterable { 45 | 46 | private static final Object _obj = new Object(); 47 | 48 | private final Map keys; 49 | private final RandomAccessFile raf; 50 | 51 | public KeyList(final File file) throws IOException { 52 | this.keys = new ConcurrentHashMap(); 53 | 54 | if (file.exists()) { 55 | InputStream is = new FileInputStream(file); 56 | if (file.getName().endsWith(".gz")) { 57 | is = new GZIPInputStream(is); 58 | } 59 | final BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)); 60 | String l; 61 | try { 62 | while ((l = reader.readLine()) != null) { 63 | if (l.isEmpty() || l.charAt(0) == '#') continue; 64 | l = l.trim().toLowerCase(); 65 | this.keys.put(l, _obj); 66 | } 67 | } catch (final IOException e) { 68 | // finish 69 | } finally { 70 | reader.close(); 71 | } 72 | } 73 | 74 | this.raf = new RandomAccessFile(file, "rw"); 75 | 76 | } 77 | 78 | public void clear() throws IOException { 79 | this.raf.setLength(0); 80 | this.keys.clear(); 81 | } 82 | 83 | public int size() { 84 | return this.keys.size(); 85 | } 86 | 87 | public boolean contains(final String key) { 88 | return this.keys.containsKey(key.trim().toLowerCase()); 89 | } 90 | 91 | public void add(final String key) throws IOException { 92 | if (this.keys.containsKey(key)) return; 93 | synchronized (this.raf) { 94 | if (this.keys.containsKey(key)) return; // check again for those threads who come late (after another has written this) 95 | this.keys.put(key, _obj); 96 | this.raf.seek(this.raf.length()); 97 | this.raf.write(key.getBytes(StandardCharsets.UTF_8)); 98 | this.raf.writeByte('\n'); 99 | } 100 | } 101 | 102 | public synchronized void close() throws IOException { 103 | synchronized (this.raf) { 104 | this.raf.close(); 105 | } 106 | } 107 | 108 | public static void main(final String[] args) { 109 | try { 110 | final KeyList kl = new KeyList(new File("/tmp/test")); 111 | kl.add("eins"); 112 | kl.add("zwei"); 113 | kl.add("drei"); 114 | System.out.println(kl.contains("eins") ? "drin" : "nicht"); 115 | } catch (final IOException e) { 116 | Log.logger.warn("", e); 117 | } 118 | 119 | } 120 | 121 | @Override 122 | public Iterator iterator() { 123 | return this.keys.keySet().iterator(); 124 | } 125 | 126 | } 127 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/json/JsonRandomAccessFile.java: -------------------------------------------------------------------------------- 1 | /** 2 | * JsonRandomAccessFile 3 | * Copyright 2015 by Michael Peter Christen 4 | * First released 04.10.2015 5 | * 6 | * This library is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU Lesser General Public 8 | * License as published by the Free Software Foundation; either 9 | * version 2.1 of the License, or (at your option) any later version. 10 | * 11 | * This library is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Lesser General Public License 17 | * along with this program in the file lgpl21.txt 18 | * If not, see . 19 | */ 20 | 21 | 22 | package net.yacy.grid.json; 23 | 24 | import java.io.File; 25 | import java.io.IOException; 26 | import java.nio.charset.StandardCharsets; 27 | import java.util.concurrent.ArrayBlockingQueue; 28 | 29 | import org.json.JSONObject; 30 | 31 | import net.yacy.grid.http.Log; 32 | import net.yacy.grid.tools.BufferedRandomAccessFile; 33 | 34 | public class JsonRandomAccessFile extends BufferedRandomAccessFile { 35 | 36 | private File file; 37 | private int concurrency; 38 | private ArrayBlockingQueue jsonline; 39 | 40 | /** 41 | * if a JsonRandomAccessFile object in initiated, it must be wrapped with a Thread object and started. 42 | * @param dumpFile 43 | * @param concurrency 44 | * @throws IOException 45 | */ 46 | public JsonRandomAccessFile(final File dumpFile, final int concurrency) throws IOException { 47 | super(dumpFile, "rw", 1 << 14); 48 | this.file = dumpFile; 49 | this.concurrency = concurrency; 50 | this.jsonline = new ArrayBlockingQueue<>(1000); 51 | } 52 | 53 | public String getName() { 54 | return this.file.getAbsolutePath(); 55 | } 56 | 57 | public int getConcurrency() { 58 | return this.concurrency; 59 | } 60 | 61 | public JsonFactory take() throws InterruptedException { 62 | return this.jsonline.take(); 63 | } 64 | 65 | /** 66 | * The JsonHandle class is a bundle of a json with the information about the 67 | * seek location in the file and the length of bytes of the original json string 68 | */ 69 | public static class JsonHandle implements JsonFactory { 70 | private JSONObject json; 71 | private long index; 72 | private int length; 73 | public JsonHandle(JSONObject json, long index, int length) { 74 | this.json = json; 75 | this.index = index; 76 | this.length = length; 77 | } 78 | public JSONObject getJSON() { 79 | return json; 80 | } 81 | public long getIndex() { 82 | return index; 83 | } 84 | public int getLength() { 85 | return length; 86 | } 87 | public String toString() { 88 | return new JSONObject(this.json).toString(); 89 | } 90 | } 91 | 92 | public JsonFactory getJsonFactory(long index, int length) { 93 | return new ReaderJsonFactory(index, length); 94 | } 95 | 96 | public class ReaderJsonFactory implements JsonFactory { 97 | 98 | private long index; 99 | private int length; 100 | 101 | public ReaderJsonFactory(long index, int length) { 102 | this.index = index; 103 | this.length = length; 104 | } 105 | 106 | @Override 107 | public JSONObject getJSON() throws IOException { 108 | byte[] b = new byte[this.length]; 109 | JsonRandomAccessFile.this.read(b, this.index); 110 | return new JSONObject(new String(b, 0, b.length, StandardCharsets.UTF_8)); 111 | } 112 | public long getIndex() { 113 | return this.index; 114 | } 115 | public int getLength() { 116 | return this.length; 117 | } 118 | public File getFile() { 119 | return JsonRandomAccessFile.this.file; 120 | } 121 | public String toString() { 122 | try { 123 | return this.getJSON().toString(); 124 | } catch (IOException e) { 125 | Log.logger.error("", e); 126 | return ""; 127 | } 128 | } 129 | } 130 | 131 | public void close() throws IOException { 132 | super.close(); 133 | } 134 | 135 | } 136 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/tools/TimeoutMatcher.java: -------------------------------------------------------------------------------- 1 | /** 2 | * TimeoutMatcher 3 | * Copyright 13.03.2017 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.tools; 21 | 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | import java.util.Map; 25 | import java.util.concurrent.Callable; 26 | import java.util.concurrent.ConcurrentHashMap; 27 | import java.util.concurrent.ExecutionException; 28 | import java.util.concurrent.ExecutorService; 29 | import java.util.concurrent.Future; 30 | import java.util.concurrent.SynchronousQueue; 31 | import java.util.concurrent.ThreadPoolExecutor; 32 | import java.util.concurrent.TimeUnit; 33 | import java.util.concurrent.TimeoutException; 34 | import java.util.regex.Matcher; 35 | 36 | import net.yacy.grid.http.Log; 37 | 38 | /** 39 | * A timeout matcher is a workaround to non-terminating matcher methods. 40 | * Such non-terminating matcher exist as a bug inside java, as documented 41 | * in https://bugs.openjdk.java.net/browse/JDK-8158615 42 | * Since the bugfix is only available in Java 9, we need a permanent patch 43 | * arount that. All matcher calls must use this method to ensure termination 44 | * of matchers. 45 | */ 46 | public class TimeoutMatcher { 47 | 48 | private final static ExecutorService EXECUTOR = new ThreadPoolExecutor( 49 | 0, 1000, 10L, TimeUnit.SECONDS, new SynchronousQueue()); 50 | private final static Map computing = new ConcurrentHashMap<>(); 51 | 52 | private final Matcher matcher; 53 | 54 | public TimeoutMatcher(Matcher matcher) { 55 | this.matcher = matcher; 56 | } 57 | 58 | public boolean matches() { 59 | Future future = EXECUTOR.submit(new Callable(){ 60 | @Override 61 | public Boolean call() throws Exception { 62 | Thread t = Thread.currentThread(); 63 | t.setName("TimeoutMatcher: '" + TimeoutMatcher.this.matcher.pattern() + "'"); 64 | computing.put(t.getId(), t); 65 | boolean matches = false; 66 | try { 67 | matches = TimeoutMatcher.this.matcher.matches(); 68 | } catch (Throwable e) {} finally { 69 | computing.remove(t.getId()); 70 | } 71 | return matches; 72 | } 73 | }); 74 | try { 75 | return future.get(1, TimeUnit.SECONDS); 76 | } catch (InterruptedException | ExecutionException | TimeoutException e) { 77 | future.cancel(true); 78 | return false; 79 | } 80 | } 81 | 82 | public boolean find() { 83 | Future future = EXECUTOR.submit(new Callable(){ 84 | @Override 85 | public Boolean call() throws Exception { 86 | Thread t = Thread.currentThread(); 87 | t.setName("TimeoutMatcher: '" + TimeoutMatcher.this.matcher.pattern() + "'"); 88 | computing.put(t.getId(), t); 89 | boolean find = false; 90 | try { 91 | find = TimeoutMatcher.this.matcher.find(); 92 | } catch (Throwable e) {} finally { 93 | computing.remove(t.getId()); 94 | } 95 | return find; 96 | } 97 | }); 98 | try { 99 | return future.get(1, TimeUnit.SECONDS); 100 | } catch (InterruptedException | ExecutionException | TimeoutException e) { 101 | return false; 102 | } 103 | } 104 | 105 | public static void terminateAll() { 106 | List deleteIds = new ArrayList<>(); 107 | computing.values().forEach(job -> { 108 | if (job.isAlive()) try { 109 | job.interrupt(); 110 | Log.logger.info("interrupting TimeoutMatcher: '" + job.getName() + "'"); 111 | } catch (Throwable e) {} 112 | else deleteIds.add(job.getId()); 113 | }); 114 | deleteIds.forEach(id -> computing.remove(id)); 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/tools/MapUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * MapUtil 3 | * Copyright 02.10.2016 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.tools; 21 | 22 | import java.io.BufferedWriter; 23 | import java.io.File; 24 | import java.io.FileInputStream; 25 | import java.io.FileWriter; 26 | import java.io.IOException; 27 | import java.util.Collections; 28 | import java.util.Comparator; 29 | import java.util.HashMap; 30 | import java.util.LinkedHashMap; 31 | import java.util.LinkedList; 32 | import java.util.List; 33 | import java.util.Map; 34 | import java.util.Properties; 35 | 36 | public class MapUtil { 37 | 38 | /** 39 | * sort a Map on the values. This algorithm is taken from 40 | * http://stackoverflow.com/questions/109383/sort-a-mapkey-value-by-values-java 41 | * provided by tunaki http://stackoverflow.com/users/1743880/tunaki 42 | * It was extended with a 'asc' attribute to select ascending (asc = true) 43 | * or descending (desc) order. 44 | * @param map 45 | * @param asc 46 | * @return the map, sorted by the value in ascending or descending order (according to asc) 47 | */ 48 | public static > Map sortByValue(final Map map, final boolean asc) { 49 | 50 | // put the given map into a list with Map.Entry objects 51 | List> list = new LinkedList>(map.entrySet()); 52 | // declare a sort method for such kind of lists and sort it. 53 | Collections.sort(list, new Comparator>() { 54 | @Override 55 | public int compare(Map.Entry o1, Map.Entry o2) { 56 | return asc ? (o1.getValue()).compareTo(o2.getValue()) : (o2.getValue()).compareTo(o1.getValue()); 57 | } 58 | }); 59 | 60 | // create a result map and put the ordered list inside. 61 | // The map is a linked map which has a defined object order 62 | Map result = new LinkedHashMap(); 63 | for (Map.Entry entry : list) { 64 | result.put(entry.getKey(), entry.getValue()); 65 | } 66 | return result; 67 | } 68 | 69 | /** 70 | * read a configuration file and overload it with custom configurations 71 | * @param conf_dir the path where the default configuration file is stored 72 | * @param user_dir the path where the user writes a custom configuration 73 | * @param confFileName the name of the configuration file 74 | * @return a map from the properties in the configuration 75 | * @throws IOException 76 | */ 77 | public static Map readConfig(File conf_dir, File user_dir, String confFileName) { 78 | Properties prop = new Properties(); 79 | File firstTry = new File(conf_dir, confFileName); 80 | if (firstTry.exists()) try { 81 | prop.load(new FileInputStream(firstTry)); 82 | } catch (IOException e) { 83 | e.printStackTrace(); 84 | } 85 | Map config = new HashMap<>(); 86 | for (Map.Entry entry: prop.entrySet()) config.put((String) entry.getKey(), (String) entry.getValue()); 87 | user_dir.mkdirs(); 88 | File customized_config = new File(user_dir, confFileName); 89 | if (!customized_config.exists()) try { 90 | BufferedWriter w = new BufferedWriter(new FileWriter(customized_config)); 91 | w.write("# This file can be used to customize the configuration\n"); 92 | w.close(); 93 | } catch (IOException e) { 94 | e.printStackTrace(); 95 | } 96 | Properties customized_config_props = new Properties(); 97 | try { 98 | customized_config_props.load(new FileInputStream(customized_config)); 99 | } catch (IOException e) { 100 | e.printStackTrace(); 101 | } 102 | for (Map.Entry entry: customized_config_props.entrySet()) config.put((String) entry.getKey(), (String) entry.getValue()); 103 | return config; 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/tools/Memory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Memory 3 | * Copyright 14.01.2017 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.tools; 21 | 22 | import java.lang.management.ManagementFactory; 23 | import java.lang.management.OperatingSystemMXBean; 24 | import java.lang.management.ThreadInfo; 25 | import java.lang.management.ThreadMXBean; 26 | 27 | import net.yacy.grid.http.Log; 28 | 29 | public class Memory { 30 | 31 | private static final Runtime runtime = Runtime.getRuntime(); 32 | public final static OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean(); 33 | public final static ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); 34 | public final static float shortmemthreshold = 0.9f; 35 | 36 | /** 37 | * memory that is free without increasing of total memory taken from os 38 | * @return bytes 39 | */ 40 | public static final long free() { 41 | return runtime.freeMemory(); 42 | } 43 | 44 | /** 45 | * memory that is available including increasing total memory up to maximum 46 | * @return bytes 47 | */ 48 | public static final long available() { 49 | return assigned() - total() + free(); 50 | } 51 | 52 | /** 53 | * maximum memory the Java virtual will allocate machine; may vary over time in some cases 54 | * @return bytes 55 | */ 56 | public static final long assigned() { 57 | return runtime.maxMemory(); // can be Long.MAX_VALUE if unlimited 58 | } 59 | 60 | /** 61 | * currently allocated memory in the Java virtual machine; may vary over time 62 | * @return bytes 63 | */ 64 | public static final long total() { 65 | return runtime.totalMemory(); 66 | } 67 | 68 | /** 69 | * memory that is currently bound in objects 70 | * @return used bytes 71 | */ 72 | public static final long used() { 73 | return total() - free(); 74 | } 75 | 76 | /** 77 | * get number of CPU cores 78 | * @return number of CPU cores 79 | */ 80 | public static final long cores() { 81 | return runtime.availableProcessors(); 82 | } 83 | 84 | /** 85 | * get the system load within the last minute 86 | * @return the system load or a negative number if the load is not available 87 | */ 88 | public static double load() { 89 | return osBean.getSystemLoadAverage(); 90 | } 91 | 92 | /** 93 | * find out the number of thread deadlocks. WARNING: this is a time-consuming task 94 | * @return the number of deadlocked threads 95 | */ 96 | public static long deadlocks() { 97 | long[] deadlockIDs = threadBean.findDeadlockedThreads(); 98 | if (deadlockIDs == null) return 0; 99 | return deadlockIDs.length; 100 | } 101 | 102 | /** 103 | * write deadlocked threads as to the log as warning 104 | */ 105 | public static void logDeadlocks() { 106 | long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads(); 107 | if (deadlockIDs == null) return; 108 | ThreadInfo[] infos = ManagementFactory.getThreadMXBean().getThreadInfo(deadlockIDs, true, true); 109 | for (ThreadInfo ti : infos) { 110 | Log.logger.warn("DEADLOCKREPORT: " + ti.toString()); 111 | } 112 | } 113 | 114 | private static long lastGCtime = 0; 115 | 116 | /** 117 | * run garbage collection 118 | * @return true if gc was actually triggered, false if no gc was done because time to latest gc was too short 119 | */ 120 | public final synchronized static boolean lazyGC() { 121 | long now = System.currentTimeMillis(); 122 | if (now - lastGCtime > 10000) { 123 | System.gc(); 124 | lastGCtime = now; 125 | return true; 126 | } 127 | return false; 128 | } 129 | 130 | /** 131 | * @return if last request failed 132 | */ 133 | public static boolean shortStatus() { 134 | lazyGC(); 135 | return used() >= assigned() * shortmemthreshold ; 136 | } 137 | 138 | } 139 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/tools/Swagger.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Swagger 3 | * Copyright 14.01.2017 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.tools; 21 | 22 | import java.io.File; 23 | import java.io.FileInputStream; 24 | import java.io.FileNotFoundException; 25 | import java.io.InputStreamReader; 26 | import java.io.UnsupportedEncodingException; 27 | import java.util.LinkedHashMap; 28 | import java.util.Map; 29 | import java.util.Set; 30 | 31 | import org.json.JSONArray; 32 | import org.json.JSONObject; 33 | import org.json.JSONTokener; 34 | 35 | public class Swagger extends JSONObject { 36 | 37 | public static enum Type { 38 | array, object, string, integer 39 | } 40 | 41 | public Swagger(File f) throws UnsupportedEncodingException, FileNotFoundException { 42 | super(new JSONTokener(new InputStreamReader(new FileInputStream(f), "UTF-8"))); 43 | } 44 | 45 | public String getBasePath() { 46 | return this.getString("basePath"); 47 | } 48 | 49 | public Set getServlets() { 50 | return this.getJSONObject("paths").keySet(); 51 | } 52 | 53 | public Servlet getServlet(String path) { 54 | return new Servlet(path, this.getJSONObject("paths").getJSONObject(path)); 55 | } 56 | 57 | public class Servlet extends JSONObject { 58 | private String path; 59 | private String method; 60 | private Map parameters; 61 | private JSONObject responses; 62 | public Servlet(String path, JSONObject o) { 63 | this.putAll(o); 64 | this.path = path; 65 | this.method = this.keys().next(); 66 | JSONArray parameterArray = this.getJSONObject(this.method).getJSONArray("parameters"); 67 | this.parameters = new LinkedHashMap<>(); 68 | for (int i = 0; i < parameterArray.length(); i++) { 69 | JSONObject p = parameterArray.getJSONObject(i); 70 | this.parameters.put(p.getString("name"), p); 71 | } 72 | this.responses = this.getJSONObject(this.method).getJSONObject("responses").getJSONObject("200"); 73 | } 74 | public String getPath() { 75 | return this.path; 76 | } 77 | public String getMethod() { 78 | return this.method; 79 | } 80 | public Set getParameterNames() { 81 | return this.parameters.keySet(); 82 | } 83 | public Type getParameterType(String paramName) { 84 | return Type.valueOf(this.parameters.get(paramName).getString("type")); 85 | } 86 | public Type getResponseType() { 87 | return Type.valueOf(this.responses.getJSONObject("schema").getString("type")); 88 | } 89 | public Definition getResponseDefinition() { 90 | String ref = this.responses.getJSONObject("schema").getJSONObject("items").getString("$ref"); 91 | int p = ref.lastIndexOf('/'); 92 | return getDefinition(ref.substring(p + 1)); 93 | } 94 | public String toString() { 95 | return getPath() + getParameterNames() + " -> " + getResponseDefinition().toString(); 96 | } 97 | } 98 | 99 | public Definition getDefinition(String definitionName) { 100 | return new Definition(definitionName, this.getJSONObject("definitions").getJSONObject(definitionName)); 101 | } 102 | 103 | public static class Definition extends JSONObject { 104 | private String name; 105 | public Definition(String name, JSONObject o) { 106 | this.putAll(o); 107 | this.name = name; 108 | } 109 | 110 | public String getName() { 111 | return this.name; 112 | } 113 | 114 | public Type getType() { 115 | return Type.valueOf(this.getString("type")); 116 | } 117 | 118 | public Set getPropertyNames() { 119 | return this.getJSONObject("properties").keySet(); 120 | } 121 | 122 | public Type getPropertyType(String propertyName) { 123 | return Type.valueOf(this.getJSONObject("properties").getString(propertyName)); 124 | } 125 | 126 | public String toString() { 127 | return getName() + "[" + this.getType() + "] == " + getPropertyNames().toString(); 128 | } 129 | } 130 | 131 | } 132 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/http/ServiceResponse.java: -------------------------------------------------------------------------------- 1 | /** 2 | * ServiceResponse 3 | * Copyright 16.01.2017 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.http; 21 | 22 | import java.nio.charset.StandardCharsets; 23 | 24 | import org.json.JSONArray; 25 | import org.json.JSONException; 26 | import org.json.JSONObject; 27 | 28 | import net.yacy.grid.graphics.RasterPlotter; 29 | 30 | /** 31 | * wrapper class for the result type of services: either 32 | * JSONObject, JSONArray, String, or byte[] 33 | */ 34 | public class ServiceResponse { 35 | 36 | private Object object; 37 | private boolean setCORS; 38 | private boolean isImage; 39 | 40 | public ServiceResponse(JSONObject json) { 41 | this.object = json; 42 | this.setCORS = false; 43 | this.isImage = false; 44 | } 45 | 46 | public ServiceResponse(JSONArray json) { 47 | this.object = json; 48 | this.setCORS = false; 49 | this.isImage = false; 50 | } 51 | 52 | public ServiceResponse(String string) { 53 | this.object = string; 54 | this.setCORS = false; 55 | this.isImage = false; 56 | } 57 | 58 | public ServiceResponse(byte[] bytes) { 59 | this.object = bytes; 60 | this.setCORS = false; 61 | this.isImage = false; 62 | } 63 | 64 | public ServiceResponse(RasterPlotter raster) { 65 | this.object = raster; 66 | this.setCORS = true; 67 | this.isImage = true; 68 | } 69 | 70 | public ServiceResponse setCORS() { 71 | this.setCORS = true; 72 | return this; 73 | } 74 | 75 | public boolean allowCORS() { 76 | return this.setCORS; 77 | } 78 | public boolean isObject() { 79 | return this.object instanceof JSONObject; 80 | } 81 | 82 | public boolean isArray() { 83 | return this.object instanceof JSONArray; 84 | } 85 | 86 | public boolean isString() { 87 | return this.object instanceof String; 88 | } 89 | 90 | public boolean isByteArray() { 91 | return this.object instanceof byte[]; 92 | } 93 | 94 | public boolean isImage() { 95 | return this.isImage; 96 | } 97 | 98 | public JSONObject getObject() throws JSONException { 99 | if (!isObject()) throw new JSONException("object type is not JSONObject: " + this.object.getClass().getName()); 100 | return (JSONObject) this.object; 101 | } 102 | 103 | public JSONArray getArray() throws JSONException { 104 | if (!isArray()) throw new JSONException("object type is not JSONArray: " + this.object.getClass().getName()); 105 | return (JSONArray) this.object; 106 | } 107 | 108 | public String getString() throws JSONException { 109 | if (!isString()) throw new JSONException("object type is not String: " + this.object.getClass().getName()); 110 | return (String) this.object; 111 | } 112 | 113 | public byte[] getByteArray() throws JSONException { 114 | if (!isByteArray()) throw new JSONException("object type is not ByteArray: " + this.object.getClass().getName()); 115 | return (byte[]) this.object; 116 | } 117 | 118 | public RasterPlotter getImage() { 119 | return (RasterPlotter) this.object; 120 | } 121 | 122 | public String getMimeType() { 123 | if (isObject() || isArray()) return "application/javascript"; 124 | if (isString()) { 125 | return getString().startsWith(". 18 | */ 19 | 20 | package net.yacy.grid.http; 21 | 22 | import java.io.IOException; 23 | import java.net.URLEncoder; 24 | import java.nio.charset.Charset; 25 | import java.nio.charset.StandardCharsets; 26 | import java.util.HashMap; 27 | import java.util.Iterator; 28 | import java.util.Map; 29 | 30 | import org.json.JSONArray; 31 | import org.json.JSONObject; 32 | import org.json.JSONTokener; 33 | 34 | public abstract class ObjectAPIHandler extends AbstractAPIHandler implements APIHandler { 35 | 36 | private static final long serialVersionUID = -2191240526448018368L; 37 | 38 | // generic keywords in JSON keys of API responses 39 | public static final String SUCCESS_KEY = "success"; 40 | public static final String SERVICE_KEY = "service"; 41 | public static final String COMMENT_KEY = "comment"; 42 | 43 | // for messages 44 | public static final String MESSAGE_KEY = "message"; 45 | public static final String DELIVERY_TAG = "deliveryTag"; 46 | public static final String AVAILABLE_KEY = "available"; 47 | 48 | /** 49 | * helper method to implement serviceImpl 50 | * @param params 51 | * @return 52 | * @throws IOException 53 | */ 54 | public static String json2url(final JSONObject params) throws IOException { 55 | StringBuilder query = new StringBuilder(); 56 | if (params != null) { 57 | Iterator i = params.keys(); int c = 0; 58 | while (i.hasNext()) { 59 | String key = i.next(); 60 | query.append(c == 0 ? '?' : '&'); 61 | query.append(key); 62 | query.append('='); 63 | query.append(URLEncoder.encode(params.get(key).toString(), "UTF-8")); 64 | c++; 65 | } 66 | } 67 | return query.toString(); 68 | } 69 | 70 | public static Map json2map(final JSONObject params) throws IOException { 71 | HashMap map = new HashMap<>(); 72 | if (params != null) { 73 | Iterator i = params.keys(); 74 | while (i.hasNext()) { 75 | String key = i.next(); 76 | map.put(key, params.get(key).toString().getBytes(StandardCharsets.UTF_8)); 77 | } 78 | } 79 | return map; 80 | } 81 | 82 | /** 83 | * POST request 84 | */ 85 | public ServiceResponse serviceImpl(final String protocolhostportstub, JSONObject params) throws IOException { 86 | /* 87 | String urlstring = protocolhostportstub + this.getAPIPath() + json2url(params); 88 | ClientConnection connection = new ClientConnection(urlstring); 89 | return doConnection(connection); 90 | */ 91 | String urlstring = protocolhostportstub + this.getAPIPath(); 92 | ClientConnection connection = new ClientConnection(urlstring, json2map(params)); 93 | return doConnection(connection); 94 | } 95 | 96 | /** 97 | * POST request 98 | */ 99 | public ServiceResponse serviceImpl(final String protocolhostportstub, Map params) throws IOException { 100 | String urlstring = protocolhostportstub + this.getAPIPath(); 101 | ClientConnection connection = new ClientConnection(urlstring, params); 102 | return doConnection(connection); 103 | } 104 | 105 | private ServiceResponse doConnection(ClientConnection connection) throws IOException { 106 | Charset charset = connection.getContentType().getCharset(); 107 | String mime = connection.getContentType().getMimeType(); //application/javascript, application/octet-stream 108 | byte[] b = connection.load(); 109 | if (b.length == 0) throw new IOException("response empty"); 110 | if (mime.indexOf("javascript") >= 0) { 111 | if (b.length > 0 && b[0] == (byte) '[') { 112 | JSONArray json = new JSONArray(new JSONTokener(new String(b, charset == null ? StandardCharsets.UTF_8 : charset))); 113 | return new ServiceResponse(json); 114 | } else { 115 | JSONObject json = new JSONObject(new JSONTokener(new String(b, charset == null ? StandardCharsets.UTF_8 : charset))); 116 | return new ServiceResponse(json); 117 | } 118 | } else { 119 | // consider this is binary 120 | return new ServiceResponse(b); 121 | } 122 | } 123 | 124 | } 125 | -------------------------------------------------------------------------------- /src/main/java/org/json/XMLParserConfiguration.java: -------------------------------------------------------------------------------- 1 | package org.json; 2 | /* 3 | Copyright (c) 2002 JSON.org 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | The Software shall be used for Good, not Evil. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | SOFTWARE. 24 | */ 25 | 26 | /** 27 | * Configuration object for the XML parser. 28 | * @author AylwardJ 29 | * 30 | */ 31 | public class XMLParserConfiguration { 32 | /** Original Configuration of the XML Parser. */ 33 | public static final XMLParserConfiguration ORIGINAL = new XMLParserConfiguration(); 34 | /** Original configuration of the XML Parser except that values are kept as strings. */ 35 | public static final XMLParserConfiguration KEEP_STRINGS = new XMLParserConfiguration(true); 36 | /** 37 | * When parsing the XML into JSON, specifies if values should be kept as strings (true), or if 38 | * they should try to be guessed into JSON values (numeric, boolean, string) 39 | */ 40 | public final boolean keepStrings; 41 | /** 42 | * The name of the key in a JSON Object that indicates a CDATA section. Historically this has 43 | * been the value "content" but can be changed. Use null to indicate no CDATA 44 | * processing. 45 | */ 46 | public final String cDataTagName; 47 | /** 48 | * When parsing the XML into JSON, specifies if values with attribute xsi:nil="true" 49 | * should be kept as attribute(false), or they should be converted to null(true) 50 | */ 51 | public final boolean convertNilAttributeToNull; 52 | 53 | /** 54 | * Default parser configuration. Does not keep strings, and the CDATA Tag Name is "content". 55 | */ 56 | public XMLParserConfiguration () { 57 | this(false, "content", false); 58 | } 59 | 60 | /** 61 | * Configure the parser string processing and use the default CDATA Tag Name as "content". 62 | * @param keepStrings true to parse all values as string. 63 | * false to try and convert XML string values into a JSON value. 64 | */ 65 | public XMLParserConfiguration (final boolean keepStrings) { 66 | this(keepStrings, "content", false); 67 | } 68 | 69 | /** 70 | * Configure the parser string processing to try and convert XML values to JSON values and 71 | * use the passed CDATA Tag Name the processing value. Pass null to 72 | * disable CDATA processing 73 | * @param cDataTagNamenull to disable CDATA processing. Any other value 74 | * to use that value as the JSONObject key name to process as CDATA. 75 | */ 76 | public XMLParserConfiguration (final String cDataTagName) { 77 | this(false, cDataTagName, false); 78 | } 79 | 80 | /** 81 | * Configure the parser to use custom settings. 82 | * @param keepStrings true to parse all values as string. 83 | * false to try and convert XML string values into a JSON value. 84 | * @param cDataTagNamenull to disable CDATA processing. Any other value 85 | * to use that value as the JSONObject key name to process as CDATA. 86 | */ 87 | public XMLParserConfiguration (final boolean keepStrings, final String cDataTagName) { 88 | this.keepStrings = keepStrings; 89 | this.cDataTagName = cDataTagName; 90 | this.convertNilAttributeToNull = false; 91 | } 92 | 93 | /** 94 | * Configure the parser to use custom settings. 95 | * @param keepStrings true to parse all values as string. 96 | * false to try and convert XML string values into a JSON value. 97 | * @param cDataTagName null to disable CDATA processing. Any other value 98 | * to use that value as the JSONObject key name to process as CDATA. 99 | * @param convertNilAttributeToNull true to parse values with attribute xsi:nil="true" as null. 100 | * false to parse values with attribute xsi:nil="true" as {"xsi:nil":true}. 101 | */ 102 | public XMLParserConfiguration (final boolean keepStrings, final String cDataTagName, final boolean convertNilAttributeToNull) { 103 | this.keepStrings = keepStrings; 104 | this.cDataTagName = cDataTagName; 105 | this.convertNilAttributeToNull = convertNilAttributeToNull; 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/tools/AnchorURL.java: -------------------------------------------------------------------------------- 1 | /** 2 | * AnchorURL 3 | * Copyright 2013 by Michael Peter Christen 4 | * first published 15.09.2013 on http://yacy.net 5 | * 6 | * This library is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU Lesser General Public 8 | * License as published by the Free Software Foundation; either 9 | * version 2.1 of the License, or (at your option) any later version. 10 | * 11 | * This library is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Lesser General Public License 17 | * along with this program in the file lgpl21.txt 18 | * If not, see . 19 | */ 20 | 21 | 22 | package net.yacy.grid.tools; 23 | 24 | import java.net.MalformedURLException; 25 | import java.util.Properties; 26 | 27 | public class AnchorURL extends MultiProtocolURL { 28 | 29 | /* 30 | * the 'new' AnchorURL class is a fork of the original yacy.cora.document.id.AnchorURL 31 | * class, but depends on MultiProtocolURL rather than DigestURL, which is removed for 32 | * YaCy Grid. 33 | */ 34 | 35 | private static final long serialVersionUID = 1586579902179962086L; 36 | 37 | private String nameProperty, relProperty, textBody; // may contain additional url properties, such as given in html a href-links 38 | private MultiProtocolURL imageURL; // in case that the anchor contains an image link, store image url; if there is no image then set this to null 39 | private String imageAlt; // in case that the anchor contains an image link, store the alt property; if there is no image then set this to null 40 | 41 | public AnchorURL(final String url) throws MalformedURLException { 42 | super(url); 43 | this.textBody = ""; 44 | this.nameProperty = ""; 45 | this.relProperty = ""; 46 | this.imageURL = null; 47 | this.imageAlt = null; 48 | } 49 | 50 | public AnchorURL(final MultiProtocolURL url) { 51 | super(url); 52 | this.textBody = ""; 53 | this.nameProperty = ""; 54 | this.relProperty = ""; 55 | this.imageURL = null; 56 | this.imageAlt = null; 57 | } 58 | 59 | public AnchorURL(final MultiProtocolURL baseURL, final String relPath) throws MalformedURLException { 60 | super(baseURL, relPath); 61 | this.textBody = ""; 62 | this.nameProperty = ""; 63 | this.relProperty = ""; 64 | this.imageURL = null; 65 | this.imageAlt = null; 66 | } 67 | 68 | public AnchorURL(final String protocol, final String host, final int port, final String path) throws MalformedURLException { 69 | super(protocol, host, port, path); 70 | this.textBody = ""; 71 | this.nameProperty = ""; 72 | this.relProperty = ""; 73 | this.imageURL = null; 74 | this.imageAlt = null; 75 | } 76 | 77 | public AnchorURL(final String protocol, final String host, final int port, final String path, final MultiProtocolURL imageURL, final String imageAlt) throws MalformedURLException { 78 | super(protocol, host, port, path); 79 | this.textBody = ""; 80 | this.nameProperty = ""; 81 | this.relProperty = ""; 82 | this.imageURL = imageURL; 83 | this.imageAlt = imageAlt; 84 | } 85 | 86 | public static AnchorURL newAnchor(final MultiProtocolURL baseURL, String relPath) throws MalformedURLException { 87 | if (relPath.startsWith("//")) { 88 | // patch for urls starting with "//" which can be found in the wild 89 | relPath = (baseURL == null) ? "http:" + relPath : baseURL.getProtocol() + ":" + relPath; 90 | } 91 | if ((baseURL == null) || 92 | isHTTP(relPath) || 93 | isHTTPS(relPath) || 94 | isFTP(relPath) || 95 | isFile(relPath) || 96 | isSMB(relPath) || 97 | relPath.startsWith("mailto:")) { 98 | return new AnchorURL(relPath); 99 | } 100 | return new AnchorURL(baseURL, relPath); 101 | } 102 | 103 | public String getNameProperty() { 104 | return nameProperty; 105 | } 106 | 107 | public void setNameProperty(String name) { 108 | this.nameProperty = name; 109 | } 110 | 111 | public String getTextProperty() { 112 | return textBody; 113 | } 114 | 115 | public void setTextProperty(String text) { 116 | this.textBody = text; 117 | } 118 | 119 | public String getRelProperty() { 120 | return relProperty; 121 | } 122 | 123 | public void setRelProperty(String rel) { 124 | this.relProperty = rel; 125 | } 126 | 127 | public MultiProtocolURL getImageURL() { 128 | return imageURL; 129 | } 130 | 131 | public void setImageURL(MultiProtocolURL imageURL) { 132 | this.imageURL = imageURL; 133 | } 134 | 135 | public String getImageAlt() { 136 | return imageAlt; 137 | } 138 | 139 | public void setImageAlt(String imageAlt) { 140 | this.imageAlt = imageAlt; 141 | } 142 | 143 | public void setAll(final Properties tagopts) { 144 | this.nameProperty = tagopts.getProperty("name", ""); 145 | this.textBody = tagopts.getProperty("text", ""); 146 | this.relProperty = tagopts.getProperty("rel", ""); 147 | } 148 | 149 | public Properties getAll() { 150 | final Properties tagopts = new Properties(); 151 | tagopts.setProperty("name", this.nameProperty); 152 | tagopts.setProperty("text", this.textBody); 153 | tagopts.setProperty("rel", this.relProperty); 154 | return tagopts; 155 | } 156 | 157 | public boolean attachedNofollow() { 158 | return this.relProperty.indexOf("nofollow") >= 0; 159 | } 160 | 161 | public String toHTML() { 162 | return " 0 ? (" name=\"" + this.nameProperty + "\"") : "") + 164 | (this.relProperty.length() > 0 ? (" rel=\"" + this.relProperty + "\"") : "") + 165 | ">" + this.textBody + ""; 166 | } 167 | 168 | } 169 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # 4 | # Copyright 2015 the original author or authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | ## 21 | ## Gradle start up script for UN*X 22 | ## 23 | ############################################################################## 24 | 25 | # Attempt to set APP_HOME 26 | # Resolve links: $0 may be a link 27 | PRG="$0" 28 | # Need this for relative symlinks. 29 | while [ -h "$PRG" ] ; do 30 | ls=`ls -ld "$PRG"` 31 | link=`expr "$ls" : '.*-> \(.*\)$'` 32 | if expr "$link" : '/.*' > /dev/null; then 33 | PRG="$link" 34 | else 35 | PRG=`dirname "$PRG"`"/$link" 36 | fi 37 | done 38 | SAVED="`pwd`" 39 | cd "`dirname \"$PRG\"`/" >/dev/null 40 | APP_HOME="`pwd -P`" 41 | cd "$SAVED" >/dev/null 42 | 43 | APP_NAME="Gradle" 44 | APP_BASE_NAME=`basename "$0"` 45 | 46 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 47 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 48 | 49 | # Use the maximum available, or set MAX_FD != -1 to use that value. 50 | MAX_FD="maximum" 51 | 52 | warn () { 53 | echo "$*" 54 | } 55 | 56 | die () { 57 | echo 58 | echo "$*" 59 | echo 60 | exit 1 61 | } 62 | 63 | # OS specific support (must be 'true' or 'false'). 64 | cygwin=false 65 | msys=false 66 | darwin=false 67 | nonstop=false 68 | case "`uname`" in 69 | CYGWIN* ) 70 | cygwin=true 71 | ;; 72 | Darwin* ) 73 | darwin=true 74 | ;; 75 | MINGW* ) 76 | msys=true 77 | ;; 78 | NONSTOP* ) 79 | nonstop=true 80 | ;; 81 | esac 82 | 83 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 84 | 85 | # Determine the Java command to use to start the JVM. 86 | if [ -n "$JAVA_HOME" ] ; then 87 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 88 | # IBM's JDK on AIX uses strange locations for the executables 89 | JAVACMD="$JAVA_HOME/jre/sh/java" 90 | else 91 | JAVACMD="$JAVA_HOME/bin/java" 92 | fi 93 | if [ ! -x "$JAVACMD" ] ; then 94 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 95 | 96 | Please set the JAVA_HOME variable in your environment to match the 97 | location of your Java installation." 98 | fi 99 | else 100 | JAVACMD="java" 101 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 102 | 103 | Please set the JAVA_HOME variable in your environment to match the 104 | location of your Java installation." 105 | fi 106 | 107 | # Increase the maximum file descriptors if we can. 108 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 109 | MAX_FD_LIMIT=`ulimit -H -n` 110 | if [ $? -eq 0 ] ; then 111 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 112 | MAX_FD="$MAX_FD_LIMIT" 113 | fi 114 | ulimit -n $MAX_FD 115 | if [ $? -ne 0 ] ; then 116 | warn "Could not set maximum file descriptor limit: $MAX_FD" 117 | fi 118 | else 119 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 120 | fi 121 | fi 122 | 123 | # For Darwin, add options to specify how the application appears in the dock 124 | if $darwin; then 125 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 126 | fi 127 | 128 | # For Cygwin or MSYS, switch paths to Windows format before running java 129 | if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then 130 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 131 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 132 | JAVACMD=`cygpath --unix "$JAVACMD"` 133 | 134 | # We build the pattern for arguments to be converted via cygpath 135 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 136 | SEP="" 137 | for dir in $ROOTDIRSRAW ; do 138 | ROOTDIRS="$ROOTDIRS$SEP$dir" 139 | SEP="|" 140 | done 141 | OURCYGPATTERN="(^($ROOTDIRS))" 142 | # Add a user-defined pattern to the cygpath arguments 143 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 144 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 145 | fi 146 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 147 | i=0 148 | for arg in "$@" ; do 149 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 150 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 151 | 152 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 153 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 154 | else 155 | eval `echo args$i`="\"$arg\"" 156 | fi 157 | i=$((i+1)) 158 | done 159 | case $i in 160 | (0) set -- ;; 161 | (1) set -- "$args0" ;; 162 | (2) set -- "$args0" "$args1" ;; 163 | (3) set -- "$args0" "$args1" "$args2" ;; 164 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 165 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 166 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 167 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 168 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 169 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 170 | esac 171 | fi 172 | 173 | # Escape application args 174 | save () { 175 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 176 | echo " " 177 | } 178 | APP_ARGS=$(save "$@") 179 | 180 | # Collect all arguments for the java command, following the shell quoting and substitution rules 181 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 182 | 183 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 184 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 185 | cd "$(dirname "$0")" 186 | fi 187 | 188 | exec "$JAVACMD" "$@" 189 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/json/JsonLDNode.java: -------------------------------------------------------------------------------- 1 | /** 2 | * JSONLDNode 3 | * Copyright 04.06.2018 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.json; 21 | 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | 25 | import org.json.JSONObject; 26 | 27 | /** 28 | * https://json-ld.org/spec/latest/json-ld/#node-objects 29 | * 30 | * A node object represents zero or more properties of a node in the graph serialized by the JSON-LD 31 | * document. A JSON object is a node object if it exists outside of a JSON-LD context and: 32 | * - it is not the top-most JSON object in the JSON-LD document consisting of no other members 33 | * than @graph and @context, 34 | * - it does not contain the @value, @list, or @set keywords, and 35 | * - it is not a graph object. 36 | * 37 | * The properties of a node in a graph may be spread among different node objects within a document. 38 | * When that happens, the keys of the different node objects need to be merged to create the 39 | * properties of the resulting node. 40 | * 41 | * A node object MUST be a JSON object. All keys which are not IRIs, compact IRIs, terms valid in 42 | * the active context, or one of the following keywords (or alias of such a keyword) MUST be ignored 43 | * when processed: 44 | * 45 | * @context, 46 | * @id, 47 | * @graph, 48 | * @nest, 49 | * @type, 50 | * @reverse, or 51 | * @index 52 | * 53 | * 54 | */ 55 | public class JsonLDNode extends JSONObject { 56 | 57 | public static final String CONTEXT = "@context"; 58 | public static final String TYPE = "@type"; 59 | public static final String ID = "@id"; 60 | 61 | public JsonLDNode(String type, String context) { 62 | super(true); 63 | this.put(CONTEXT, context); 64 | this.put(TYPE, type); 65 | } 66 | 67 | 68 | public JsonLDNode(String type) { 69 | super(true); 70 | this.put(TYPE, type); 71 | } 72 | 73 | public JsonLDNode setSubject(String subject) { 74 | this.put(ID, subject); 75 | return this; 76 | } 77 | 78 | public String getSubject() { 79 | return this.optString(ID); 80 | } 81 | 82 | public String getContext() { 83 | return this.optString(CONTEXT); 84 | } 85 | 86 | public String getType() { 87 | return this.optString(TYPE); 88 | } 89 | 90 | public String getVocabulary() { 91 | return getContext() + "/" + getType(); 92 | } 93 | 94 | public JsonLDNode setPredicate(String key, Object value) { 95 | assert value instanceof String || value instanceof JsonLDNode; 96 | this.put(key, value); 97 | return this; 98 | } 99 | 100 | public String getPredicateName(String key) { 101 | return this.getVocabulary() + "#" + key; 102 | } 103 | 104 | public Object getPredicateValue(String key) { 105 | return this.get(key); 106 | } 107 | 108 | public List getPredicates() { 109 | ArrayList predicates = new ArrayList<>(); 110 | this.keySet().forEach(key -> {if (key.charAt(0) != '@') predicates.add(key);}); 111 | return predicates; 112 | } 113 | 114 | public String toRDFTriple() { 115 | return toRDFTriple(this.getSubject(), this.getVocabulary()); 116 | } 117 | private String toRDFTriple(String subject, String vocabulary) { 118 | StringBuilder sb = new StringBuilder(); 119 | for (String predicate: getPredicates()) { 120 | Object value = this.getPredicateValue(predicate); 121 | if (value instanceof String) sb.append("<" + subject + "> <" + vocabulary + "#" + predicate + "> <" + ((String) value) + ">\n"); 122 | if (value instanceof JsonLDNode) sb.append(((JsonLDNode) value).toRDFTriple(subject, vocabulary)); 123 | } 124 | return sb.toString(); 125 | } 126 | 127 | /* 128 | { 129 | "@context": "http://schema.org", 130 | "@type": "Event", 131 | "location": { 132 | "@type": "Place", 133 | "address": { 134 | "@type": "PostalAddress", 135 | "addressLocality": "Denver", 136 | "addressRegion": "CO", 137 | "postalCode": "80209", 138 | "streetAddress": "7 S. Broadway" 139 | }, 140 | "name": "The Hi-Dive" 141 | }, 142 | "name": "Typhoon with Radiation City", 143 | "offers": { 144 | "@type": "Offer", 145 | "price": "13.00", 146 | "priceCurrency": "USD", 147 | "url": "http://www.ticketfly.com/purchase/309433" 148 | }, 149 | "startDate": "2013-09-14T21:30" 150 | } 151 | */ 152 | 153 | public static void main(String[] args) { 154 | JsonLDNode event = new JsonLDNode("Event", "http://schema.org") 155 | .setSubject("http://an.event.home.page.ninja/tomorrow.html") 156 | .setPredicate("name", "Typhoon with Radiation City") 157 | .setPredicate("startDate", "2013-09-14T21:30") 158 | .setPredicate("location", 159 | new JsonLDNode("Place") 160 | .setPredicate("name", "The Hi-Dive") 161 | .setPredicate("address", 162 | new JsonLDNode("PostalAddress") 163 | .setPredicate("addressLocality", "Denver") 164 | .setPredicate("addressRegion", "CO") 165 | .setPredicate("postalCode", "80209") 166 | .setPredicate("streetAddress", "7 S. Broadway") 167 | ) 168 | ) 169 | .setPredicate("offers", 170 | new JsonLDNode("Offer") 171 | .setPredicate("price", "13.00") 172 | .setPredicate("priceCurrency", "USD") 173 | .setPredicate("postalCode", "80209") 174 | .setPredicate("url", "http://www.ticketfly.com/purchase/309433") 175 | ); 176 | System.out.println(event.toString(2)); 177 | System.out.println(event.toRDFTriple()); 178 | } 179 | } 180 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/core/api/vis/MapService.java: -------------------------------------------------------------------------------- 1 | /** 2 | * MarkdownServlet 3 | * Copyright 04.06.2015 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.core.api.vis; 21 | 22 | 23 | import javax.imageio.ImageIO; 24 | import javax.servlet.http.HttpServletResponse; 25 | 26 | import net.yacy.grid.geo.OSMTile; 27 | import net.yacy.grid.graphics.PrintTool; 28 | import net.yacy.grid.graphics.RasterPlotter; 29 | import net.yacy.grid.graphics.RasterPlotter.DrawMode; 30 | import net.yacy.grid.graphics.RasterPlotter.FilterMode; 31 | import net.yacy.grid.http.APIException; 32 | import net.yacy.grid.http.APIHandler; 33 | import net.yacy.grid.http.Log; 34 | import net.yacy.grid.http.ObjectAPIHandler; 35 | import net.yacy.grid.http.Query; 36 | import net.yacy.grid.http.ServiceResponse; 37 | 38 | import java.awt.image.BufferedImage; 39 | import java.io.IOException; 40 | import java.nio.file.FileSystems; 41 | 42 | /** 43 | * Draw a map with a position marker inside. 44 | * http://localhost:8010/yacy/grid/core/vis/map.png?text=Test&mlat=1.28373&mlon=103.84379&zoom=18 45 | */ 46 | public class MapService extends ObjectAPIHandler implements APIHandler { 47 | 48 | private static final long serialVersionUID = -9112326721290824443L; 49 | 50 | @Override 51 | public String getAPIPath() { 52 | return "/yacy/grid/core/vis/map.png"; 53 | } 54 | 55 | @Override 56 | public ServiceResponse serviceImpl(Query post, HttpServletResponse response) throws APIException { 57 | // parse arguments 58 | String text = post.get("text", ""); 59 | boolean uppercase = post.get("uppercase", true); 60 | 61 | int zoom = post.get("zoom", 19); 62 | if (zoom > 19) throw new APIException(503, "zoom must be <= 19"); 63 | double lat = post.get("mlat", 50.11362d); 64 | if (lat < -90.0) throw new APIException(503, "mlat must be >= -90.0"); 65 | if (lat > 90.0) throw new APIException(503, "mlat must be <= 90.0"); 66 | double lon = post.get("mlon", 8.67919d); 67 | if (lon < -180.0) throw new APIException(503, "mlon must be >= -180.0"); 68 | if (lon > 180.0) throw new APIException(503, "mlon must be <= 180.0"); 69 | int width = post.get("width", 512); 70 | if (width > 2560) throw new APIException(503, "width must be <= -2560"); 71 | int height = post.get("height", 256); 72 | if (height > 2560) throw new APIException(503, "height must be <= -2560"); 73 | int tiles_horizontal = (width - 1) / 256 + 1; if (((tiles_horizontal / 2 ) * 2) == tiles_horizontal) tiles_horizontal++; 74 | int tiles_vertical = (height - 1) / 256 + 1; if (((tiles_vertical / 2 ) * 2) == tiles_vertical) tiles_vertical++; 75 | // one tile has the size 256x256 76 | 77 | // compute image 78 | final OSMTile.TileCoordinates coord = new OSMTile.TileCoordinates(lat, lon, zoom); 79 | RasterPlotter map = OSMTile.getCombinedTiles(coord, tiles_horizontal, tiles_vertical); 80 | 81 | // compute the bbox of the map 82 | double north_lat = coord.north_lat - (coord.south_lat - coord.north_lat) * (tiles_vertical / 2), 83 | south_lat = north_lat + (coord.south_lat - coord.north_lat) * tiles_vertical, 84 | west_lon = coord.west_lon - (coord.east_lon - coord.west_lon) * (tiles_horizontal / 2), 85 | east_lon = west_lon + (coord.east_lon - coord.west_lon) * tiles_horizontal; 86 | 87 | // cut away parts of the map if less was wanted 88 | if (map.getHeight() > height || map.getWidth() > width) { 89 | BufferedImage bi = map.getImage(); 90 | // calculate the cut-off size for each side (in pixels) 91 | int xoff = (map.getWidth() - width) / 2; 92 | int yoff = (map.getHeight() - height) / 2; 93 | 94 | // correct the bbox coordinates 95 | double lonoff = (east_lon - west_lon) * xoff / map.getWidth(); 96 | double latoff = (south_lat - north_lat) * yoff / map.getHeight(); 97 | west_lon += lonoff; east_lon -= lonoff; 98 | north_lat += latoff; south_lat -= latoff; 99 | 100 | // cut away equal border parts 101 | bi = bi.getSubimage(xoff, yoff, width, height); 102 | map = new RasterPlotter(width, height, RasterPlotter.DrawMode.MODE_REPLACE, "FFFFFF"); 103 | map.insertBitmap(bi, 0, 0); 104 | } 105 | 106 | // draw message text on map 107 | map.setDrawMode(DrawMode.MODE_SUB); 108 | map.setColor(0xffffff); 109 | if (text.length() > 0) PrintTool.print(map, 6, 12, 0, uppercase ? text.toUpperCase() : text, -1, false, 100); 110 | PrintTool.print(map, map.getWidth() - 6, map.getHeight() - 6, 0, "MADE WITH YACY GRID", 1, false, 50); 111 | 112 | // draw marker on map 113 | int mx = (int) (map.getWidth() * (lon - west_lon) / (east_lon - west_lon)); 114 | int my = (int) (map.getHeight() * (lat - north_lat) / (south_lat - north_lat)); 115 | // PrintTool.print(map, mx, my, 0, "X", 1, 100); 116 | // the marker has a height of 40 pixel and a width of 25 pixel 117 | try { 118 | BufferedImage logo = ImageIO.read(FileSystems.getDefault().getPath("html").resolve("artwork").resolve("marker-red.png").toFile()); 119 | map.insertBitmap(logo, Math.min(map.getWidth() - 25, Math.max(0, mx - 12)), Math.min(map.getHeight() - 40, Math.max(0, my - 40)), FilterMode.FILTER_ANTIALIASING); 120 | } catch (IOException e) { 121 | Log.logger.warn("", e); 122 | } 123 | 124 | // draw copyright notice on map 125 | /* 126 | * copyright notice on OSM Tiles 127 | * According to http://www.openstreetmap.org/copyright/ the (C) of the map tiles is (CC BY-SA) 128 | * while the OpenStreetMap raw data is licensed with (ODbL) http://opendatacommons.org/licenses/odbl/ 129 | * Map tiles shall be underlined with the statement "(C) OpenStreetMap contributors". In our 5-dot character 130 | * set the lowercase letters do not look good, so we use uppercase only. 131 | * The (C) symbol is not available in our font, so we use the letters (C) instead. 132 | */ 133 | PrintTool.print(map, 6, map.getHeight() - 6, 0, "(C) OPENSTREETMAP CONTRIBUTORS", -1, false, 100); 134 | 135 | // write image 136 | return new ServiceResponse(map); 137 | } 138 | 139 | } 140 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/http/AbstractAPIHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * AbstractAPIHandler 3 | * Copyright 17.05.2016 by Michael Peter Christen, @0rb1t3r 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU Lesser General Public 7 | * License as published by the Free Software Foundation; either 8 | * version 2.1 of the License, or (at your option) any later version. 9 | * 10 | * This library is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | * Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program in the file lgpl21.txt 17 | * If not, see . 18 | */ 19 | 20 | package net.yacy.grid.http; 21 | 22 | import java.io.IOException; 23 | import java.io.PrintWriter; 24 | import java.util.Map; 25 | 26 | import javax.servlet.ServletException; 27 | import javax.servlet.http.HttpServlet; 28 | import javax.servlet.http.HttpServletRequest; 29 | import javax.servlet.http.HttpServletResponse; 30 | 31 | import org.json.JSONObject; 32 | 33 | import net.yacy.grid.http.RemoteAccess.FileTypeEncoding; 34 | 35 | @SuppressWarnings("serial") 36 | public abstract class AbstractAPIHandler extends HttpServlet implements APIHandler { 37 | 38 | public static final Long defaultCookieTime = (long) (60 * 60 * 24 * 7); 39 | public static final Long defaultAnonymousTime = (long) (60 * 60 * 24); 40 | 41 | public AbstractAPIHandler() { 42 | } 43 | 44 | public abstract ServiceResponse serviceImpl(final String protocolhostportstub, JSONObject params) throws IOException; 45 | public abstract ServiceResponse serviceImpl(final String protocolhostportstub, Map params) throws IOException; 46 | public abstract ServiceResponse serviceImpl(Query call, HttpServletResponse response) throws APIException; 47 | 48 | public String getAPIName() { 49 | String path = this.getAPIPath(); 50 | int p = path.lastIndexOf('/'); 51 | if (p >= 0) path = path.substring(p + 1); 52 | p = path.indexOf('.'); 53 | if (p >= 0) path = path.substring(0, p); 54 | return path; 55 | } 56 | 57 | private void setCORS(HttpServletResponse response) { 58 | response.setHeader("Access-Control-Allow-Origin", "*"); 59 | response.setHeader("Access-Control-Allow-Methods", "POST"); 60 | response.setHeader("Access-Control-Allow-Headers", "accept, content-type"); 61 | } 62 | 63 | @Override 64 | protected void doOptions(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 65 | super.doOptions(request, response); 66 | setCORS(response); // required by angular framework; detailed CORS can be set within the servlet 67 | } 68 | 69 | @Override 70 | protected void doHead(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 71 | super.doHead(request, response); 72 | } 73 | 74 | @Override 75 | protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 76 | Query post = RemoteAccess.evaluate(request); 77 | process(request, response, post); 78 | } 79 | 80 | @Override 81 | protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 82 | Query query = RemoteAccess.evaluate(request); 83 | query.initPOST(RemoteAccess.getPostMap(request)); 84 | process(request, response, query); 85 | } 86 | 87 | private void process(HttpServletRequest request, HttpServletResponse response, Query query) throws ServletException, IOException { 88 | 89 | long startTime = System.currentTimeMillis(); 90 | 91 | // extract standard query attributes 92 | String callback = query.get("callback", ""); 93 | boolean jsonp = callback.length() > 0; 94 | boolean minified = query.get("minified", false); 95 | 96 | try { 97 | ServiceResponse serviceResponse = serviceImpl(query, response); 98 | if (serviceResponse == null) { 99 | String message = "your request does not contain the required data"; 100 | logClient(startTime, query, 400, message); 101 | response.sendError(400, message); 102 | return; 103 | } 104 | if (serviceResponse.allowCORS()) { 105 | setCORS(response); 106 | } 107 | 108 | // write json 109 | query.setResponse(response, serviceResponse.getMimeType()); 110 | response.setCharacterEncoding("UTF-8"); 111 | if (serviceResponse.isObject() || serviceResponse.isArray()) { 112 | PrintWriter sos = response.getWriter(); 113 | if (jsonp) sos.print(callback + "("); 114 | String out = serviceResponse.toString(minified); 115 | sos.print(out); 116 | if (jsonp) sos.println(");"); 117 | sos.println(); 118 | logClient(startTime, query, 200, "ok: " + (minified ? out : serviceResponse.toString(true))); 119 | } else if (serviceResponse.isString()) { 120 | PrintWriter sos = response.getWriter(); 121 | String out = serviceResponse.toString(false); 122 | sos.print(out); 123 | logClient(startTime, query, 200, "ok: " + out); 124 | } else if (serviceResponse.isByteArray()) { 125 | response.getOutputStream().write(serviceResponse.getByteArray()); 126 | response.setHeader("Access-Control-Allow-Origin", "*"); 127 | logClient(startTime, query, 200, "ok (ByteArray)"); 128 | } else if (serviceResponse.isImage()) { 129 | FileTypeEncoding fileType = RemoteAccess.getFileType(request); 130 | RemoteAccess.writeImage(fileType, response, query, serviceResponse.getImage()); 131 | response.setHeader("Access-Control-Allow-Origin", "*"); 132 | logClient(startTime, query, 200, "ok (Image)"); 133 | } 134 | } catch (APIException e) { 135 | String message = e.getMessage(); 136 | logClient(startTime, query, e.getStatusCode(), message); 137 | response.sendError(e.getStatusCode(), message); 138 | return; 139 | } 140 | } 141 | 142 | public void logClient( 143 | long startTime, 144 | Query query, 145 | int httpResponseCode, 146 | String message) { 147 | String host = query.getClientHost(); 148 | String q = query.toString(512); 149 | if (q.length() > 512) q = q.substring(0, 512) + "..."; 150 | long t = System.currentTimeMillis() - startTime; 151 | String path = getAPIPath(); 152 | if (q.length() > 0) path = path + "?" + q; 153 | if (message.length() > 512) message = message.substring(0, 512) + "..."; 154 | String m = host + " - " + httpResponseCode + " - " + t + "ms - " + path + " - " + message; 155 | Log.logger.info(m); 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/graphics/CircleTool.java: -------------------------------------------------------------------------------- 1 | /** 2 | * CircleTool 3 | * Copyright 2007 by Michael Christen 4 | * First released 22.05.2007 at http://yacy.net 5 | * 6 | * This library is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU Lesser General Public 8 | * License as published by the Free Software Foundation; either 9 | * version 2.1 of the License, or (at your option) any later version. 10 | * 11 | * This library is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Lesser General Public License 17 | * along with this program in the file lgpl21.txt 18 | * If not, see . 19 | */ 20 | 21 | package net.yacy.grid.graphics; 22 | 23 | import java.util.ArrayList; 24 | import java.util.HashSet; 25 | import java.util.List; 26 | import java.util.Set; 27 | 28 | 29 | public class CircleTool { 30 | 31 | private static List circles = new ArrayList<>(); 32 | 33 | public static void clearcache() { 34 | circles.clear(); 35 | } 36 | 37 | private static int[] getCircleCoords(final short radius) { 38 | if (radius - 1 < circles.size()) return circles.get(radius - 1); 39 | 40 | // read some lines from known circles 41 | Set crds = new HashSet<>(); 42 | Integer co; 43 | for (short i = (short) Math.max(0, circles.size() - 1); i < circles.size(); i++) { 44 | int[] circle = circles.get(i); 45 | for (int c: circle) crds.add(c); 46 | } 47 | 48 | // compute more lines in new circles 49 | short x, y; 50 | List crc; 51 | short r1; 52 | int rc = radius < 200 ? (radius < 100 ? 100 : radius + 9) : radius; 53 | for (short r = (short) circles.size(); r < rc; r++) { 54 | r1 = (short) (r + 1); 55 | crc = new ArrayList<>(); 56 | for (short a = 0; a < 2 * r1; a++) { 57 | double h = RasterPlotter.PI4 * a / r1; 58 | double cosh = Math.cos(h); 59 | double sinh = Math.sin(h); 60 | x = (short) (r1 * cosh); 61 | y = (short) (r1 * sinh); 62 | co = x << 16 | y; 63 | if (!(crds.contains(co))) { 64 | crc.add(co); 65 | crds.add(co); 66 | } 67 | x = (short) ((r + 0.5) * cosh); 68 | y = (short) ((r + 0.5) * sinh); 69 | co = x << 16 | y; 70 | if (!(crds.contains(co))) { 71 | crc.add(co); 72 | crds.add(co); 73 | } 74 | } 75 | // put coordinates into array 76 | //System.out.print("Radius " + r + " => " + crc.size() + " points: "); 77 | int[] newCircle = new int[crc.size() - 1]; 78 | int coords; 79 | for (short i = 0; i < crc.size() - 1; i++) { 80 | coords = crc.get(i); 81 | newCircle[i] = coords; 82 | } 83 | circles.add(newCircle); 84 | } 85 | crc = null; 86 | crds = null; 87 | 88 | // finally return wanted slice 89 | return circles.get(radius - 1); 90 | } 91 | 92 | public static void circle(final RasterPlotter matrix, final int xc, final int yc, final int radius, final int intensity) { 93 | if (radius == 0) { 94 | //matrix.plot(xc, yc, 100); 95 | } else { 96 | final int[] c = getCircleCoords((short) radius); 97 | short x, y; 98 | short limit = (short) c.length; 99 | int co; 100 | for (short i = 0; i < limit; i++) { 101 | co = c[i]; 102 | x = (short) (0xffff & (co >> 16)); 103 | y = (short) (0xffff & co); 104 | matrix.plot(xc + x , yc - y - 1, intensity); // quadrant 1 105 | matrix.plot(xc - x + 1, yc - y - 1, intensity); // quadrant 2 106 | matrix.plot(xc + x , yc + y , intensity); // quadrant 4 107 | matrix.plot(xc - x + 1, yc + y , intensity); // quadrant 3 108 | } 109 | } 110 | } 111 | 112 | public static void circle(final RasterPlotter matrix, final int xc, final int yc, final int radius, int fromArc, int toArc) { 113 | // draws only a part of a circle 114 | // arc is given in degree 115 | while (fromArc > 360) fromArc -=360; 116 | while (fromArc < 0 ) fromArc +=360; 117 | while ( toArc > 360) toArc -=360; 118 | while ( toArc < 0 ) toArc +=360; 119 | if (radius == 0) { 120 | //matrix.plot(xc, yc, 100); 121 | } else { 122 | int[] c = getCircleCoords((short) radius); 123 | if (c == null) c = getCircleCoords((short) radius); 124 | final short q = (short) c.length; 125 | final short q2 = (short) (q * 2); 126 | final short q3 = (short) (q * 3); 127 | final short q4 = (short) (q * 4); 128 | final short[] c4x = new short[q4]; 129 | final short[] c4y = new short[q4]; 130 | short a0, a1, a2, a3, b0, b1; 131 | int co; 132 | for (short i = 0; i < q; i++) { 133 | b0 = i; 134 | b1 = (short) (q - 1 - i); 135 | co = c[b0]; 136 | a0 = (short) (0xffff & (co >> 16)); 137 | a1 = (short) (0xffff & co); 138 | co = c[b1]; 139 | a2 = (short) (0xffff & (co >> 16)); 140 | a3 = (short) (0xffff & co); 141 | c4x[i ] = a0 ; // quadrant 1 142 | c4y[i ] = (short) (-a1 - 1); // quadrant 1 143 | c4x[i + q ] = (short) ( 1 - a2); // quadrant 2 144 | c4y[i + q ] = (short) (-a3 - 1); // quadrant 2 145 | c4x[i + q2] = (short) ( 1 - a0); // quadrant 3 146 | c4y[i + q2] = a1 ; // quadrant 3 147 | c4x[i + q3] = a2 ; // quadrant 4 148 | c4y[i + q3] = a3 ; // quadrant 4 149 | } 150 | if (fromArc == toArc) { 151 | int i = q4 * fromArc / 360; 152 | matrix.plot(xc + c4x[i], yc + c4y[i], 100); 153 | } else if (fromArc > toArc) { 154 | // draw two parts 155 | int from = q4 * fromArc / 360; 156 | int to = q4 * toArc / 360; 157 | for (int i = from; i < q4; i++) { 158 | matrix.plot(xc + c4x[i], yc + c4y[i], 100); 159 | } 160 | for (int i = 0; i < to; i++) { 161 | matrix.plot(xc + c4x[i], yc + c4y[i], 100); 162 | } 163 | } else { 164 | // can be drawn in one part 165 | int from = q4 * fromArc / 360; 166 | int to = q4 * toArc / 360; 167 | for (int i = from; i < to; i++) { 168 | matrix.plot(xc + c4x[i], yc + c4y[i], 100); 169 | } 170 | } 171 | } 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/json/JsonTray.java: -------------------------------------------------------------------------------- 1 | /** 2 | * JsonTray 3 | * Copyright 2016 by Michael Peter Christen 4 | * First released 05.06.2016 5 | * 6 | * This library is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU Lesser General Public 8 | * License as published by the Free Software Foundation; either 9 | * version 2.1 of the License, or (at your option) any later version. 10 | * 11 | * This library is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Lesser General Public License 17 | * along with this program in the file lgpl21.txt 18 | * If not, see . 19 | */ 20 | 21 | 22 | package net.yacy.grid.json; 23 | 24 | import java.io.File; 25 | import java.io.IOException; 26 | import java.util.ArrayList; 27 | import java.util.Collection; 28 | import java.util.HashMap; 29 | import java.util.LinkedHashMap; 30 | import java.util.Map; 31 | 32 | import org.json.JSONObject; 33 | 34 | import net.yacy.grid.http.Log; 35 | import net.yacy.grid.tools.CacheMap; 36 | 37 | /** 38 | * The JsonTray class is a very simple database storage solution for json objects. 39 | * Any json object is addressed with a single key. 40 | * Json objects may be either persistent (it's stored to a file right after it was written here), 41 | * or they might be volatile (they are not written at all and only stored in case the object is closed). 42 | */ 43 | public class JsonTray { 44 | 45 | private static Map trayRegistry = new HashMap<>(); // all tray files are registered here 46 | 47 | private JsonFile per; 48 | private CacheMap vol; 49 | private File file_persistent; 50 | private File file_volatile; 51 | private long file_volatile_lastModified; 52 | private int cachesize; 53 | 54 | public JsonTray(File file_persistent, File file_volatile, int cachesize) throws IOException { 55 | this.cachesize = cachesize; 56 | this.per = new JsonFile(file_persistent, false); 57 | this.vol = null; 58 | this.file_persistent = file_persistent; 59 | this.file_volatile = file_volatile; 60 | this.file_volatile_lastModified = this.file_volatile.lastModified(); 61 | ensureVolatileInitBase(); 62 | 63 | //DAO.log("init JsonTray persistent '" + file_persistent.getAbsolutePath() + "' " + getPersistentSize()); 64 | //DAO.log("init JsonTray volatile '" + file_volatile.getAbsolutePath() + "' " + getVolatileSize()); 65 | trayRegistry.put(this.hashCode(), this); 66 | } 67 | 68 | @Override 69 | public int hashCode() { 70 | return (this.file_persistent.getAbsolutePath() + this.file_volatile.getAbsolutePath()).hashCode(); 71 | } 72 | 73 | private void ensureVolatileInit() { 74 | ensureVolatileInitBase(); 75 | } 76 | 77 | private void ensureVolatileInitBase() { 78 | if (this.vol != null) return; 79 | this.vol = new CacheMap(this.cachesize); 80 | if (!this.file_volatile.exists()) return; 81 | 82 | try { 83 | JSONObject j = JsonFile.readJson(this.file_volatile); 84 | for (String key: j.keySet()) this.vol.put(key, j.getJSONObject(key)); 85 | } catch (Exception e) { 86 | Log.logger.error("", e); 87 | } 88 | } 89 | 90 | public int getPersistentSize() { 91 | return this.per.size(); 92 | } 93 | 94 | public int getVolatileSize() { 95 | ensureVolatileInit(); 96 | return this.vol == null ? 0 : this.vol.size(); 97 | } 98 | 99 | public void close() { 100 | // commit any data that has not yet been stored. 101 | this.commit(); 102 | 103 | if (this.file_volatile != null) try { 104 | // copy the volatile data into a JSONObject to store it in a dump file 105 | JSONObject j = new JSONObject(true); 106 | for (Map.Entry entry: this.vol.getMap().entrySet()) { 107 | j.put(entry.getKey(), entry.getValue()); 108 | } 109 | 110 | // check the storage time of the volatile file; if it was stored in between, read it again and merge the data 111 | if (this.file_volatile.lastModified() > this.file_volatile_lastModified) { 112 | JSONObject k = JsonFile.readJson(this.file_volatile); 113 | for (String s: k.keySet()) { 114 | if (!j.has(s)) j.put(s, k.get(s)); 115 | } 116 | } 117 | 118 | // finally store the data 119 | JsonFile.writeJson(this.file_volatile, j, false); 120 | } catch (IOException e) { 121 | e.printStackTrace(); 122 | } 123 | } 124 | 125 | public boolean has(String key) { 126 | ensureVolatileInit(); 127 | synchronized(this.vol) { 128 | if (this.vol.exist(key)) return true; 129 | } 130 | return this.per.has(key); 131 | } 132 | 133 | public JsonTray put(String key, JSONObject value, boolean persistent) { 134 | if (persistent) putPersistent(key, value); else putVolatile(key, value); 135 | return this; 136 | } 137 | 138 | private JsonTray putPersistent(String key, JSONObject value) { 139 | this.per.put(key, value); 140 | return this; 141 | } 142 | 143 | private JsonTray putVolatile(String key, JSONObject value) { 144 | ensureVolatileInit(); 145 | synchronized (this.vol) { 146 | this.vol.put(key, value); 147 | } 148 | return this; 149 | } 150 | 151 | public JsonTray remove(String key) { 152 | ensureVolatileInit(); 153 | synchronized(this.vol) { 154 | if (this.vol.exist(key)){ 155 | this.vol.remove(key); 156 | return this; 157 | } 158 | } 159 | if(this.per.has(key)){ 160 | this.per.remove(key); 161 | } 162 | return this; 163 | } 164 | 165 | public JsonTray commit() { 166 | this.per.commit(); 167 | return this; 168 | } 169 | 170 | public JSONObject getJSONObject(String key) { 171 | ensureVolatileInit(); 172 | synchronized(this.vol) { 173 | JSONObject value = this.vol.get(key); 174 | if (value != null) return value; 175 | } 176 | return this.per.getJSONObject(key); 177 | } 178 | 179 | public JSONObject toJSON() { 180 | JSONObject j = new JSONObject(); 181 | for (String key : this.per.keySet()){ 182 | j.put(key, this.per.get(key)); 183 | } 184 | ensureVolatileInit(); 185 | synchronized (this.vol) { 186 | LinkedHashMap map = this.vol.getMap(); 187 | for(String key : map.keySet()){ 188 | j.put(key, map.get(key)); 189 | } 190 | } 191 | return j; 192 | } 193 | 194 | public Collection keys() { 195 | ArrayList keys = new ArrayList<>(); 196 | keys.addAll(this.per.keySet()); 197 | ensureVolatileInit(); 198 | synchronized (this.vol) { 199 | keys.addAll(this.vol.getMap().keySet()); 200 | } 201 | return keys; 202 | } 203 | 204 | } 205 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/graphics/PrintTool.java: -------------------------------------------------------------------------------- 1 | /** 2 | * PrintTool 3 | * Copyright 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany 4 | * First released 22.05.2007 at http://yacy.net 5 | * 6 | * This library is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU Lesser General Public 8 | * License as published by the Free Software Foundation; either 9 | * version 2.1 of the License, or (at your option) any later version. 10 | * 11 | * This library is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Lesser General Public License 17 | * along with this program in the file lgpl21.txt 18 | * If not, see . 19 | */ 20 | 21 | package net.yacy.grid.graphics; 22 | 23 | 24 | public class PrintTool { 25 | 26 | 27 | private static long[] font = new long[]{ 28 | 0x00000000000000L,0x00300C03000030L,0x00CC3300000000L,0x00CCFFCCCFFCCCL,0x02FFCC2FE0CFFEL,0x00C3CF0FC3CF0CL,0x02FCE02ECCE2ECL,0x00300C00000000L, 29 | 0x00030380C03803L,0x0300B00C0B0000L,0x0000332BA03000L,0x00000C0FC0C000L,0x0000000302C0C0L,0x0000000FC00000L,0x00000000000030L,0x00030383838380L, 30 | 0x00FCE3F33F2CFCL,0x00303C0300C0FCL,0x00FCE2CBEB83FFL,0x00FCE2C0EE2CFCL,0x00BCBB3FF0300CL,0x03FFC03FC02FFCL,0x00FCE03FEE2CFCL,0x03FF0B8380C030L, 31 | 0x00FCE2EFCE2CFCL,0x00FCE2EFF02CFCL,0x00000C0000C000L,0x00000C0002C0C0L,0x000E0E0E00E00EL,0x0000FFC00FFC00L,0x02C02C02C2C2C0L,0x00FC0B03000030L, 32 | 0x02FEFCFBEE00FCL,0x00B8BBB8BFFF03L,0x03F8CB3FEC2FFCL,0x00FCE0300E00FCL,0x03FCC2F03C2FFCL,0x03FFC03F0C03FFL,0x03FFC03F0C0300L,0x00FCE033FC2CFCL, 33 | 0x0303C0FFFC0F03L,0x03FF0C0300C3FFL,0x03FF00C03E2CFCL,0x030BCB3F0CB30BL,0x0300C0300C03FFL,0x0303F3FBBC8F03L,0x0383F8FBBCBF0BL,0x00FCE2F03E2CFCL, 34 | 0x03FCC2FFCC0300L,0x00FCE2F3BEB8FBL,0x03FCC2FFCCB30BL,0x00FEE00FC02EFCL,0x03FF0C0300C030L,0x0303C0F03E2CFCL,0x0303C0F8B3B030L,0x0303C0F23EECCCL, 35 | 0x038BBB8B8BBB8BL,0x0303B38B80C030L,0x03FF0B8B8B83FFL,0x003F0C0300C03FL,0x0380B80B80B80BL,0x03F00C0300C3F0L,0x00B8BBB8B00000L,0x000000000003FFL, 36 | 0x00E02C00000000L,0x0000002FFE0CFFL,0x0300C03FCC2FFCL,0x0000000FFE00FFL,0x000300CFFE0CFFL,0x0000BE32CEE0FFL,0x003C0E0FC0C030L,0x00002F8E32EFFCL, 37 | 0x0300C03BEFAF83L,0x0030000300C030L,0x0030000302C0E0L,0x0300C032FFE32FL,0x00300C0300C030L,0x0000000ECEEF33L,0x0000000FCE2F03L,0x0000000FCE2CFCL, 38 | 0x0000FF30BFF300L,0x00003FF833FC03L,0x00000B0380C030L,0x0000000BCBAFACL,0x00303F0300C030L,0x000000303E2CFCL,0x00000038BBB8B8L,0x000000333EECECL, 39 | 0x0000000EC2E0ECL,0x0000000EC2E030L,0x0000000FF2E3FCL,0x000F0B8B80B80FL,0x00300C0300C030L,0x03C0B80B8B83C0L,0x0000B83BB0B800L,0x03FFC0F03C0FFFL 40 | }; 41 | 42 | public static void print(final RasterPlotter matrix, int x, int y, final int angle, final char letter, final boolean tilt, final int intensity) { 43 | final int index = letter - 0x20; 44 | if (index >= font.length) return; 45 | long character = font[index]; 46 | long row; 47 | long c; 48 | int i2 = intensity / 2; 49 | int i3 = intensity / 3; 50 | int i5 = intensity / 5; 51 | for (int i = 0; i < 5; i++) { 52 | row = character & 0x3FFL; 53 | character = character >> 10; 54 | if (angle == 0) { 55 | for (int j = 0; j < 5; j++) { 56 | c = row & 3L; 57 | int xx = x + 5 - j; 58 | if (tilt) xx += (i + 1) / 2; 59 | if (c == 3) matrix.plot(xx, y, intensity); 60 | else if (c == 2) matrix.plot(xx, y, i3); 61 | row = row >> 2; 62 | } 63 | y--; 64 | } 65 | if (angle == 90) { 66 | for (int j = 0; j < 5; j++) { 67 | c = row & 3L; 68 | if (c == 3) matrix.plot(x, y - 5 + j, intensity); 69 | else if (c == 2) matrix.plot(x, y - 5 + j, i3); 70 | row = row >> 2; 71 | } 72 | x--; 73 | } 74 | if (angle == 315) { 75 | for (int j = 0; j < 5; j++) { 76 | c = row & 3L; 77 | if (c == 3) { matrix.plot(x + 5 - j, y + 5 - j, intensity); matrix.plot(x + 6 - j, y + 5 - j, i2); matrix.plot(x + 5 - j, y + 6 - j, i2); } 78 | else if (c == 2) { matrix.plot(x + 5 - j, y + 5 - j, i3); matrix.plot(x + 6 - j, y + 5 - j, i5); matrix.plot(x + 5 - j, y + 6 - j, i5); } 79 | row = row >> 2; 80 | } 81 | x++; 82 | y--; 83 | } 84 | } 85 | } 86 | 87 | public static void print(final RasterPlotter matrix, final int x, final int y, final int angle, final String message, final int align, boolean tilt, int intensity) { 88 | // align = -1 : left 89 | // align = 1 : right 90 | // align = 0 : center 91 | int xx = 0, yy = 0; 92 | if (angle == 0) { 93 | xx = (align == -1) ? x : (align == 1) ? x - 6 * message.length() : x - 3 * message.length(); 94 | yy = y; 95 | } else if (angle == 90) { 96 | xx = x; 97 | yy = (align == -1) ? y : (align == 1) ? y + 6 * message.length() : y + 3 * message.length(); 98 | } else if (angle == 315) { 99 | xx = (align == -1) ? x : (align == 1) ? x - 6 * message.length() : x - 3 * message.length(); 100 | yy = (align == -1) ? y : (align == 1) ? y - 6 * message.length() : y - 3 * message.length(); 101 | } 102 | for (int i = 0; i < message.length(); i++) { 103 | print(matrix, xx, yy, angle, message.charAt(i), tilt, intensity); 104 | if (angle == 0) xx += 6; 105 | else if (angle == 90) yy -= 6; 106 | else if (angle == 315) {xx += 6; yy += 6;} 107 | } 108 | } 109 | 110 | 111 | private static final int arcDist = 8; 112 | /** 113 | * print a string at the distance of a circle 114 | * @param matrix the RasterPlotter 115 | * @param cx center of circle, x 116 | * @param cy center of circle, y 117 | * @param radius radius == distance of text from circle center 118 | * @param angle angle == position of text on a circle in distance of radius 119 | * @param message the message to be printed 120 | */ 121 | public static void arcPrint(final RasterPlotter matrix, final int cx, final int cy, final int radius, final double angle, final String message, final int intensity) { 122 | final int x = cx + (int) ((radius + 1) * Math.cos(RasterPlotter.PI180 * angle)); 123 | final int y = cy - (int) ((radius + 1) * Math.sin(RasterPlotter.PI180 * angle)); 124 | int yp = y + 3; 125 | if ((angle > arcDist) && (angle < 180 - arcDist)) yp = y; 126 | if ((angle > 180 + arcDist) && (angle < 360 - arcDist)) yp = y + 6; 127 | if ((angle > ( 90 - arcDist)) && (angle < ( 90 + arcDist))) yp -= 6; 128 | if ((angle > (270 - arcDist)) && (angle < (270 + arcDist))) yp += 6; 129 | int xp = x - 3 * message.length(); 130 | if ((angle > (90 + arcDist)) && (angle < (270 - arcDist))) xp = x - 6 * message.length(); 131 | if ((angle < (90 - arcDist)) || (angle > (270 + arcDist))) xp = x; 132 | print(matrix, xp, yp, 0, message, -1, false, intensity); 133 | } 134 | 135 | 136 | } 137 | -------------------------------------------------------------------------------- /src/main/java/net/yacy/grid/tools/Punycode.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2004, 2005, 2006, 2007 Free Software Foundation, Inc. 3 | * 4 | * Author: Oliver Hitz 5 | * 6 | * This file is part of GNU Libidn. 7 | * 8 | * This library is free software; you can redistribute it and/or 9 | * modify it under the terms of the GNU Lesser General Public License 10 | * as published by the Free Software Foundation; either version 2.1 of 11 | * the License, or (at your option) any later version. 12 | * 13 | * This library is distributed in the hope that it will be useful, but 14 | * WITHOUT ANY WARRANTY; without even the implied warranty of 15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 | * Lesser General Public License for more details. 17 | * 18 | * You should have received a copy of the GNU Lesser General Public 19 | * License along with this library; if not, write to the Free Software 20 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 21 | * USA 22 | */ 23 | 24 | package net.yacy.grid.tools; 25 | 26 | public class Punycode { 27 | /* Punycode parameters */ 28 | private final static int TMIN = 1; 29 | private final static int TMAX = 26; 30 | private final static int BASE = 36; 31 | private final static int INITIAL_N = 128; 32 | private final static int INITIAL_BIAS = 72; 33 | private final static int DAMP = 700; 34 | private final static int SKEW = 38; 35 | private final static char DELIMITER = '-'; 36 | 37 | /** 38 | * Punycodes a unicode string. 39 | * 40 | * @param input Unicode string. 41 | * @return Punycoded string. 42 | */ 43 | public static String encode(final String input) throws PunycodeException { 44 | int n = INITIAL_N; 45 | int delta = 0; 46 | int bias = INITIAL_BIAS; 47 | final StringBuilder output = new StringBuilder(input.length() + 1); 48 | 49 | // Copy all basic code points to the output 50 | int b = 0; 51 | for (int i = 0; i < input.length(); i++) { 52 | final char c = input.charAt(i); 53 | if (isBasic(c)) { 54 | output.append(c); 55 | b++; 56 | } 57 | } 58 | 59 | // Append delimiter 60 | if (b > 0) { 61 | output.append(DELIMITER); 62 | } 63 | 64 | int h = b; 65 | while (h < input.length()) { 66 | int m = Integer.MAX_VALUE; 67 | 68 | // Find the minimum code point >= n 69 | for (int i = 0; i < input.length(); i++) { 70 | final int c = input.charAt(i); 71 | if (c >= n && c < m) { 72 | m = c; 73 | } 74 | } 75 | 76 | if (m - n > (Integer.MAX_VALUE - delta) / (h + 1)) { 77 | throw new PunycodeException(PunycodeException.OVERFLOW); 78 | } 79 | delta = delta + (m - n) * (h + 1); 80 | n = m; 81 | 82 | for (int j = 0; j < input.length(); j++) { 83 | final int c = input.charAt(j); 84 | if (c < n) { 85 | delta++; 86 | if (0 == delta) { 87 | throw new PunycodeException(PunycodeException.OVERFLOW); 88 | } 89 | } 90 | if (c == n) { 91 | int q = delta; 92 | 93 | for (int k = BASE;; k += BASE) { 94 | int t; 95 | if (k <= bias) { 96 | t = TMIN; 97 | } else if (k >= bias + TMAX) { 98 | t = TMAX; 99 | } else { 100 | t = k - bias; 101 | } 102 | if (q < t) { 103 | break; 104 | } 105 | output.append((char) digit2codepoint(t + (q - t) % (BASE - t))); 106 | q = (q - t) / (BASE - t); 107 | } 108 | 109 | output.append((char) digit2codepoint(q)); 110 | bias = adapt(delta, h + 1, h == b); 111 | delta = 0; 112 | h++; 113 | } 114 | } 115 | 116 | delta++; 117 | n++; 118 | } 119 | 120 | return output.toString(); 121 | } 122 | 123 | /** 124 | * Decode a punycoded string. 125 | * 126 | * @param input Punycode string 127 | * @return Unicode string. 128 | */ 129 | public static String decode(final String input) 130 | throws PunycodeException 131 | { 132 | int n = INITIAL_N; 133 | int i = 0; 134 | int bias = INITIAL_BIAS; 135 | 136 | int d = input.lastIndexOf(DELIMITER); 137 | final StringBuilder output = new StringBuilder(d + 1); 138 | if (d > 0) { 139 | for (int j = 0; j < d; j++) { 140 | final char c = input.charAt(j); 141 | if (!isBasic(c)) { 142 | throw new PunycodeException(PunycodeException.BAD_INPUT); 143 | } 144 | output.append(c); 145 | } 146 | d++; 147 | } else { 148 | d = 0; 149 | } 150 | 151 | while (d < input.length()) { 152 | final int oldi = i; 153 | int w = 1; 154 | 155 | for (int k = BASE; ; k += BASE) { 156 | if (d == input.length()) { 157 | throw new PunycodeException(PunycodeException.BAD_INPUT); 158 | } 159 | final int c = input.charAt(d++); 160 | final int digit = codepoint2digit(c); 161 | if (digit > (Integer.MAX_VALUE - i) / w) { 162 | throw new PunycodeException(PunycodeException.OVERFLOW); 163 | } 164 | 165 | i = i + digit * w; 166 | 167 | int t; 168 | if (k <= bias) { 169 | t = TMIN; 170 | } else if (k >= bias + TMAX) { 171 | t = TMAX; 172 | } else { 173 | t = k - bias; 174 | } 175 | if (digit < t) { 176 | break; 177 | } 178 | w = w * (BASE - t); 179 | } 180 | 181 | bias = adapt(i - oldi, output.length()+1, oldi == 0); 182 | 183 | if (i / (output.length() + 1) > Integer.MAX_VALUE - n) { 184 | throw new PunycodeException(PunycodeException.OVERFLOW); 185 | } 186 | 187 | n = n + i / (output.length() + 1); 188 | i = i % (output.length() + 1); 189 | output.insert(i, (char) n); 190 | i++; 191 | } 192 | 193 | return output.toString(); 194 | } 195 | 196 | public final static int adapt(int delta, final int numpoints, final boolean first) 197 | { 198 | if (first) { 199 | delta = delta / DAMP; 200 | } else { 201 | delta = delta / 2; 202 | } 203 | 204 | delta = delta + (delta / numpoints); 205 | 206 | int k = 0; 207 | while (delta > ((BASE - TMIN) * TMAX) / 2) { 208 | delta = delta / (BASE - TMIN); 209 | k = k + BASE; 210 | } 211 | 212 | return k + ((BASE - TMIN + 1) * delta) / (delta + SKEW); 213 | } 214 | 215 | public final static boolean isBasic(final char c) 216 | { 217 | return c < 0x80; 218 | } 219 | 220 | // the following method has been added by Michael Christen 221 | public static boolean isBasic(final String input) { 222 | if (input == null) return true; 223 | for (int j = 0; j < input.length(); j++) { 224 | if (!isBasic(input.charAt(j))) return false; 225 | } 226 | return true; 227 | } 228 | 229 | public final static int digit2codepoint(final int d) 230 | throws PunycodeException 231 | { 232 | if (d < 26) { 233 | // 0..25 : 'a'..'z' 234 | return d + 'a'; 235 | } else if (d < 36) { 236 | // 26..35 : '0'..'9'; 237 | return d - 26 + '0'; 238 | } else { 239 | throw new PunycodeException(PunycodeException.BAD_INPUT); 240 | } 241 | } 242 | 243 | public final static int codepoint2digit(final int c) 244 | throws PunycodeException 245 | { 246 | if (c - '0' < 10) { 247 | // '0'..'9' : 26..35 248 | return c - '0' + 26; 249 | } else if (c - 'a' < 26) { 250 | // 'a'..'z' : 0..25 251 | return c - 'a'; 252 | } else { 253 | throw new PunycodeException(PunycodeException.BAD_INPUT); 254 | } 255 | } 256 | 257 | public static class PunycodeException 258 | extends Exception 259 | { 260 | /** 261 | * 262 | */ 263 | private static final long serialVersionUID = 1L; 264 | public static final String OVERFLOW = "Overflow."; 265 | public static final String BAD_INPUT = "Bad input."; 266 | 267 | /** 268 | * Creates a new PunycodeException. 269 | * 270 | * @param m message. 271 | */ 272 | public PunycodeException(final String m) 273 | { 274 | super(m); 275 | } 276 | } 277 | } --------------------------------------------------------------------------------