├── .github └── FUNDING.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── anomaly_detection ├── .gitignore ├── Makefile ├── data │ └── cleaned_wisconsin_cancer_data.csv ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── markwatson │ └── anomaly_detection │ ├── AnomalyDetection.java │ ├── PrintHistogram.java │ └── WisconsinAnomalyDetection.java ├── genetic-algorithms ├── .gitignore ├── .settings │ ├── org.eclipse.jdt.apt.core.prefs │ ├── org.eclipse.jdt.core.prefs │ └── org.eclipse.m2e.core.prefs ├── Makefile ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── markwatson │ └── geneticalgorithm │ ├── Genetic.java │ └── TestGenetic.java ├── info_gathering ├── .gitignore ├── Makefile ├── pom.xml └── src │ ├── main │ └── java │ │ ├── com │ │ └── markwatson │ │ │ ├── info_spiders │ │ │ ├── DBpediaLookupClient.java │ │ │ ├── GeoNameData.java │ │ │ ├── GeoNamesClient.java │ │ │ └── WebSpider.java │ │ │ └── web_scraping │ │ │ └── MySitesExamples.java │ │ └── org │ │ └── geonames │ │ ├── Address.java │ │ ├── BoundingBox.java │ │ ├── FeatureClass.java │ │ ├── GeoNamesException.java │ │ ├── InsufficientStyleException.java │ │ ├── Intersection.java │ │ ├── InvalidParameterException.java │ │ ├── PostalCode.java │ │ ├── PostalCodeSearchCriteria.java │ │ ├── StreetSegment.java │ │ ├── Style.java │ │ ├── Timezone.java │ │ ├── Toponym.java │ │ ├── ToponymSearchCriteria.java │ │ ├── ToponymSearchResult.java │ │ ├── WeatherObservation.java │ │ ├── WebService.java │ │ ├── WikipediaArticle.java │ │ ├── package.html │ │ ├── utils │ │ ├── Bearing.java │ │ ├── Distance.java │ │ ├── DistanceOrder.java │ │ └── package.html │ │ └── wikipedia │ │ └── TextSummaryExtractor.java │ └── test │ └── java │ └── com │ └── markwatson │ └── info_spiders │ ├── GeoNamesClientTest.java │ └── WebClientTest.java ├── kgc ├── .gitignore ├── Makefile ├── README.md ├── pom.xml ├── src │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── knowledgegraphcreator │ │ │ └── KGC.java │ └── test │ │ └── java │ │ └── com.knowledgegraphcreator │ │ └── KgcTest.java └── test_data │ ├── CREDITS │ ├── test1.meta │ ├── test1.txt │ ├── test2.meta │ ├── test2.txt │ ├── test3.meta │ └── test3.txt ├── kgn ├── .gitignore ├── Makefile ├── README.md ├── dependency-reduced-pom.xml ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── knowledgegraphnavigator │ ├── EntityAndDescription.java │ ├── EntityDetail.java │ ├── EntityRelationships.java │ ├── KGN.java │ ├── Log.java │ ├── PrintEntityResearchResults.java │ ├── Sparql.java │ └── Utils.java ├── langchain4j-ollama ├── .gitignore ├── Makefile ├── README.md ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── markwatson │ │ └── langchain4j_ollama │ │ └── OllamaLlmLangChain4j.java │ └── test │ └── java │ └── com │ └── markwatson │ └── langchain4j_ollama │ └── OllamaLlmLangChain4jTest.java ├── ner_dbpedia ├── Makefile ├── pom.xml └── src │ ├── .DS_Store │ ├── main │ ├── .DS_Store │ ├── java │ │ └── com │ │ │ └── markwatson │ │ │ └── ner_dbpedia │ │ │ ├── NerMaps.java │ │ │ └── TextToDbpediaUris.java │ └── resources │ │ ├── .DS_Store │ │ ├── BroadcastNetworkNamesDbPedia.txt │ │ ├── CityNamesDbpedia.txt │ │ ├── CompanyNamesDbPedia.txt │ │ ├── CountryNamesDbpedia.txt │ │ ├── META-INF │ │ └── file-manefest-for-text-resource-files.txt │ │ ├── MusicGroupNamesDbPedia.txt │ │ ├── PeopleDbPedia.txt │ │ ├── PoliticalPartyNamesDbPedia.txt │ │ ├── TradeUnionNamesDbPedia.txt │ │ └── UniversityNamesDbPedia.txt │ └── test │ └── java │ └── com │ └── markwatson │ └── ner_dbpedia │ └── TextToDbpediaUrisTest.java ├── neuralnetworks ├── .gitignore ├── Makefile ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── markwatson │ │ └── neuralnetworks │ │ ├── Graph.java │ │ ├── GraphPanel.java │ │ ├── Neural_1H.java │ │ ├── Neural_2H.java │ │ ├── Neural_2H_momentum.java │ │ ├── Plot1DPanel.java │ │ ├── Plot2DPanel.java │ │ └── gui_examples │ │ ├── GUITest_1H.java │ │ ├── GUITest_2H.java │ │ └── GUITest_2H_momentum.java │ └── test │ └── java │ └── com │ └── markwatson │ └── neuralnetworks │ ├── NeuralNetwork_1H_Test.java │ ├── NeuralNetwork_2H_Test.java │ ├── NeuralNetwork_2H_momentum_Test.java │ └── xor.java ├── nlp ├── .gitignore ├── Makefile ├── README.md ├── pom.xml ├── src │ └── main │ │ └── java │ │ ├── com │ │ └── markwatson │ │ │ └── nlp │ │ │ ├── AutoTagger.java │ │ │ ├── ComparableDocument.java │ │ │ ├── ExtractNames.java │ │ │ ├── FastTag.java │ │ │ └── util │ │ │ ├── NameValue.java │ │ │ ├── NoiseWords.java │ │ │ ├── RunExternal.java │ │ │ ├── ScoredList.java │ │ │ └── Tokenizer.java │ │ └── public_domain │ │ └── Stemmer.java └── test_data │ ├── classification_tags.xml │ ├── dictionary │ ├── README │ └── english.0 │ └── propername.ser ├── ollama-llm-client ├── Makefile ├── README.md ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── markwatson │ │ └── ollama │ │ └── OllamaLlmClient.java │ └── test │ └── java │ └── com │ └── markwatson │ └── ollama │ └── OllamaLlmClientTest.java ├── openai-llm-client ├── Makefile ├── README.md ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── markwatson │ │ └── openai │ │ └── OpenAICompletions.java │ └── test │ └── java │ └── com │ └── markwatson │ └── openai │ └── OpenAICompletionsTest.java ├── prompts ├── README.md ├── extraction_prompt.txt ├── summarization_prompt.txt ├── two-shot-2-var.txt └── two-shot-2.txt ├── search ├── .gitignore ├── .settings │ ├── org.eclipse.jdt.apt.core.prefs │ ├── org.eclipse.jdt.core.prefs │ └── org.eclipse.m2e.core.prefs ├── Makefile ├── pom.xml └── src │ └── main │ └── java │ └── search │ ├── game │ ├── Chess.java │ ├── ChessMove.java │ ├── ChessPosition.java │ ├── GameSearch.java │ ├── Move.java │ ├── Position.java │ ├── TicTacToe.java │ ├── TicTacToeMove.java │ ├── TicTacToePosition.java │ └── out.txt │ ├── graph │ ├── AbstractGraphSearch.java │ ├── BreadthFirstSearch.java │ ├── DepthFirstSearch.java │ ├── GraphBreadthFirstSearch.java │ └── GraphDepthFirstSearch.java │ └── maze │ ├── AbstractSearchEngine.java │ ├── BreadthFirstSearchEngine.java │ ├── DepthFirstSearchEngine.java │ ├── Location.java │ ├── Maze.java │ ├── MazeBreadthFirstSearch.java │ └── MazeDepthFirstSearch.java └── semantic_web_apache_jena ├── .gitignore ├── Makefile ├── data ├── .DS_Store ├── business.sql ├── foaf.nt ├── inferencing_1.txt ├── inferencing_2.txt ├── inferencing_3.txt ├── inferencing_4.txt ├── news.n3 ├── news.nt ├── rdfs_business.n3 ├── rdfs_business.nt ├── rdfs_sample_1.n3 ├── rdfs_sample_1.owl ├── rdfs_sample_2.n3 ├── rdfs_sample_2.owl ├── sample_news.n3 ├── sample_news.nt ├── sparql_ask_test.txt ├── sparql_combine_rdfs_test.txt ├── sparql_combine_test.txt ├── sparql_construct_test.txt ├── sparql_describe_test.txt └── sparql_select_test.txt ├── jenaApisLog.log ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── markwatson │ │ └── semanticweb │ │ ├── Cache.java │ │ ├── JenaApis.java │ │ └── QueryResult.java └── resources │ └── log4j.xml └── test └── java └── com └── markwatson └── semanticweb └── JenaApisTest.java /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: mark-watson # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | markwatson: # Replace with a single Patreon username 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | out 2 | .classpath 3 | .project 4 | *.iml 5 | *.ipr 6 | *.iws 7 | *.class 8 | .DS_Store 9 | target 10 | .DS_Store 11 | */.DS_Store 12 | .settings 13 | .idea 14 | *.log 15 | .vscode 16 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | install: 2 | cd genetic-algorithms ; make install 3 | cd neuralnetworks ; make install 4 | cd semantic_web_apache_jena ; make install 5 | cd info_gathering ; make install 6 | cd nlp ; make install 7 | cd opennlp ; make install 8 | cd wordnet ; make install 9 | cd ner_dbpedia ; make install 10 | cd kgc ; make install 11 | cd kgn ; make install 12 | cd anomaly_detection ; make install 13 | #cd powerloom ; make install 14 | #cd deeplearning ; make install 15 | cd search ; make install 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Code for the April 2024 edition of "Practical Artificial Intelligence With Java" 2 | 3 | The new 2024 edition is largely a rewrite of older material with the addition of new material (mostly Large Language Models) 4 | 5 | [Leanpub Link for latest edition](https://leanpub.com/javaai) 6 | 7 | This book is a combination of 8 | 9 | - new coverage of LLMs 10 | - new coverage of deep learning 11 | - new material: creating and using knowledge graphs 12 | - examples from my discontinued book "Power Java": anomaly detection, linked data, using DBPedia, OpenNLP, and web scraping 13 | - examples from the original editions of this book: genetic algorithms and search algorithms 14 | - a few examples updated from my discontinued book "Practical Semantic Web and Linked Data Applications, Java Edition" 15 | -------------------------------------------------------------------------------- /anomaly_detection/.gitignore: -------------------------------------------------------------------------------- 1 | saved-model.bin 2 | conf.json 3 | *.iml 4 | target 5 | -------------------------------------------------------------------------------- /anomaly_detection/Makefile: -------------------------------------------------------------------------------- 1 | wisconsin: 2 | mvn install 3 | mvn exec:java -Dexec.mainClass="com.markwatson.anomaly_detection.WisconsinAnomalyDetection" 4 | 5 | install: 6 | mvn install -DskipTest -Dmaven.test.skip=true 7 | 8 | 9 | -------------------------------------------------------------------------------- /anomaly_detection/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.markwatson 5 | anomaly_detection 6 | jar 7 | 1.0-SNAPSHOT 8 | anomaly_detection 9 | 10 | http://maven.apache.org 11 | 12 | 11.0 13 | 14 | 15 | 16 | 17 | 18 | junit 19 | junit 20 | 4.13.1 21 | test 22 | 23 | 24 | org.slf4j 25 | slf4j-api 26 | 1.7.12 27 | 28 | 29 | org.apache.commons 30 | commons-io 31 | 1.3.2 32 | 33 | 34 | 35 | 36 | 37 | org.apache.maven.plugins 38 | maven-compiler-plugin 39 | 2.3.2 40 | 41 | 1.8 42 | 1.8 43 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /anomaly_detection/src/main/java/com/markwatson/anomaly_detection/PrintHistogram.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.anomaly_detection; 2 | 3 | /** 4 | * Created by markw on 10/9/15. 5 | */ 6 | public class PrintHistogram { 7 | public static void historam(String title, double [][] values, int indexToDisplay, double min, double max, int numBins) { 8 | int [] bins = new int[numBins]; 9 | for (int i=0; i max) max = xs[i]; 31 | } 32 | for (int i=0; i<9; i++) xs[i] = (xs[i] - min) / (max - min); 33 | 34 | xs[9] = (xs[9] - 2) * 0.5; // make target output be [0,1] instead of [2,4] 35 | training_data[line_count++] = xs; 36 | } 37 | 38 | if (PRINT_HISTOGRAMS) { 39 | PrintHistogram.historam("Clump Thickness", training_data, 0, 0.0, 1.0, NUM_HISTOGRAM_BINS); 40 | PrintHistogram.historam("Uniformity of Cell Size", training_data, 1, 0.0, 1.0, NUM_HISTOGRAM_BINS); 41 | PrintHistogram.historam("Uniformity of Cell Shape", training_data, 2, 0.0, 1.0, NUM_HISTOGRAM_BINS); 42 | PrintHistogram.historam("Marginal Adhesion", training_data, 3, 0.0, 1.0, NUM_HISTOGRAM_BINS); 43 | PrintHistogram.historam("Single Epithelial Cell Size", training_data, 4, 0.0, 1.0, NUM_HISTOGRAM_BINS); 44 | PrintHistogram.historam("Bare Nuclei", training_data, 5, 0.0, 1.0, NUM_HISTOGRAM_BINS); 45 | PrintHistogram.historam("Bland Chromatin", training_data, 6, 0.0, 1.0, NUM_HISTOGRAM_BINS); 46 | PrintHistogram.historam("Normal Nucleoli", training_data, 7, 0.0, 1.0, NUM_HISTOGRAM_BINS); 47 | PrintHistogram.historam("Mitoses", training_data, 8, 0.0, 1.0, NUM_HISTOGRAM_BINS); 48 | } 49 | 50 | AnomalyDetection detector = new AnomalyDetection(10, line_count - 1, training_data); 51 | 52 | // the train method will print training results like 53 | // precision, recall, and F1: 54 | detector.train(); 55 | 56 | // get best model parameters: 57 | double best_epsilon = detector.bestEpsilon(); 58 | double [] mean_values = detector.muValues(); 59 | double [] sigma_squared = detector.sigmaSquared(); 60 | 61 | // to use this model, use the method AnomalyDetection.isAnamoly(double []): 62 | 63 | double [] test_malignant = new double[] {0.5,1,1,0.8,0.5,0.5,0.7,1,0.1}; 64 | double [] test_benign = new double[] {0.5,0.4,0.5,0.1,0.8,0.1,0.3,0.6,0.1}; 65 | boolean malignant_result = detector.isAnamoly(test_malignant); 66 | boolean benign_result = detector.isAnamoly(test_benign); 67 | System.out.println("\n\nUsing the trained model:"); 68 | System.out.println("malignant result = " + malignant_result + ", benign result = " + benign_result); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /genetic-algorithms/.gitignore: -------------------------------------------------------------------------------- 1 | saved-model.bin 2 | conf.json 3 | *.iml 4 | target 5 | .idea 6 | 7 | -------------------------------------------------------------------------------- /genetic-algorithms/.settings/org.eclipse.jdt.apt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.apt.aptEnabled=false 3 | -------------------------------------------------------------------------------- /genetic-algorithms/.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 3 | org.eclipse.jdt.core.compiler.compliance=1.8 4 | org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled 5 | org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning 6 | org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore 7 | org.eclipse.jdt.core.compiler.processAnnotations=disabled 8 | org.eclipse.jdt.core.compiler.release=disabled 9 | org.eclipse.jdt.core.compiler.source=1.8 10 | -------------------------------------------------------------------------------- /genetic-algorithms/.settings/org.eclipse.m2e.core.prefs: -------------------------------------------------------------------------------- 1 | activeProfiles= 2 | eclipse.preferences.version=1 3 | resolveWorkspaceProjects=true 4 | version=1 5 | -------------------------------------------------------------------------------- /genetic-algorithms/Makefile: -------------------------------------------------------------------------------- 1 | test: 2 | mvn install 3 | mvn exec:java -Dexec.mainClass="com.markwatson.geneticalgorithm.TestGenetic" 4 | 5 | 6 | install: 7 | mvn install -DskipTest -Dmaven.test.skip=true 8 | -------------------------------------------------------------------------------- /genetic-algorithms/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.markwatson 5 | geneticalgorithm 6 | jar 7 | 1.0-SNAPSHOT 8 | geneticalgorithm 9 | http://maven.apache.org 10 | 11 | 1.8 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | org.apache.maven.plugins 21 | maven-compiler-plugin 22 | 2.3.2 23 | 24 | 1.8 25 | 1.8 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /genetic-algorithms/src/main/java/com/markwatson/geneticalgorithm/TestGenetic.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.geneticalgorithm; 2 | 3 | /** 4 | * This is an example of genetic algorithm test class. 5 | * 6 | * Copyright 1996-2020 Mark Watson. All Rights Reserved. Apache 2 license. 7 | * 8 | * For documentation see my book "Practical Artificial Intelligence Programming 9 | * With Java", chapter "Genetic Algorithms" 10 | * at https://leanpub.com/javaai that can be read for free online. 11 | * 12 | */ 13 | 14 | public class TestGenetic { 15 | 16 | static MyGenetic genetic_experiment; 17 | 18 | static public void main(String args[]) { 19 | // we will use chromosomes with 10 1 bit genes per 20 | // chromosomes, and a population of 12 chromosomes: 21 | genetic_experiment = new MyGenetic(10, 20, 0.85f, 0.3f); 22 | int geneIndex = 0; // debug only 23 | for (Chromosome ll : genetic_experiment.chromosomes) { 24 | System.out.println(ll.chromosome + " : " + genetic_experiment.geneToFloat(geneIndex++)); 25 | } 26 | int NUM_CYCLES = 15000; 27 | for (int i=0; i 3 | 4.0.0 4 | com.markwatson 5 | web_scraper 6 | jar 7 | 1.0-SNAPSHOT 8 | web_scraper 9 | 10 | http://maven.apache.org 11 | 12 | 8 13 | UTF-8 14 | UTF-8 15 | 16 | 17 | 18 | 19 | 20 | junit 21 | junit 22 | 4.13.1 23 | test 24 | 25 | 26 | org.jsoup 27 | jsoup 28 | 1.15.3 29 | 30 | 31 | org.apache.httpcomponents 32 | httpclient 33 | 4.5.13 34 | 35 | 36 | net.htmlparser.jericho 37 | jericho-html 38 | 3.4 39 | 40 | 41 | org.jdom 42 | jdom 43 | 1.1.3 44 | 45 | 46 | 47 | 48 | 49 | org.apache.maven.plugins 50 | maven-compiler-plugin 51 | 3.8.0 52 | 53 | 8 54 | 55 | 56 | 57 | org.apache.maven.plugins 58 | maven-surefire-plugin 59 | 2.22.0 60 | 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /info_gathering/src/main/java/com/markwatson/info_spiders/GeoNameData.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.info_spiders; 2 | 3 | import org.geonames.Toponym; 4 | 5 | /** 6 | * Copyright Mark Watson 2008-2020. All Rights Reserved. 7 | * License: Apache-2.0 8 | */ 9 | public class GeoNameData { 10 | public enum GeoType { 11 | CITY, COUNTRY, STATE, RIVER, MOUNTAIN, UNKNOWN 12 | } 13 | 14 | public int geoNameId = 0; 15 | public GeoType geoType = GeoType.UNKNOWN; 16 | public String name = ""; 17 | public double latitude = 0; 18 | public double longitude = 0; 19 | public String countryCode = ""; 20 | 21 | public GeoNameData(Toponym toponym) { 22 | geoNameId = toponym.getGeoNameId(); 23 | latitude = toponym.getLatitude(); 24 | longitude = toponym.getLongitude(); 25 | name = toponym.getName(); 26 | countryCode = toponym.getCountryCode(); 27 | if (toponym.getFeatureClassName().startsWith("city")) geoType = GeoType.CITY; 28 | if (toponym.getFeatureClassName().startsWith("country")) geoType = GeoType.COUNTRY; 29 | if (toponym.getFeatureClassName().startsWith("state")) geoType = GeoType.STATE; 30 | if (toponym.getFeatureClassName().startsWith("stream")) geoType = GeoType.RIVER; 31 | if (toponym.getFeatureClassName().startsWith("mountain")) geoType = GeoType.MOUNTAIN; 32 | } 33 | 34 | public GeoNameData() { 35 | } 36 | 37 | public String toString() { 38 | return "[GeoNameData: " + name + ", type: " + geoType + ", country code: " + countryCode + ", ID: " + geoNameId + ", latitude: " + latitude + ", longitude: " + longitude + "]"; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /info_gathering/src/main/java/com/markwatson/info_spiders/GeoNamesClient.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.info_spiders; 2 | 3 | import org.geonames.*; 4 | 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | /** 9 | * Copyright Mark Watson 2008-2020. All Rights Reserved. 10 | * License: Apache 2 11 | */ 12 | 13 | // You will need a free GeoNames account. Sign up: https://www.geonames.org/login 14 | // Then, set an environment variable: export GEONAMES=your-geonames-account-name 15 | 16 | public class GeoNamesClient { 17 | public GeoNamesClient() { 18 | } 19 | 20 | private List helper(String name, String type) throws Exception { 21 | List ret = new ArrayList(); 22 | 23 | String geonames_account_name = System.getenv("GEONAMES"); 24 | if (geonames_account_name == null) { 25 | System.err.println("You will need a free GeoNames account."); 26 | System.err.println("Sign up: https://www.geonames.org/login"); 27 | System.err.println("Then, set an environment variable:"); 28 | System.err.println(" export GEONAMES=your-geonames-account-name"); 29 | throw new Exception("Need API key"); 30 | } 31 | WebService.setUserName(System.getenv("GEONAMES")); 32 | 33 | ToponymSearchCriteria searchCriteria = new ToponymSearchCriteria(); 34 | searchCriteria.setStyle(Style.LONG); 35 | searchCriteria.setQ(name); 36 | ToponymSearchResult searchResult = WebService.search(searchCriteria); 37 | for (Toponym toponym : searchResult.getToponyms()) { 38 | //System.out.println("* " + toponym.getName() + " : " +toponym.getFeatureClassName()); 39 | if (toponym.getFeatureClassName() != null && 40 | toponym.getFeatureClassName().toString().indexOf(type) > -1 && 41 | toponym.getName().indexOf(name) > -1 && 42 | valid(toponym.getName())) { 43 | ret.add(new GeoNameData(toponym)); 44 | } 45 | } 46 | return ret; 47 | } 48 | 49 | private boolean valid(String str) { 50 | if (str.contains("0")) return false; 51 | if (str.contains("1")) return false; 52 | if (str.contains("2")) return false; 53 | if (str.contains("3")) return false; 54 | if (str.contains("4")) return false; 55 | if (str.contains("5")) return false; 56 | if (str.contains("6")) return false; 57 | if (str.contains("7")) return false; 58 | if (str.contains("8")) return false; 59 | return !str.contains("9"); 60 | } 61 | 62 | public List getCityData(String city_name) throws Exception { 63 | return helper(city_name, "city"); 64 | } 65 | 66 | public List getCountryData(String country_name) throws Exception { 67 | return helper(country_name, "country"); 68 | } 69 | 70 | public List getStateData(String state_name) throws Exception { 71 | List states = helper(state_name, "state"); 72 | for (GeoNameData state : states) { 73 | state.geoType = GeoNameData.GeoType.STATE; 74 | } 75 | return states; 76 | } 77 | 78 | public List getRiverData(String river_name) throws Exception { 79 | return helper(river_name, "stream"); 80 | } 81 | 82 | public List getMountainData(String mountain_name) throws Exception { 83 | return helper(mountain_name, "mountain"); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /info_gathering/src/main/java/com/markwatson/info_spiders/WebSpider.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.info_spiders; 2 | 3 | import net.htmlparser.jericho.*; 4 | 5 | import java.io.InputStream; 6 | import java.net.URL; 7 | import java.net.URLConnection; 8 | import java.util.*; 9 | 10 | /** 11 | * This simple web spider returns a list of lists, each containing two 12 | * strings representing "URL" and "text". Specifically, I do not return links on each page. 13 | */ 14 | 15 | /** 16 | * Copyright Mark Watson 2008-2020. All Rights Reserved. 17 | * License: Apache 2 18 | */ 19 | 20 | public class WebSpider { 21 | public WebSpider(String root_url, int max_returned_pages) throws Exception { 22 | String host = new URL(root_url).getHost(); 23 | System.out.println("+ host: " + host); 24 | List urls = new ArrayList(); 25 | Set already_visited = new HashSet(); 26 | urls.add(root_url); 27 | int num_fetched = 0; 28 | while (num_fetched <= max_returned_pages && !urls.isEmpty()) { 29 | try { 30 | System.out.println("+ urls: " + urls); 31 | String url_str = urls.remove(0); 32 | System.out.println("+ url_str: " + url_str); 33 | //if (url_str.toLowerCase().indexOf(host) > -1 && url_str.indexOf("https:") == -1 && !already_visited.contains(url_str)) { 34 | if (url_str.toLowerCase().indexOf(host) > -1 && !already_visited.contains(url_str)) { 35 | already_visited.add(url_str); 36 | URL url = new URL(url_str); 37 | URLConnection connection = url.openConnection(); 38 | connection.setAllowUserInteraction(false); 39 | InputStream ins = url.openStream(); 40 | Source source = new Source(ins); 41 | num_fetched++; 42 | TextExtractor te = new TextExtractor(source); 43 | String text = te.toString(); 44 | // Skip any pages where text on page is identical to existing 45 | // page (e.g., http://example.com and http://exaple.com/index.html 46 | boolean process = true; 47 | for (List ls : url_content_lists) { 48 | if (text.equals(ls.get(1))) { 49 | process = false; 50 | break; 51 | } 52 | } 53 | if (process) { 54 | try { 55 | Thread.sleep(500); 56 | } catch (Exception ignore) { 57 | } 58 | List anchorTags = source.getAllStartTags("a "); 59 | ListIterator iter = anchorTags.listIterator(); 60 | while (iter.hasNext()) { 61 | StartTag anchor = (StartTag) iter.next(); 62 | Attributes attr = anchor.parseAttributes(); 63 | Attribute link = attr.get("href"); 64 | String link_str = link.getValue(); 65 | if (link_str.indexOf("http:") == -1) { 66 | String path = url.getPath(); 67 | if (path.endsWith("/")) path = path.substring(0, path.length() - 1); 68 | int index = path.lastIndexOf("/"); 69 | if (index > -1) path = path.substring(0, index); 70 | link_str = url.getHost() + "/" + path + "/" + link_str; 71 | link_str = "http://" + link_str.replaceAll("///", "/").replaceAll("//", "/"); 72 | } 73 | urls.add(link_str); 74 | } 75 | List ls = new ArrayList(2); 76 | ls.add(url_str); 77 | ls.add(text); 78 | url_content_lists.add(ls); 79 | } 80 | } 81 | } catch (Exception ex) { 82 | System.out.println("Error: " + ex); 83 | ex.printStackTrace(); 84 | } 85 | } 86 | } 87 | 88 | public List> url_content_lists = new ArrayList>(); 89 | } 90 | -------------------------------------------------------------------------------- /info_gathering/src/main/java/com/markwatson/web_scraping/MySitesExamples.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.web_scraping; 2 | 3 | import org.jsoup.*; 4 | import org.jsoup.nodes.Document; 5 | import org.jsoup.nodes.Element; 6 | import org.jsoup.select.Elements; 7 | 8 | /** 9 | * Examples of using jsoup 10 | */ 11 | public class MySitesExamples { 12 | 13 | public static void main(String[] args) throws Exception { 14 | Document doc = Jsoup.connect("https://markwatson.com") 15 | .userAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10.0; rv:77.0) Gecko/20100101 Firefox/77.0") 16 | .timeout(2000).get(); 17 | Elements newsHeadlines = doc.select("div p"); 18 | for (Element element : newsHeadlines) { 19 | System.out.println(" next element text: " + element.text()); 20 | } 21 | String all_page_text = doc.text(); 22 | System.out.println("All text on web page:\n" + all_page_text); 23 | Elements anchors = doc.select("a[href]"); 24 | for (Element anchor : anchors) { 25 | String uri = anchor.attr("href"); 26 | System.out.println(" next anchor uri: " + uri); 27 | System.out.println(" next anchor text: " + anchor.text()); 28 | } 29 | Elements absolute_uri_anchors = doc.select("a[href]"); 30 | for (Element anchor : absolute_uri_anchors) { 31 | String uri = anchor.attr("abs:href"); 32 | System.out.println(" next anchor absolute uri: " + uri); 33 | System.out.println(" next anchor absolute text: " + anchor.text()); 34 | } 35 | 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /info_gathering/src/main/java/org/geonames/Address.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2008 Marc Wick, geonames.org 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package org.geonames; 18 | 19 | /** 20 | * a street address 21 | * 22 | * @author Mark Thomas 23 | * 24 | */ 25 | public class Address extends PostalCode { 26 | 27 | private String street; 28 | 29 | private String streetNumber; 30 | 31 | private String mtfcc; 32 | 33 | /** 34 | * @return the street 35 | */ 36 | public String getStreet() { 37 | return street; 38 | } 39 | 40 | /** 41 | * @param street 42 | * the street to set 43 | */ 44 | public void setStreet(String street) { 45 | this.street = street; 46 | } 47 | 48 | /** 49 | * @return the streetNumber 50 | */ 51 | public String getStreetNumber() { 52 | return streetNumber; 53 | } 54 | 55 | /** 56 | * @param streetNumber 57 | * the streetNumber to set 58 | */ 59 | public void setStreetNumber(String streetNumber) { 60 | this.streetNumber = streetNumber; 61 | } 62 | 63 | /** 64 | * @return the mtfcc 65 | */ 66 | public String getMtfcc() { 67 | return mtfcc; 68 | } 69 | 70 | /** 71 | * @param mtfcc 72 | * the mtfcc to set 73 | */ 74 | public void setMtfcc(String mtfcc) { 75 | this.mtfcc = mtfcc; 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /info_gathering/src/main/java/org/geonames/BoundingBox.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Marc Wick, geonames.org 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package org.geonames; 18 | 19 | /** 20 | * @author Marc Wick 21 | * @since 15.08.2012 22 | * 23 | */ 24 | public class BoundingBox { 25 | 26 | public BoundingBox(double west, double east, double south, double north) { 27 | setWest(west); 28 | setEast(east); 29 | setSouth(south); 30 | setNorth(north); 31 | } 32 | 33 | /** 34 | * @return the west 35 | */ 36 | public double getWest() { 37 | return west; 38 | } 39 | 40 | /** 41 | * @param west 42 | * the west to set 43 | */ 44 | public void setWest(double west) { 45 | this.west = west; 46 | } 47 | 48 | /** 49 | * @return the east 50 | */ 51 | public double getEast() { 52 | return east; 53 | } 54 | 55 | /** 56 | * @param east 57 | * the east to set 58 | */ 59 | public void setEast(double east) { 60 | this.east = east; 61 | } 62 | 63 | /** 64 | * @return the south 65 | */ 66 | public double getSouth() { 67 | return south; 68 | } 69 | 70 | /** 71 | * @param south 72 | * the south to set 73 | */ 74 | public void setSouth(double south) { 75 | this.south = south; 76 | } 77 | 78 | /** 79 | * @return the north 80 | */ 81 | public double getNorth() { 82 | return north; 83 | } 84 | 85 | /** 86 | * @param north 87 | * the north to set 88 | */ 89 | public void setNorth(double north) { 90 | this.north = north; 91 | } 92 | 93 | private double west; 94 | private double east; 95 | private double south; 96 | private double north; 97 | 98 | } 99 | -------------------------------------------------------------------------------- /info_gathering/src/main/java/org/geonames/FeatureClass.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2008 Marc Wick, geonames.org 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package org.geonames; 18 | 19 | /** 20 | * Enumeration for the GeoNames feature classes A,H,L,P,R,S,T,U,V 21 | * 22 | * @author marc 23 | * 24 | */ 25 | public enum FeatureClass { 26 | /** 27 | * Administrative Boundary Features 28 | */ 29 | A, 30 | /** 31 | * Hydrographic Features 32 | */ 33 | H, 34 | /** 35 | * Area Features 36 | */ 37 | L, 38 | /** 39 | * Populated Place Features 40 | */ 41 | P, 42 | /** 43 | * Road / Railroad Features 44 | */ 45 | R, 46 | /** 47 | * Spot Features 48 | */ 49 | S, 50 | /** 51 | * Hypsographic Features 52 | */ 53 | T, 54 | /** 55 | * Undersea Features 56 | */ 57 | U, 58 | /** 59 | * Vegetation Features 60 | */ 61 | V; 62 | 63 | public static FeatureClass fromValue(String value) { 64 | if (value == null || "".equals(value)) { 65 | return null; 66 | } 67 | return valueOf(value); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /info_gathering/src/main/java/org/geonames/GeoNamesException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Marc Wick, geonames.org 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package org.geonames; 18 | 19 | /** 20 | * @author marc 21 | * @since 20.01.2011 22 | * 23 | */ 24 | public class GeoNamesException extends Exception { 25 | 26 | private static final long serialVersionUID = 746586385626445380L; 27 | private String message; 28 | private int exceptionCode; 29 | 30 | public GeoNamesException(int exceptionCode, String msg) { 31 | super(msg); 32 | this.message = msg; 33 | this.exceptionCode = exceptionCode; 34 | } 35 | 36 | public GeoNamesException(String msg) { 37 | super(msg); 38 | this.message=msg; 39 | } 40 | 41 | /** 42 | * @return the message 43 | */ 44 | public String getMessage() { 45 | return message; 46 | } 47 | 48 | /** 49 | * @return the exceptionCode 50 | */ 51 | public int getExceptionCode() { 52 | return exceptionCode; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /info_gathering/src/main/java/org/geonames/InsufficientStyleException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2008 Marc Wick, geonames.org 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package org.geonames; 18 | 19 | /** 20 | * Is thrown when trying to access a field that has not been set as the style 21 | * for the request was not sufficiently verbose to return this information. 22 | * 23 | * @author marc 24 | * 25 | */ 26 | public class InsufficientStyleException extends GeoNamesException { 27 | public InsufficientStyleException(String msg) { 28 | super(msg); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /info_gathering/src/main/java/org/geonames/Intersection.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2008 Marc Wick, geonames.org 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package org.geonames; 18 | 19 | /** 20 | * an intersection between two streets 21 | * 22 | * @author Mark Thomas 23 | * 24 | */ 25 | public class Intersection { 26 | private String street2; 27 | 28 | private Address address = new Address(); 29 | 30 | public double getDistance() { 31 | return address.getDistance(); 32 | } 33 | 34 | public void setDistance(double d) { 35 | address.setDistance(d); 36 | } 37 | 38 | public String getAdminCode1() { 39 | return address.getAdminCode1(); 40 | } 41 | 42 | public void setAdminCode1(String s) { 43 | address.setAdminCode1(s); 44 | } 45 | 46 | public String getAdminName1() { 47 | return address.getAdminName1(); 48 | } 49 | 50 | public void setAdminName1(String s) { 51 | address.setAdminName1(s); 52 | } 53 | 54 | public String getAdminName2() { 55 | return address.getAdminName2(); 56 | } 57 | 58 | public void setAdminName2(String s) { 59 | address.setAdminName2(s); 60 | } 61 | 62 | public String getCountryCode() { 63 | return address.getCountryCode(); 64 | } 65 | 66 | public void setCountryCode(String s) { 67 | address.setCountryCode(s); 68 | } 69 | 70 | public double getLatitude() { 71 | return address.getLatitude(); 72 | } 73 | 74 | public void setLatitude(double d) { 75 | address.setLatitude(d); 76 | } 77 | 78 | public double getLongitude() { 79 | return address.getLongitude(); 80 | } 81 | 82 | public void setLongitude(double d) { 83 | address.setLongitude(d); 84 | } 85 | 86 | public String getPlaceName() { 87 | return address.getPlaceName(); 88 | } 89 | 90 | public void setPlaceName(String s) { 91 | address.setPlaceName(s); 92 | } 93 | 94 | public String getPostalCode() { 95 | return address.getPostalCode(); 96 | } 97 | 98 | public void setPostalCode(String s) { 99 | address.setPostalCode(s); 100 | } 101 | 102 | public String getStreet1() { 103 | return address.getStreet(); 104 | } 105 | 106 | public void setStreet1(String s) { 107 | address.setStreet(s); 108 | } 109 | 110 | public String getStreet2() { 111 | return street2; 112 | } 113 | 114 | public void setStreet2(String street2) { 115 | this.street2 = street2; 116 | } 117 | 118 | } 119 | -------------------------------------------------------------------------------- /info_gathering/src/main/java/org/geonames/InvalidParameterException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2008 Marc Wick, geonames.org 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package org.geonames; 18 | 19 | /** 20 | * is thrown when the search criteria is initialized with obviously invalid parameters, 21 | * such as an invalid country code. 22 | * 23 | * @author marc 24 | * 25 | */ 26 | public class InvalidParameterException extends GeoNamesException { 27 | 28 | public InvalidParameterException(String msg) { 29 | super(msg); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /info_gathering/src/main/java/org/geonames/StreetSegment.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2008 Marc Wick, geonames.org 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package org.geonames; 18 | 19 | /** 20 | * a street line segment. Includes house number information for the beginning 21 | * and end of the line as well as right and left hand side of the line. 22 | * 23 | * @author marc@geonames 24 | * 25 | */ 26 | public class StreetSegment extends PostalCode { 27 | 28 | private double[] latArray; 29 | 30 | private double[] lngArray; 31 | 32 | /** 33 | * census feature class codes see 34 | * http://www.geonames.org/maps/Census-Feature-Class-Codes.txt 35 | */ 36 | private String cfcc; 37 | 38 | private String name; 39 | 40 | /** 41 | * from address left 42 | */ 43 | private String fraddl; 44 | 45 | /** 46 | * from address right 47 | */ 48 | private String fraddr; 49 | 50 | /** 51 | * to address left 52 | */ 53 | private String toaddl; 54 | 55 | /** 56 | * to address right 57 | */ 58 | private String toaddr; 59 | 60 | /** 61 | * @return the latArray 62 | */ 63 | public double[] getLatArray() { 64 | return latArray; 65 | } 66 | 67 | /** 68 | * @param latArray 69 | * the latArray to set 70 | */ 71 | public void setLatArray(double[] latArray) { 72 | this.latArray = latArray; 73 | } 74 | 75 | /** 76 | * @return the lngArray 77 | */ 78 | public double[] getLngArray() { 79 | return lngArray; 80 | } 81 | 82 | /** 83 | * @param lngArray 84 | * the lngArray to set 85 | */ 86 | public void setLngArray(double[] lngArray) { 87 | this.lngArray = lngArray; 88 | } 89 | 90 | /** 91 | * @return the cfcc 92 | */ 93 | public String getCfcc() { 94 | return cfcc; 95 | } 96 | 97 | /** 98 | * @param cfcc 99 | * the cfcc to set 100 | */ 101 | public void setCfcc(String cfcc) { 102 | this.cfcc = cfcc; 103 | } 104 | 105 | /** 106 | * @return the name 107 | */ 108 | public String getName() { 109 | return name; 110 | } 111 | 112 | /** 113 | * @param name 114 | * the name to set 115 | */ 116 | public void setName(String name) { 117 | this.name = name; 118 | } 119 | 120 | /** 121 | * @return the fraddl 122 | */ 123 | public String getFraddl() { 124 | return fraddl; 125 | } 126 | 127 | /** 128 | * @param fraddl 129 | * the fraddl to set 130 | */ 131 | public void setFraddl(String fraddl) { 132 | this.fraddl = fraddl; 133 | } 134 | 135 | /** 136 | * @return the fraddr 137 | */ 138 | public String getFraddr() { 139 | return fraddr; 140 | } 141 | 142 | /** 143 | * @param fraddr 144 | * the fraddr to set 145 | */ 146 | public void setFraddr(String fraddr) { 147 | this.fraddr = fraddr; 148 | } 149 | 150 | /** 151 | * @return the toaddl 152 | */ 153 | public String getToaddl() { 154 | return toaddl; 155 | } 156 | 157 | /** 158 | * @param toaddl 159 | * the toaddl to set 160 | */ 161 | public void setToaddl(String toaddl) { 162 | this.toaddl = toaddl; 163 | } 164 | 165 | /** 166 | * @return the toaddr 167 | */ 168 | public String getToaddr() { 169 | return toaddr; 170 | } 171 | 172 | /** 173 | * @param toaddr 174 | * the toaddr to set 175 | */ 176 | public void setToaddr(String toaddr) { 177 | this.toaddr = toaddr; 178 | } 179 | 180 | } 181 | -------------------------------------------------------------------------------- /info_gathering/src/main/java/org/geonames/Style.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2008 Marc Wick, geonames.org 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package org.geonames; 19 | 20 | /** 21 | * Enumeration for style parameter specifying the verbosity of geonames web 22 | * services 23 | * 24 | * @author marc@geonames 25 | * 26 | */ 27 | public enum Style { 28 | SHORT, MEDIUM, LONG, FULL 29 | } -------------------------------------------------------------------------------- /info_gathering/src/main/java/org/geonames/Timezone.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2008 Marc Wick, geonames.org 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package org.geonames; 18 | 19 | import java.util.Date; 20 | 21 | /** 22 | * gmtOffset and dstOffset are computed on the server with the 23 | * {@link java.util.TimeZone} and included in the web service as not all 24 | * geonames users are using java. 25 | * 26 | * @author marc 27 | * 28 | */ 29 | public class Timezone { 30 | 31 | private String timezoneId; 32 | private String countryCode; 33 | private Date time; 34 | private Date sunrise; 35 | private Date sunset; 36 | 37 | @Deprecated 38 | private double gmtOffset; 39 | 40 | @Deprecated 41 | private double dstOffset; 42 | 43 | /** 44 | * the dstOffset as of first of July of current year 45 | * 46 | * @return the dstOffset 47 | */ 48 | @Deprecated 49 | public double getDstOffset() { 50 | return dstOffset; 51 | } 52 | 53 | /** 54 | * @param dstOffset 55 | * the dstOffset to set 56 | */ 57 | public void setDstOffset(double dstOffset) { 58 | this.dstOffset = dstOffset; 59 | } 60 | 61 | /** 62 | * the gmtOffset as of first of January of current year 63 | * 64 | * @return the gmtOffset 65 | */ 66 | @Deprecated 67 | public double getGmtOffset() { 68 | return gmtOffset; 69 | } 70 | 71 | /** 72 | * @param gmtOffset 73 | * the gmtOffset to set 74 | */ 75 | public void setGmtOffset(double gmtOffset) { 76 | this.gmtOffset = gmtOffset; 77 | } 78 | 79 | /** 80 | * the timezoneId (example : "Pacific/Honolulu") 81 | * 82 | * see also {@link java.util.TimeZone} and 83 | * http://www.twinsun.com/tz/tz-link.htm 84 | * 85 | * @return the timezoneId 86 | */ 87 | public String getTimezoneId() { 88 | return timezoneId; 89 | } 90 | 91 | /** 92 | * @param timezoneId 93 | * the timezoneId to set 94 | */ 95 | public void setTimezoneId(String timezoneId) { 96 | this.timezoneId = timezoneId; 97 | } 98 | 99 | /** 100 | * @return the countryCode 101 | */ 102 | public String getCountryCode() { 103 | return countryCode; 104 | } 105 | 106 | /** 107 | * @param countryCode 108 | * the countryCode to set 109 | */ 110 | public void setCountryCode(String countryCode) { 111 | this.countryCode = countryCode; 112 | } 113 | 114 | /** 115 | * @return the time 116 | */ 117 | public Date getTime() { 118 | return time; 119 | } 120 | 121 | /** 122 | * @param time 123 | * the time to set 124 | */ 125 | public void setTime(Date time) { 126 | this.time = time; 127 | } 128 | 129 | /** 130 | * @return the sunrise 131 | */ 132 | public Date getSunrise() { 133 | return sunrise; 134 | } 135 | 136 | /** 137 | * @param sunrise 138 | * the sunrise to set 139 | */ 140 | public void setSunrise(Date sunrise) { 141 | this.sunrise = sunrise; 142 | } 143 | 144 | /** 145 | * @return the sunset 146 | */ 147 | public Date getSunset() { 148 | return sunset; 149 | } 150 | 151 | /** 152 | * @param sunset 153 | * the sunset to set 154 | */ 155 | public void setSunset(Date sunset) { 156 | this.sunset = sunset; 157 | } 158 | 159 | } 160 | -------------------------------------------------------------------------------- /info_gathering/src/main/java/org/geonames/ToponymSearchResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2008 Marc Wick, geonames.org 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package org.geonames; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | /** 23 | * a toponym search result as returned by the geonames webservice. 24 | * 25 | * @author marc@geonames 26 | * 27 | */ 28 | public class ToponymSearchResult { 29 | 30 | List toponyms = new ArrayList(); 31 | 32 | int totalResultsCount; 33 | 34 | Style style; 35 | 36 | /** 37 | * @return Returns the toponyms. 38 | */ 39 | public List getToponyms() { 40 | return toponyms; 41 | } 42 | 43 | /** 44 | * @param toponyms 45 | * The toponyms to set. 46 | */ 47 | public void setToponyms(List toponyms) { 48 | this.toponyms = toponyms; 49 | } 50 | 51 | /** 52 | * @return Returns the totalResultsCount. 53 | */ 54 | public int getTotalResultsCount() { 55 | return totalResultsCount; 56 | } 57 | 58 | /** 59 | * @param totalResultsCount 60 | * The totalResultsCount to set. 61 | */ 62 | public void setTotalResultsCount(int totalResultsCount) { 63 | this.totalResultsCount = totalResultsCount; 64 | } 65 | 66 | /** 67 | * @return the style 68 | */ 69 | public Style getStyle() { 70 | return style; 71 | } 72 | 73 | /** 74 | * @param style the style to set 75 | */ 76 | public void setStyle(Style style) { 77 | this.style = style; 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /info_gathering/src/main/java/org/geonames/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | GeoNames Java API main classes. 11 | 12 |

Webservice Code Examples

13 | 14 |

geonames full text search

15 | 16 | 17 | Code Example : 18 | 
   
19 |   ToponymSearchCriteria searchCriteria = new ToponymSearchCriteria();
20 |   searchCriteria.setQ("zurich");
21 |   ToponymSearchResult searchResult = WebService.search(searchCriteria);
22 |   for (Toponym toponym : searchResult.toponyms) {
23 |      System.out.println(toponym.getName()+" "+ toponym.getCountryName());
24 |   }
25 |
26 | 27 | 28 |


29 | 30 | 31 |

Reverse Geocoding - find next postal codes for given latitude and longitude

32 | 33 | 34 | Code Example : 35 | 
   
36 |   PostalCodeSearchCriteria postalCodeSearchCriteria = new PostalCodeSearchCriteria();
37 |   postalCodeSearchCriteria.setLatitude(37.373636);
38 |   postalCodeSearchCriteria.setLongitude(-121.972146);
39 |   List postalCodes = WebService.findNearbyPostalCodes(postalCodeSearchCriteria);
40 |
41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /info_gathering/src/main/java/org/geonames/utils/Bearing.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2006 Marc Wick, geonames.org 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package org.geonames.utils; 18 | 19 | import static java.lang.Math.atan2; 20 | import static java.lang.Math.cos; 21 | import static java.lang.Math.sin; 22 | import static java.lang.Math.toDegrees; 23 | import static java.lang.Math.toRadians; 24 | 25 | /** 26 | * compass bearing from the first point to the second point in degrees. 27 | * 28 | * @author Marc Wick 29 | */ 30 | public class Bearing { 31 | 32 | /** 33 | * Returns the direction from the first point to the second point in 34 | * degrees. The direction is the clockwise angle between the magnetic north 35 | * and the direction from point1 to point2 36 | * 37 | * @param lat1 38 | * @param lng1 39 | * @param lat2 40 | * @param lng2 41 | * @return 42 | */ 43 | // http://www.movable-type.co.uk/scripts/latlong.html 44 | public static double calculateBearing(double lat1, double lng1, 45 | double lat2, double lng2) { 46 | double dLon = toRadians(lng2 - lng1); 47 | lat1 = toRadians(lat1); 48 | lat2 = toRadians(lat2); 49 | double y = sin(dLon) * cos(lat2); 50 | double x = cos(lat1) * sin(lat2) - sin(lat1) * cos(lat2) * cos(dLon); 51 | double brng = toDegrees(atan2(y, x)); 52 | return (brng + 360) % 360; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /info_gathering/src/main/java/org/geonames/utils/Distance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2006 Marc Wick, geonames.org 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package org.geonames.utils; 18 | 19 | /** 20 | * Distance calculations. 21 | * 22 | * @author marc@geonames 23 | * 24 | */ 25 | public class Distance { 26 | 27 | /** 28 | * mean radius = 6372.0 29 | * 30 | * The Earth's equatorial radius = 6335.437 km. 31 | * 32 | * The Earth's polar radius = 6399.592 km. 33 | * 34 | * 35 | */ 36 | public static final double EARTH_RADIUS_KM = 6372.0; 37 | 38 | /** 39 | * statute miles 40 | */ 41 | public static final double EARTH_RADIUS_MILES = 3963.0; 42 | 43 | /** 44 | * http://mathworld.wolfram.com/GreatCircle.html 45 | * 46 | * and 47 | * 48 | * http://www.mathforum.com/library/drmath/view/51711.html 49 | * 50 | * @return 51 | */ 52 | public static double distance(double lat1, double lng1, double lat2, 53 | double lng2, char unit, int numberOfDigits) { 54 | double a1 = Math.toRadians(lat1); 55 | double b1 = Math.toRadians(lng1); 56 | double a2 = Math.toRadians(lat2); 57 | double b2 = Math.toRadians(lng2); 58 | double d = Math.acos(Math.cos(a1) * Math.cos(b1) * Math.cos(a2) 59 | * Math.cos(b2) + Math.cos(a1) * Math.sin(b1) * Math.cos(a2) 60 | * Math.sin(b2) + Math.sin(a1) * Math.sin(a2)); 61 | 62 | double dist = 0; 63 | if (unit == 'M') { 64 | dist = d * EARTH_RADIUS_MILES; 65 | } else { 66 | dist = d * EARTH_RADIUS_KM; 67 | } 68 | 69 | if (Double.isNaN(dist)) { 70 | // use pytagoras for very small distances, 71 | dist = Math.sqrt(Math.pow(Math.abs(lat1 - lat2), 2) 72 | + Math.pow(Math.abs(lng1 - lng2), 2)); 73 | // as rule of thumb multiply with 110km =1 degree 74 | if (unit == 'M') { 75 | dist *= 69; 76 | } else { 77 | dist *= 110; 78 | } 79 | } 80 | 81 | if (numberOfDigits == 0) { 82 | dist = (int) dist; 83 | } else if (numberOfDigits > 0) { 84 | double factor = Math.pow(10, numberOfDigits); 85 | dist = Math.floor(dist * factor) / factor; 86 | } 87 | return dist; 88 | } 89 | 90 | public static double distanceKM(double lat1, double lng1, double lat2, 91 | double lng2) { 92 | return distance(lat1, lng1, lat2, lng2, 'K', 3); 93 | } 94 | 95 | public static double distanceMiles(double lat1, double lng1, double lat2, 96 | double lng2) { 97 | return distance(lat1, lng1, lat2, lng2, 'M', 3); 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /info_gathering/src/main/java/org/geonames/utils/DistanceOrder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Marc Wick, geonames.org 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package org.geonames.utils; 18 | 19 | import java.util.Comparator; 20 | 21 | import org.geonames.WikipediaArticle; 22 | 23 | /** 24 | * @author marc 25 | * @since 16.02.2011 26 | * 27 | */ 28 | public class DistanceOrder implements Comparator { 29 | private double latitude; 30 | private double longitude; 31 | 32 | public DistanceOrder(double pLat, double pLng) { 33 | latitude = pLat; 34 | longitude = pLng; 35 | } 36 | 37 | public int compare(WikipediaArticle o1, WikipediaArticle o2) { 38 | double dist1 = dist(o1); 39 | double dist2 = dist(o2); 40 | return Double.compare(dist1, dist2); 41 | } 42 | 43 | private double dist(WikipediaArticle o) { 44 | return Distance.distanceKM(latitude, longitude, o.getLatitude(), o 45 | .getLongitude()); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /info_gathering/src/main/java/org/geonames/utils/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | Utility classes for GeoNames 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /info_gathering/src/test/java/com/markwatson/info_spiders/GeoNamesClientTest.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.info_spiders; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for NLP. 9 | */ 10 | public class GeoNamesClientTest extends TestCase { 11 | 12 | public GeoNamesClientTest(String testName) { 13 | super(testName); 14 | } 15 | 16 | public static Test suite() { 17 | return new TestSuite(GeoNamesClientTest.class); 18 | } 19 | 20 | public void testFetchAndDisplay() throws Exception { 21 | assertTrue(true); 22 | GeoNamesClient client = new GeoNamesClient(); 23 | System.out.println(client.getCityData("Paris")); pause(); 24 | System.out.println(client.getCountryData("Canada")); pause(); 25 | System.out.println(client.getStateData("California")); pause(); 26 | System.out.println(client.getRiverData("Amazon")); pause(); 27 | System.out.println(client.getMountainData("Whitney")); 28 | } 29 | private static void pause() { 30 | try { Thread.sleep(2000); 31 | } catch (Exception ignore) { } 32 | } 33 | } 34 | 35 | -------------------------------------------------------------------------------- /info_gathering/src/test/java/com/markwatson/info_spiders/WebClientTest.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.info_spiders; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for NLP. 9 | */ 10 | public class WebClientTest extends TestCase { 11 | 12 | public WebClientTest(String testName) { 13 | super(testName); 14 | } 15 | 16 | public static Test suite() { 17 | return new TestSuite(WebClientTest.class); 18 | } 19 | 20 | public void testFetchAndDisplay() throws Exception { 21 | assertTrue(true); 22 | WebSpider client = new WebSpider("https://markwatson.com", 10); 23 | //WebSpider client = new WebSpider("http://pbs.org", 10); 24 | //WebSpider client = new WebSpider("http://kbsportal.com", 10); 25 | System.out.println("Found URIs: " + client.url_content_lists); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /kgc/.gitignore: -------------------------------------------------------------------------------- 1 | output.rdf 2 | 3 | -------------------------------------------------------------------------------- /kgc/Makefile: -------------------------------------------------------------------------------- 1 | create_data_and_remove_duplicates: install 2 | mvn test -q 3 | echo "Removing duplicate RDF statements" 4 | awk '!visited[$$0]++' output_with_duplicates.rdf > output.rdf 5 | rm -f output_with_duplicates.rdf 6 | 7 | 8 | install: 9 | mvn install -DskipTest -Dmaven.test.skip=true -q 10 | 11 | clean: 12 | rm -r -f target output.rdf output_with_duplicates.rdf -------------------------------------------------------------------------------- /kgc/README.md: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | 3 | Run make in the directory Java-AI-Book-Code/ner_dbpedia before building this example. 4 | 5 | -------------------------------------------------------------------------------- /kgc/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.knowledgegraphcreator 5 | kgcreator 6 | 1.0-SNAPSHOT 7 | 8 | 11.0 9 | UTF-8 10 | UTF-8 11 | 12 | 13 | 14 | org.apache.opennlp 15 | opennlp-tools 16 | 2.3.2 17 | 18 | 19 | com.markwatson 20 | nerdbpedia 21 | 1.2-SNAPSHOT 22 | 23 | 24 | junit 25 | junit 26 | 4.13.1 27 | test 28 | 29 | 30 | 31 | 32 | 33 | maven-compiler-plugin 34 | 3.8.0 35 | 36 | 11 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /kgc/src/test/java/com.knowledgegraphcreator/KgcTest.java: -------------------------------------------------------------------------------- 1 | package com.knowledgegraphcreator; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | public class KgcTest extends TestCase { 8 | 9 | public KgcTest(String testName) { 10 | super(testName); 11 | } 12 | 13 | public static Test suite() { 14 | return new TestSuite(KgcTest.class); 15 | } 16 | 17 | public void testKGC() throws Exception { 18 | assertTrue(true); 19 | KGC client = new KGC("test_data/", "output_with_duplicates.rdf"); 20 | } 21 | private static void pause() { 22 | try { Thread.sleep(2000); 23 | } catch (Exception ignore) { } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /kgc/test_data/CREDITS: -------------------------------------------------------------------------------- 1 | The sample news story in test3.txt was taken from an ABC News story. 2 | 3 | Many thanks to ABC News for the use of a paragraph of their text from a news story 4 | in file test3.txt 5 | 6 | -------------------------------------------------------------------------------- /kgc/test_data/test1.meta: -------------------------------------------------------------------------------- 1 | https://newsshop.com/may/a1023.html -------------------------------------------------------------------------------- /kgc/test_data/test1.txt: -------------------------------------------------------------------------------- 1 | Plunging European stocks, wobbly bonds and grave concerns about 2 | the health of Portuguese lender Banco Espiritomade last 3 | week feel like a rerun of the euro crisis according to commentator John Evans, but most investors, like Jill Hines, say 4 | it was no more than a blip for a resurgent region. Banco Espirito 5 | has been in investors’ sights since December, when The Wall 6 | Street Journal first reported on accounting irregularities at the 7 | complex firm. Nerves frayed on Thursday when Banco Espirito 8 | parent company said it wouldn't be able to meet some short-term debt 9 | obligations. Ben Cole gave a concert at IBM headquarters in Canada and then in France. 10 | I heard him on the Australian Broadcasting Corporation being 11 | critical of Australian Broadcasting Corporation. 12 | Story was written by Frank Smith a member of the Australian Writers Guild 13 | as taught at the American University. 14 | -------------------------------------------------------------------------------- /kgc/test_data/test2.meta: -------------------------------------------------------------------------------- 1 | https://localnews.com/june/z902.html -------------------------------------------------------------------------------- /kgc/test_data/test2.txt: -------------------------------------------------------------------------------- 1 | The Wall Street Journal covered the stock market crash in Mexico and Canada, troubling President Bill Clinton. 2 | Stock prices, especially IBM and Microsoft, are increasing. The weather was bad yesterday. 3 | -------------------------------------------------------------------------------- /kgc/test_data/test3.meta: -------------------------------------------------------------------------------- 1 | https://abcnews.go.com/US/violent-long-lasting-tornadoes-threaten-oklahoma-texas/story?id=63146361 -------------------------------------------------------------------------------- /kgc/test_data/test3.txt: -------------------------------------------------------------------------------- 1 | Weather forcaster Jane Deerborn (who is also a member of the Screen Actors Guild) said that over the weekend, 41 tornadoes were reported across eight states as severe storms again target the heartland. 2 | Violent, potentially life-threatening tornadoes may strike western Texas and Oklahoma as the threat of severe storms stretches all the way into Kansas. 3 | Six states also are under flood alerts, with flash flooding a major threat from Texas all the way up to Fargo North Dakota. 4 | Some areas in Texas, Oklahoma and Kansas could see half a foot of rain. 5 | Part of the system that delivered severe weather to the central U.S. over the weekend is moving into the Northeast today, producing strong to severe storms -- damaging winds, hail or isolated tornadoes can't be ruled out. 6 | The biggest threats will be in from the Hudson Valley toward Albany, New York, and into New England. 7 | A performance by DOA had to be cancelled according to a report on PTL Satellite Network. 8 | Severe weather is forecast to continue on Tuesday, with the western storm moving east into the Midwest and parts of the mid-Mississippi Valley. 9 | The biggest threat tomorrow will be damaging winds, large hail and a few tornadoes -- with the tornado threat largest in the morning. 10 | The storm was studied at Brandeis University. -------------------------------------------------------------------------------- /kgn/.gitignore: -------------------------------------------------------------------------------- 1 | *.db 2 | *.log 3 | 4 | -------------------------------------------------------------------------------- /kgn/Makefile: -------------------------------------------------------------------------------- 1 | main: 2 | mvn install ; java -jar target/KGN/KGN.jar 3 | 4 | 5 | install: 6 | mvn install -DskipTest -Dmaven.test.skip=true 7 | -------------------------------------------------------------------------------- /kgn/README.md: -------------------------------------------------------------------------------- 1 | This exmample will only work when run as a junit test or in an IDE. 2 | 3 | Trying to run using: 4 | 5 | mvn exec:java -Dexec.mainClass="com.knowledgegraphnavigator.KGN" 6 | 7 | will not work because "mvn exec" works by only having the 8 | jar file plexus-classworlds-2.6.0.jar (or some other version) 9 | on the CLASSPATH. This prevents the dependency 10 | 11 | 12 | com.markwatson 13 | nerdbpedia 14 | 1.1-SNAPSHOT 15 | 16 | 17 | from a previous chapter from being able to load entity definition 18 | text files from its packaged jar file. 19 | 20 | 21 | -------------------------------------------------------------------------------- /kgn/dependency-reduced-pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | com.knowledgegraphnavigator 5 | kgn 6 | 1.0-SNAPSHOT 7 | 8 | 9 | 10 | maven-compiler-plugin 11 | 3.8.0 12 | 13 | 11 14 | 15 | 16 | 17 | maven-jar-plugin 18 | 3.2.0 19 | 20 | 21 | 22 | com.knowledgegraphnavigator.KGN 23 | 24 | 25 | 26 | 27 | 28 | maven-shade-plugin 29 | 2.4.3 30 | 31 | 32 | package 33 | 34 | shade 35 | 36 | 37 | 38 | 39 | 40 | com.knowledgegraphnavigator.KGN 41 | 1.0 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | org.apache.jena 54 | apache-jena-libs 55 | 3.15.0 56 | pom 57 | compile 58 | 59 | 60 | junit 61 | junit 62 | 3.8.1 63 | test 64 | 65 | 66 | 67 | UTF-8 68 | 11.0 69 | UTF-8 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /kgn/src/main/java/com/knowledgegraphnavigator/EntityAndDescription.java: -------------------------------------------------------------------------------- 1 | package com.knowledgegraphnavigator; 2 | 3 | public class EntityAndDescription { 4 | public String entityName; 5 | public String entityUri; 6 | public EntityAndDescription(String entityName, String entityUri) { 7 | this.entityName = entityName; 8 | this.entityUri = entityUri; 9 | } 10 | public String toString() { 11 | return "[EntityAndDescription name: " + entityName + 12 | " description: " + entityUri + "]"; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /kgn/src/main/java/com/knowledgegraphnavigator/EntityRelationships.java: -------------------------------------------------------------------------------- 1 | package com.knowledgegraphnavigator; 2 | 3 | import com.markwatson.semanticweb.QueryResult; 4 | 5 | import java.sql.SQLException; 6 | 7 | public class EntityRelationships { 8 | 9 | static public QueryResult results(Sparql endpoint, 10 | String entity1Uri, String entity2Uri) 11 | throws SQLException, ClassNotFoundException { 12 | String query = 13 | String.format("select ?p where { %s ?p %s . FILTER (!regex(str(?p), 'wikiPage', 'i')) } limit 10", 14 | entity1Uri, entity2Uri); 15 | return endpoint.query(query); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /kgn/src/main/java/com/knowledgegraphnavigator/Log.java: -------------------------------------------------------------------------------- 1 | package com.knowledgegraphnavigator; 2 | 3 | public class Log { 4 | static public void out(String s) { System.out.println(s); } 5 | static public StringBuilder sparql = new StringBuilder(); 6 | static public void clearSparql() { sparql.delete(0, sparql.length()); } 7 | } 8 | -------------------------------------------------------------------------------- /kgn/src/main/java/com/knowledgegraphnavigator/PrintEntityResearchResults.java: -------------------------------------------------------------------------------- 1 | package com.knowledgegraphnavigator; 2 | 3 | import static com.knowledgegraphnavigator.Log.out; 4 | import static com.knowledgegraphnavigator.Utils.removeBrackets; 5 | 6 | import java.sql.SQLException; 7 | import java.util.List; 8 | 9 | public class PrintEntityResearchResults { 10 | 11 | /** 12 | * Note for Windows users: the Windows console may not render the following 13 | * ANSI terminal escape sequences correctly. If yo have problems, just 14 | * change the following to the empty string "": 15 | */ 16 | public static final String RESET = "\u001B[0m"; // ANSI characters for styling 17 | public static final String GREEN = "\u001B[32m"; 18 | public static final String YELLOW = "\u001B[33m"; 19 | public static final String PURPLE = "\u001B[35m"; 20 | public static final String CYAN = "\u001B[36m"; 21 | 22 | private PrintEntityResearchResults() { } 23 | 24 | public PrintEntityResearchResults(Sparql endpoint, 25 | List people, 26 | List companies, 27 | List cities, 28 | List countries) 29 | throws SQLException, ClassNotFoundException { 30 | out("\n" + GREEN + "Individual People:\n" + RESET); 31 | for (EntityAndDescription person : people) { 32 | out(" " + GREEN + String.format("%-25s", person.entityName) + 33 | PURPLE + " : " + removeBrackets(person.entityUri) + RESET); 34 | out(EntityDetail.personAsString(endpoint, person.entityUri)); 35 | } 36 | out("\n" + CYAN + "Individual Companies:\n" + RESET); 37 | for (EntityAndDescription company : companies) { 38 | out(" " + CYAN + String.format("%-25s", company.entityName) + 39 | YELLOW + " : " + removeBrackets(company.entityUri) + RESET); 40 | out(EntityDetail.companyAsString(endpoint, company.entityUri)); 41 | } 42 | out("\n" + GREEN + "Individual Cities:\n" + RESET); 43 | for (EntityAndDescription city : cities) { 44 | out(" " + GREEN + String.format("%-25s", city.entityName) + 45 | PURPLE + " : " + removeBrackets(city.entityUri) + RESET); 46 | out(EntityDetail.cityAsString(endpoint, city.entityUri)); 47 | } 48 | out("\n" + GREEN + "Individual Countries:\n" + RESET); 49 | for (EntityAndDescription country : countries) { 50 | out(" " + GREEN + String.format("%-25s", country.entityName) + 51 | PURPLE + " : " + removeBrackets(country.entityUri) + RESET); 52 | out(EntityDetail.countryAsString(endpoint, country.entityUri)); 53 | } 54 | out(""); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /kgn/src/main/java/com/knowledgegraphnavigator/Sparql.java: -------------------------------------------------------------------------------- 1 | package com.knowledgegraphnavigator; 2 | 3 | import com.markwatson.semanticweb.QueryResult; 4 | import com.markwatson.semanticweb.JenaApis; 5 | import static com.knowledgegraphnavigator.Log.sparql; 6 | import static com.knowledgegraphnavigator.Log.out; 7 | 8 | import java.sql.SQLException; 9 | 10 | public class Sparql { 11 | //static private String endpoint = "https://query.wikidata.org/bigdata/namespace/wdq/sparql"; 12 | static private String endpoint = "https://dbpedia.org/sparql"; 13 | public Sparql() { 14 | this.jenaApis = new JenaApis(); 15 | } 16 | 17 | public QueryResult query(String sparqlQuery) throws SQLException, ClassNotFoundException { 18 | //out(sparqlQuery); // debug for now... 19 | sparql.append(sparqlQuery); 20 | sparql.append(("\n\n")); 21 | return jenaApis.queryRemote(endpoint, sparqlQuery); 22 | } 23 | private JenaApis jenaApis; 24 | 25 | public static void main(String[] args) throws Exception { 26 | Sparql sp = new Sparql(); 27 | QueryResult qr = sp.query("select ?s ?p ?o where { ?s ?p ?o } limit 5"); 28 | out(qr.toString()); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /kgn/src/main/java/com/knowledgegraphnavigator/Utils.java: -------------------------------------------------------------------------------- 1 | package com.knowledgegraphnavigator; 2 | 3 | public class Utils { 4 | static public String removeBrackets(String s) { 5 | if (s.startsWith("<")) return s.substring(1, s.length() - 1); 6 | return s; 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /langchain4j-ollama/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .codegpt 3 | 4 | -------------------------------------------------------------------------------- /langchain4j-ollama/Makefile: -------------------------------------------------------------------------------- 1 | run: 2 | mvn test -q # run test in quiet mode 3 | 4 | -------------------------------------------------------------------------------- /langchain4j-ollama/README.md: -------------------------------------------------------------------------------- 1 | # WORK IN PROGRESS: LangChain4j Ollama Local LLM REST API Example 2 | 3 | Make sure Ollama serve is running: 4 | 5 | ollama serve 6 | 7 | Use the make file default target to run the test: 8 | 9 | make -------------------------------------------------------------------------------- /langchain4j-ollama/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.markwatson.langchain4j 5 | langchain4j-llm-client 6 | jar 7 | 1.0-SNAPSHOT 8 | langchain4j-llm-client 9 | http://maven.apache.org 10 | 11 | 12 | dev.langchain4j 13 | langchain4j 14 | 0.30.0 15 | 16 | 17 | 18 | dev.langchain4j 19 | langchain4j-open-ai 20 | 0.30.0 21 | 22 | 23 | 24 | junit 25 | junit 26 | 3.8.1 27 | test 28 | 29 | 30 | org.json 31 | json 32 | 20240303 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /langchain4j-ollama/src/main/java/com/markwatson/langchain4j_ollama/OllamaLlmLangChain4j.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.langchain4j_ollama; 2 | 3 | import dev.langchain4j.model.chat.ChatLanguageModel; 4 | import dev.langchain4j.model.openai.OpenAiChatModel; 5 | 6 | 7 | import java.io.BufferedReader; 8 | import java.io.IOException; 9 | import java.io.InputStreamReader; 10 | import java.io.OutputStream; 11 | import java.net.HttpURLConnection; 12 | import java.net.URI; 13 | import java.net.URL; 14 | import java.net.URLConnection; 15 | import java.nio.file.Files; 16 | import java.nio.file.Path; 17 | import java.nio.file.Paths; 18 | 19 | import org.json.JSONObject; 20 | 21 | public class OllamaLlmLangChain4j { 22 | 23 | public static void main(String[] args) throws Exception { 24 | String prompt = "Translate the following English text to French: 'Hello, how are you?'"; 25 | String completion = getCompletion(prompt, "mistral"); 26 | System.out.println("completion: " + completion); 27 | } 28 | 29 | public static String getCompletion(String prompt, String modelName) throws Exception { 30 | System.out.println("\n\n**********\n\nprompt: " + prompt + ", modelName: " + modelName); 31 | String api_key = System.getenv("OPENAI_API_KEY"); 32 | 33 | ChatLanguageModel model = OpenAiChatModel.withApiKey(api_key); 34 | String answer = model.generate(prompt); 35 | 36 | System.out.println(answer); // Hello! How can I assist you today? 37 | return answer; 38 | } 39 | 40 | /*** 41 | * Utilities for using the Ollama LLM APIs 42 | */ 43 | 44 | // read the contents of a file path into a Java string 45 | public static String readFileToString(String filePath) throws IOException { 46 | Path path = Paths.get(filePath); 47 | return new String(Files.readAllBytes(path)); 48 | } 49 | 50 | public static String replaceSubstring(String originalString, String substringToReplace, String replacementString) { 51 | return originalString.replace(substringToReplace, replacementString); 52 | } 53 | public static String promptVar(String prompt0, String varName, String varValue) { 54 | String prompt = replaceSubstring(prompt0, varName, varValue); 55 | return replaceSubstring(prompt, varName, varValue); 56 | } 57 | } 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /langchain4j-ollama/src/test/java/com/markwatson/langchain4j_ollama/OllamaLlmLangChain4jTest.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.langchain4j_ollama; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class OllamaLlmLangChain4jTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public OllamaLlmLangChain4jTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( OllamaLlmLangChain4jTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | * @throws Exception 34 | */ 35 | public void testCompletion() throws Exception { 36 | String r = 37 | OllamaLlmLangChain4j.getCompletion("Translate the following English text to French: 'Hello, how are you?'", "llama3.2:latest"); 38 | System.out.println("\n\n&&&&&&&&&&\n\ncompletion: " + r); 39 | assertTrue( true ); 40 | } 41 | 42 | public void testTwoShotTemplate() throws Exception { 43 | String input_text = "Mark Smith enjoys living in Berkeley California at 102 Dunston Street and use mjess@foobar.com for contacting him."; 44 | String prompt0 = OllamaLlmLangChain4j.readFileToString("../prompts/two-shot-2-var.txt"); 45 | System.out.println("prompt0: " + prompt0); 46 | String prompt = OllamaLlmLangChain4j.promptVar(prompt0, "{input_text}", input_text); 47 | System.out.println("prompt: " + prompt); 48 | String r = 49 | OllamaLlmLangChain4j.getCompletion(prompt, "llama3:instruct"); 50 | System.out.println("two shot extraction completion: " + r); 51 | assertTrue( true ); 52 | } 53 | 54 | public void testSummarization() throws Exception { 55 | String input_text = "Jupiter is the fifth planet from the Sun and the largest in the Solar System. It is a gas giant with a mass one-thousandth that of the Sun, but two-and-a-half times that of all the other planets in the Solar System combined. Jupiter is one of the brightest objects visible to the naked eye in the night sky, and has been known to ancient civilizations since before recorded history. It is named after the Roman god Jupiter.[19] When viewed from Earth, Jupiter can be bright enough for its reflected light to cast visible shadows,[ and is on average the third-brightest natural object in the night sky after the Moon and Venus."; 56 | String prompt0 = OllamaLlmLangChain4j.readFileToString("../prompts/summarization_prompt.txt"); 57 | System.out.println("prompt0: " + prompt0); 58 | String prompt = OllamaLlmLangChain4j.promptVar(prompt0, "{input_text}", input_text); 59 | System.out.println("prompt: " + prompt); 60 | String r = 61 | OllamaLlmLangChain4j.getCompletion(prompt, "llama3:instruct"); 62 | System.out.println("summarization completion: " + r); 63 | assertTrue( true ); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /ner_dbpedia/Makefile: -------------------------------------------------------------------------------- 1 | test: install 2 | mvn test -q 3 | 4 | install: 5 | mvn install -DskipTest -Dmaven.test.skip=true -q 6 | 7 | -------------------------------------------------------------------------------- /ner_dbpedia/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.markwatson 8 | nerdbpedia 9 | 1.2-SNAPSHOT 10 | 11 | 11.0 12 | UTF-8 13 | UTF-8 14 | 15 | 16 | 17 | 18 | junit 19 | junit 20 | 4.13.1 21 | test 22 | 23 | 24 | 25 | 26 | 27 | org.apache.maven.plugins 28 | maven-compiler-plugin 29 | 3.3 30 | 31 | 1.8 32 | 1.8 33 | 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /ner_dbpedia/src/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mark-watson/Java-AI-Book-Code/f62e1231f4cf2de4641ed01518334696d4d07ff0/ner_dbpedia/src/.DS_Store -------------------------------------------------------------------------------- /ner_dbpedia/src/main/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mark-watson/Java-AI-Book-Code/f62e1231f4cf2de4641ed01518334696d4d07ff0/ner_dbpedia/src/main/.DS_Store -------------------------------------------------------------------------------- /ner_dbpedia/src/main/resources/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mark-watson/Java-AI-Book-Code/f62e1231f4cf2de4641ed01518334696d4d07ff0/ner_dbpedia/src/main/resources/.DS_Store -------------------------------------------------------------------------------- /ner_dbpedia/src/main/resources/META-INF/file-manefest-for-text-resource-files.txt: -------------------------------------------------------------------------------- 1 | BroadcastNetworkNamesDbPedia.txt 2 | PeopleDbPedia.txt 3 | CityNamesDbpedia.txt 4 | PoliticalPartyNamesDbPedia.txt 5 | CompanyNamesDbPedia.txt 6 | TradeUnionNamesDbPedia.txt 7 | CountryNamesDbpedia.txt 8 | UniversityNamesDbPedia.txt 9 | MusicGroupNamesDbPedia.txt 10 | -------------------------------------------------------------------------------- /ner_dbpedia/src/test/java/com/markwatson/ner_dbpedia/TextToDbpediaUrisTest.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.ner_dbpedia; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | public class TextToDbpediaUrisTest extends TestCase { 8 | /** 9 | * Create the test case 10 | * 11 | * @param testName name of the test case 12 | */ 13 | public TextToDbpediaUrisTest(String testName) 14 | { 15 | super( testName ); 16 | } 17 | 18 | /** 19 | * @return the suite of tests being tested 20 | */ 21 | public static Test suite() 22 | { 23 | return new TestSuite( TextToDbpediaUrisTest.class ); 24 | } 25 | 26 | /** 27 | * Test that is just for side effect printouts: 28 | */ 29 | public void test1() throws Exception { 30 | String s = "PTL Satellite Network covered President Bill Clinton going to Guatemala and visiting the Coca Cola Company."; 31 | TextToDbpediaUris test = new TextToDbpediaUris(s); 32 | System.out.println(test); 33 | assert(true); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /neuralnetworks/.gitignore: -------------------------------------------------------------------------------- 1 | test.neural 2 | 3 | -------------------------------------------------------------------------------- /neuralnetworks/Makefile: -------------------------------------------------------------------------------- 1 | all_examples: 1H 2H 2H_momentum 2 | 3 | 1H: 4 | mvn install -DskipTests -q 5 | mvn test -Dtest=NeuralNetwork_1H_Test -q 6 | 7 | 2H: 8 | mvn install -DskipTests -q 9 | mvn test -Dtest=NeuralNetwork_2H_Test -q 10 | 11 | 12 | 2H_momentum: 13 | mvn install -DskipTests -q 14 | mvn test -Dtest=NeuralNetwork_2H_momentum_Test -q 15 | 16 | 17 | install: 18 | mvn install -DskipTest -Dmaven.test.skip=true -q 19 | -------------------------------------------------------------------------------- /neuralnetworks/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | NeuralNetworksFromScratch 8 | neuralnetworks 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 13 | junit 14 | junit 15 | 4.13.1 16 | test 17 | 18 | 19 | 20 | 21 | 22 | org.apache.maven.plugins 23 | maven-compiler-plugin 24 | 3.3 25 | 26 | 1.8 27 | 1.8 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /neuralnetworks/src/main/java/com/markwatson/neuralnetworks/Graph.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.neuralnetworks; 2 | 3 | import javax.swing.*; 4 | import java.awt.*; 5 | 6 | /** 7 | * Title: Graph

8 | * Description: Simple program to produce a graph for the book

9 | * Copyright: Copyright (c) Mark Watson

10 | * Company:

11 | * @author Mark Watson 12 | * @version 1.0 13 | */ 14 | 15 | 16 | public class Graph extends JFrame { 17 | GraphPanel jPanel1; 18 | float[] data1; 19 | float[] data2; 20 | 21 | public Graph() { 22 | try { 23 | int size = 500; 24 | data1 = new float[size]; 25 | data2 = new float[size]; 26 | float xmin = -5; 27 | float xmax = 5; 28 | for (int i = 0; i < size; i++) { 29 | float x = i; 30 | x = xmin + x * (xmax - xmin) / (float) size; 31 | data1[i] = sigmoid(x); 32 | data2[i] = sigmoidP(x); 33 | } 34 | jbInit(); 35 | } catch (Exception e) { 36 | e.printStackTrace(); 37 | } 38 | } 39 | 40 | protected float sigmoid(float x) { 41 | return 42 | (float) (1.0f / (1.0f + Math.exp((double) (-x)))); 43 | // (float)((1.0f/(1.0f+Math.exp((double)(-x))))-0.5f); 44 | } 45 | 46 | protected float sigmoidP(float x) { 47 | double z = sigmoid(x); // + 0.5f; 48 | return (float) (z * (1.0f - z)); 49 | } 50 | 51 | 52 | public static void main(String[] args) { 53 | Graph untitled11 = new Graph(); 54 | } 55 | 56 | private void jbInit() throws Exception { 57 | jPanel1 = new GraphPanel(data1, data2); 58 | jPanel1.setBackground(Color.white); 59 | this.setDefaultCloseOperation(3); 60 | this.getContentPane().add(jPanel1, BorderLayout.CENTER); 61 | setSize(550, 300); 62 | jPanel1.setVisible(true); 63 | setVisible(true); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /neuralnetworks/src/main/java/com/markwatson/neuralnetworks/GraphPanel.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.neuralnetworks; 2 | 3 | 4 | /** 5 | * Title:

6 | * Description:

7 | * Copyright: Copyright (c)

8 | * Company:

9 | * @author 10 | * @version 1.0 11 | */ 12 | 13 | import javax.swing.*; 14 | import java.awt.*; 15 | 16 | public class GraphPanel extends java.awt.Canvas { // JPanel { 17 | 18 | public GraphPanel(float[] data1, float[] data2) { 19 | super(); 20 | this.data1 = data1; 21 | this.data2 = data2; 22 | } 23 | 24 | Color black = new Color(0, 0, 0); 25 | float[] data1; 26 | float[] data2; 27 | 28 | public void paint(Graphics g) { 29 | if (data1 == null || data2 == null) return; 30 | int width = this.getWidth(); 31 | int height = this.getHeight(); 32 | System.out.println("height=" + height); 33 | float min = 99999999.9f; 34 | float max = -min; 35 | int maxindex = 0; 36 | float maxval = 0.0f; 37 | for (int i = 0; i < data1.length; i++) { // assume length of data1 and data2 are the same 38 | if (min > data1[i]) min = data1[i]; 39 | if (max < data1[i]) max = data1[i]; 40 | if (min > data2[i]) min = data2[i]; 41 | if (max < data2[i]) max = data2[i]; 42 | } 43 | System.out.println("min=" + min + ", max=" + max); 44 | g.setColor(Color.red); 45 | for (int i = 0; i < data1.length - 1; i++) { 46 | float y1 = height - 5 - 0.95f * height * ((data1[i] - min) / (max - min)); 47 | float y2 = height - 5 - 0.95f * height * ((data1[i + 1] - min) / (max - min)); 48 | //System.out.println("data["+i+"]="+data[i]+", y1="+y1+", y2="+y2); 49 | g.drawLine(i + 20, (int) y1, i + 21, (int) y2); 50 | y1 = height - 5 - 0.95f * height * ((data2[i] - min) / (max - min)); 51 | y2 = height - 5 - 0.95f * height * ((data2[i + 1] - min) / (max - min)); 52 | //System.out.println("data["+i+"]="+data[i]+", y1="+y1+", y2="+y2); 53 | g.drawLine(i + 20, (int) y1, i + 21, (int) y2); 54 | } 55 | float yzero = height - 5 - 0.95f * height * ((0.0f - min) / (max - min)); 56 | g.setColor(black); 57 | g.drawLine(20, (int) yzero, data2.length + 19, (int) yzero); 58 | g.drawLine(width / 2, height / 2 - 118, width / 2, height / 2 + 118); 59 | g.drawString("Sigmoid", width / 2 + 100, height / 4 - 10); 60 | g.drawString("SigmoidP", width / 2 +60, 3 * height / 4 + 10); 61 | g.drawString("-5", 4, (int) yzero); 62 | g.drawString("5", width - 19, (int) yzero); 63 | g.drawString("1.0", width / 2 - 7, 12); 64 | g.drawString("0.0", width / 2 - 9, height - 5); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /neuralnetworks/src/main/java/com/markwatson/neuralnetworks/Plot1DPanel.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.neuralnetworks; 2 | 3 | 4 | import javax.swing.*; 5 | import java.awt.*; 6 | 7 | public class Plot1DPanel extends java.awt.Canvas { // JPanel { 8 | 9 | public Plot1DPanel(int num, float min, float max, float[] values) { 10 | super(); 11 | this.num = num; 12 | this.min = min; 13 | this.max = max; 14 | this.values = values; 15 | colors = new Color[100]; 16 | for (int i = 0; i < 100; i++) { 17 | float x = 1.0f - ((float) i) * 0.0096f; 18 | colors[i] = new Color(x, x, x); 19 | } 20 | this.setBackground(Color.white); 21 | } 22 | 23 | private int num; 24 | private float min, max; 25 | private float temp; 26 | private float[] values = null; 27 | private Color[] colors; 28 | 29 | //public void plot(float [] values) { 30 | //} 31 | public void paint(Graphics g) { 32 | //System.out.println("Plot1DPanel: values="+values); 33 | if (values == null) return; 34 | int delta_width = this.getWidth() / num; 35 | int delta_height = this.getHeight() / num; 36 | for (int i = 0; i < num; i++) { 37 | //System.out.println(this.toString() + ", values[" + i + "]=" + values[i]); 38 | temp = 100.0f * (values[i] - min) / (max - min); 39 | int ii = (int) temp; 40 | if (ii < 0) ii = 0; 41 | if (ii > 99) ii = 99; 42 | g.setColor(colors[ii]); 43 | g.fillRect(i * delta_width, 0, delta_width, delta_height); 44 | g.setColor(Color.black); 45 | g.drawRect(i * delta_width, 0, delta_width, delta_height); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /neuralnetworks/src/main/java/com/markwatson/neuralnetworks/Plot2DPanel.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.neuralnetworks; 2 | 3 | import javax.swing.*; 4 | import java.awt.*; 5 | 6 | public class Plot2DPanel extends java.awt.Canvas { // JPanel { 7 | 8 | public Plot2DPanel(int num1, int num2, float min, float max, float[][] values) { 9 | super(); 10 | this.num1 = num1; 11 | this.num2 = num2; 12 | this.min = min; 13 | this.max = max; 14 | this.values = values; 15 | colors = new Color[100]; 16 | for (int i = 0; i < 100; i++) { 17 | float x = 1.0f - ((float) i) * 0.0096f; 18 | colors[i] = new Color(x, x, x); 19 | } 20 | } 21 | 22 | private int num1; 23 | private int num2; 24 | private float min, max; 25 | private float temp; 26 | private float[][] values = null; 27 | private Color[] colors; 28 | 29 | public void paint(Graphics g) { 30 | if (values == null) return; 31 | int delta_width = this.getWidth() / num1; 32 | int delta_height = this.getHeight() / num2; 33 | for (int i = 0; i < num1; i++) { 34 | for (int j = 0; j < num2; j++) { 35 | //System.out.println(this.toString() + ", values[" + i + "]=" + values[i]); 36 | temp = 100.0f * (values[i][j] - min) / (max - min); 37 | int ii = (int) temp; 38 | if (ii < 0) ii = 0; 39 | if (ii > 99) ii = 99; 40 | g.setColor(colors[ii]); 41 | g.fillRect(i * delta_width, j * delta_height, delta_width, delta_height); 42 | g.setColor(Color.black); 43 | g.drawRect((i * delta_width), (j * delta_height), delta_width, delta_height); 44 | } 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /neuralnetworks/src/test/java/com/markwatson/neuralnetworks/NeuralNetwork_1H_Test.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.neuralnetworks; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | public class NeuralNetwork_1H_Test extends TestCase { 8 | /** 9 | * Create the test case 10 | * 11 | * @param testName name of the test case 12 | */ 13 | public NeuralNetwork_1H_Test(String testName) 14 | { 15 | super( testName ); 16 | } 17 | 18 | static float[] in1 = {0.1f, 0.1f, 0.9f}; 19 | static float[] in2 = {0.1f, 0.9f, 0.1f}; 20 | static float[] in3 = {0.9f, 0.1f, 0.1f}; 21 | 22 | static float[] out1 = {0.9f, 0.1f, 0.1f}; 23 | static float[] out2 = {0.1f, 0.1f, 0.9f}; 24 | static float[] out3 = {0.1f, 0.9f, 0.1f}; 25 | 26 | static float[] test1 = {0.1f, 0.1f, 0.9f}; 27 | static float[] test2 = {0.1f, 0.9f, 0.1f}; 28 | static float[] test3 = {0.9f, 0.1f, 0.1f}; 29 | 30 | void test_recall(Neural_1H nn, float[] inputs) { 31 | float[] results = nn.recall(inputs); 32 | System.out.print("Test case: "); 33 | for (float input : inputs) System.out.print(pp(input) + " "); 34 | System.out.print(" results: "); 35 | for (float result : results) System.out.print(pp(result) + " "); 36 | System.out.println(); 37 | } 38 | 39 | 40 | /** 41 | * @return the suite of tests being tested 42 | */ 43 | public static Test suite() 44 | { 45 | return new TestSuite( NeuralNetwork_1H_Test.class ); 46 | } 47 | 48 | /** 49 | * Test that is just for side effect printouts: 50 | */ 51 | public void testTraining() { 52 | Neural_1H nn = new Neural_1H(3, 3, 3); 53 | nn.addTrainingExample(in1, out1); 54 | nn.addTrainingExample(in2, out2); 55 | nn.addTrainingExample(in3, out3); 56 | double error = 0; 57 | for (int i = 0; i < 10000; i++) { 58 | error += nn.train(); 59 | if (i > 0 && (i % 1000 == 0)) { 60 | error /= 100; 61 | System.out.println("cycle " + i + " error is " + error); 62 | error = 0; 63 | } 64 | } 65 | System.out.println("Test results should rotate inputs:"); 66 | test_recall(nn, test1); 67 | test_recall(nn, test2); 68 | test_recall(nn, test3); 69 | } 70 | 71 | public static String pp(float x) { 72 | String s = new String("" + x + "00"); 73 | int index = s.indexOf("."); 74 | if (index > -1) s = s.substring(0, index + 3); 75 | if (s.startsWith("-") == false) s = " " + s; 76 | return s; 77 | } 78 | } -------------------------------------------------------------------------------- /neuralnetworks/src/test/java/com/markwatson/neuralnetworks/NeuralNetwork_2H_Test.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.neuralnetworks; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | public class NeuralNetwork_2H_Test extends TestCase { 8 | /** 9 | * Create the test case 10 | * 11 | * @param testName name of the test case 12 | */ 13 | public NeuralNetwork_2H_Test(String testName) 14 | { 15 | super( testName ); 16 | } 17 | 18 | static float[] in1 = {0.1f, 0.1f, 0.9f}; 19 | static float[] in2 = {0.1f, 0.9f, 0.1f}; 20 | static float[] in3 = {0.9f, 0.1f, 0.1f}; 21 | 22 | static float[] out1 = {0.9f, 0.1f, 0.1f}; 23 | static float[] out2 = {0.1f, 0.1f, 0.9f}; 24 | static float[] out3 = {0.1f, 0.9f, 0.1f}; 25 | 26 | static float[] test1 = {0.1f, 0.1f, 0.9f}; 27 | static float[] test2 = {0.1f, 0.9f, 0.1f}; 28 | static float[] test3 = {0.9f, 0.1f, 0.1f}; 29 | 30 | void test_recall(Neural_2H nn, float[] inputs) { 31 | float[] results = nn.recall(inputs); 32 | System.out.print("Test case: "); 33 | for (float input : inputs) System.out.print(pp(input) + " "); 34 | System.out.print(" results: "); 35 | for (float result : results) System.out.print(pp(result) + " "); 36 | System.out.println(); 37 | } 38 | 39 | 40 | /** 41 | * @return the suite of tests being tested 42 | */ 43 | public static Test suite() 44 | { 45 | return new TestSuite( NeuralNetwork_2H_Test.class ); 46 | } 47 | 48 | /** 49 | * Test that is just for side effect printouts: 50 | */ 51 | public void testTraining() { 52 | Neural_2H nn = new Neural_2H(3, 3, 3, 3); 53 | nn.addTrainingExample(in1, out1); 54 | nn.addTrainingExample(in2, out2); 55 | nn.addTrainingExample(in3, out3); 56 | double error = 0; 57 | for (int i = 0; i < 10000; i++) { 58 | error += nn.train(); 59 | if (i > 0 && (i % 1000 == 0)) { 60 | error /= 100; 61 | System.out.println("cycle " + i + " error is " + error); 62 | error = 0; 63 | } 64 | } 65 | System.out.println("Test results should rotate inputs:"); 66 | test_recall(nn, test1); 67 | test_recall(nn, test2); 68 | test_recall(nn, test3); 69 | } 70 | 71 | public static String pp(float x) { 72 | String s = new String("" + x + "00"); 73 | int index = s.indexOf("."); 74 | if (index > -1) s = s.substring(0, index + 3); 75 | if (s.startsWith("-") == false) s = " " + s; 76 | return s; 77 | } 78 | } -------------------------------------------------------------------------------- /neuralnetworks/src/test/java/com/markwatson/neuralnetworks/NeuralNetwork_2H_momentum_Test.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.neuralnetworks; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | public class NeuralNetwork_2H_momentum_Test extends TestCase { 8 | /** 9 | * Create the test case 10 | * 11 | * @param testName name of the test case 12 | */ 13 | public NeuralNetwork_2H_momentum_Test(String testName) 14 | { 15 | super( testName ); 16 | } 17 | 18 | static float[] in1 = {0.1f, 0.1f, 0.9f}; 19 | static float[] in2 = {0.1f, 0.9f, 0.1f}; 20 | static float[] in3 = {0.9f, 0.1f, 0.1f}; 21 | 22 | static float[] out1 = {0.9f, 0.1f, 0.1f}; 23 | static float[] out2 = {0.1f, 0.1f, 0.9f}; 24 | static float[] out3 = {0.1f, 0.9f, 0.1f}; 25 | 26 | static float[] test1 = {0.1f, 0.1f, 0.9f}; 27 | static float[] test2 = {0.1f, 0.9f, 0.1f}; 28 | static float[] test3 = {0.9f, 0.1f, 0.1f}; 29 | 30 | void test_recall(Neural_2H_momentum nn, float[] inputs) { 31 | float[] results = nn.recall(inputs); 32 | System.out.print("Test case: "); 33 | for (float input : inputs) System.out.print(pp(input) + " "); 34 | System.out.print(" results: "); 35 | for (float result : results) System.out.print(pp(result) + " "); 36 | System.out.println(); 37 | } 38 | 39 | 40 | /** 41 | * @return the suite of tests being tested 42 | */ 43 | public static Test suite() 44 | { 45 | return new TestSuite( NeuralNetwork_2H_momentum_Test.class ); 46 | } 47 | 48 | /** 49 | * Test that is just for side effect printouts: 50 | */ 51 | public void testTraining() { 52 | Neural_2H_momentum nn = new Neural_2H_momentum(3, 3, 3, 3); 53 | nn.addTrainingExample(in1, out1); 54 | nn.addTrainingExample(in2, out2); 55 | nn.addTrainingExample(in3, out3); 56 | double error = 0; 57 | for (int i = 0; i < 10000; i++) { 58 | error += nn.train(); 59 | if (i > 0 && (i % 1000 == 0)) { 60 | error /= 100; 61 | System.out.println("cycle " + i + " error is " + error); 62 | error = 0; 63 | } 64 | } 65 | System.out.println("Test results should rotate inputs:"); 66 | test_recall(nn, test1); 67 | test_recall(nn, test2); 68 | test_recall(nn, test3); 69 | } 70 | 71 | public static String pp(float x) { 72 | String s = new String("" + x + "00"); 73 | int index = s.indexOf("."); 74 | if (index > -1) s = s.substring(0, index + 3); 75 | if (s.startsWith("-") == false) s = " " + s; 76 | return s; 77 | } 78 | } -------------------------------------------------------------------------------- /neuralnetworks/src/test/java/com/markwatson/neuralnetworks/xor.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.neuralnetworks; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | // 3/7/2004: note: someone asked for an XOR example... -Mark 8 | 9 | public class xor { 10 | static float FALSE = 0.01f; 11 | static float TRUE = 0.99f; 12 | 13 | static float[] in1 = {FALSE, FALSE}; 14 | static float[] in2 = {FALSE, TRUE}; 15 | static float[] in3 = {TRUE, FALSE}; 16 | static float[] in4 = {TRUE, TRUE}; 17 | 18 | static float[] out1 = {FALSE}; 19 | static float[] out2 = {TRUE}; 20 | static float[] out3 = {TRUE}; 21 | static float[] out4 = {FALSE}; 22 | 23 | static float[] test1 = {FALSE, TRUE}; 24 | static float[] test2 = {TRUE, FALSE}; 25 | static float[] test3 = {TRUE, TRUE}; 26 | 27 | public static void main(String[] args) { 28 | Neural_1H nn = new Neural_1H(2, 3, 1); 29 | nn.addTrainingExample(in1, out1); 30 | nn.addTrainingExample(in3, out3); 31 | nn.addTrainingExample(in2, out2); 32 | nn.addTrainingExample(in4, out4); 33 | // loop until we get a low enough error: 34 | try_again: for (int iter = 0; iter < 500; iter++) { 35 | nn.randomizeWeights(); 36 | System.out.println("\nStarting with a new random set of weights (iter="+iter+")"); 37 | float weightJiggleFactor = 0.15f; 38 | double learningRate = 0.4; // 0.15; 39 | float error = 0; 40 | for (long i = 0; i < 2000000; i++) { 41 | nn.jiggleWeights(weightJiggleFactor); 42 | weightJiggleFactor *= 0.9999; 43 | nn.setLearningRate((float) learningRate); 44 | learningRate *= 0.999995; 45 | if (learningRate < 0.08f) learningRate = 0.08f; 46 | if (weightJiggleFactor < 0.0002f) weightJiggleFactor = 0.02f; 47 | error += nn.train(); 48 | //if (error > 0.25f) weightJiggleFactor = 0.5f; 49 | //if (error > 0.31f) nn.randomizeWeights(); 50 | if (i>0 && i % 100 == 0) { 51 | error /= 100; 52 | //System.out.println("cycle " + i + " error is " + error + ", learningRate=" + learningRate + ", weightJiggleFactor=" + weightJiggleFactor); 53 | // test to see if this set of initial random weights is 54 | // producing poor results - if so, bail and start with a new set 55 | // of weights: 56 | if (i > 30000 & error > 0.38f) continue try_again; 57 | if (i > 90000 & error > 0.28f) continue try_again; 58 | if (i >150000 & error > 0.21f) continue try_again; 59 | if (i >350000 & error > 0.18f) continue try_again; 60 | // if the error is low enough, simply stop training now: 61 | if (error < 0.1) break try_again; 62 | error = 0; 63 | } 64 | } 65 | } 66 | test_recall(nn, test1); 67 | test_recall(nn, test2); 68 | test_recall(nn, test3); 69 | } 70 | 71 | public static void test_recall(Neural_1H nn, float[] inputs) { 72 | float[] results = nn.recall(inputs); 73 | System.out.print("Test case: "); 74 | for (int i = 0; i < inputs.length; i++) System.out.print(pp(inputs[i]) + " "); 75 | System.out.print(" results: "); 76 | for (int i = 0; i < results.length; i++) System.out.print(pp(results[i]) + " "); 77 | System.out.println(); 78 | } 79 | 80 | public static String pp(float x) { 81 | String s = new String("" + x + "00"); 82 | int index = s.indexOf("."); 83 | if (index > -1) s = s.substring(0, index + 3); 84 | if (s.startsWith("-") == false) s = " " + s; 85 | return s; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /nlp/.gitignore: -------------------------------------------------------------------------------- 1 | .classpath 2 | .project 3 | *.iml 4 | *.txt 5 | -------------------------------------------------------------------------------- /nlp/Makefile: -------------------------------------------------------------------------------- 1 | names: 2 | mvn install -DskipTests -q 3 | mvn exec:java -Dexec.mainClass="com.markwatson.nlp.ExtractNames" -q 4 | 5 | autotagger: 6 | mvn install -DskipTests -q 7 | mvn exec:java -Dexec.mainClass="com.markwatson.nlp.AutoTagger" -q 8 | 9 | fasttag: 10 | mvn install -DskipTests -q 11 | mvn exec:java -Dexec.mainClass="com.markwatson.nlp.FastTag" -q 12 | 13 | 14 | install: 15 | mvn install -DskipTest -Dmaven.test.skip=true -q 16 | -------------------------------------------------------------------------------- /nlp/README.md: -------------------------------------------------------------------------------- 1 | # Notes for Mark Watson's NLP library 2 | 3 | TBD: history 4 | 5 | ## ExtractNames - named entity recognition 6 | 7 | The file test_data/propernames.ser is a serialized object file containing first names, last names, and place names. 8 | 9 | Just for educational purposes, the class constructor writes these names to the text files firstnames.txt, lastnames.txt, and placenames.txt. 10 | 11 | These text files are not used for anything except for giving you an easy way to see what entity name data we are using in this example. 12 | -------------------------------------------------------------------------------- /nlp/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | MarkWatsonNlpLib 8 | nlplib 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 13 | junit 14 | junit 15 | 4.13.1 16 | test 17 | 18 | 19 | 20 | 21 | 22 | org.apache.maven.plugins 23 | maven-compiler-plugin 24 | 3.3 25 | 26 | 1.8 27 | 1.8 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /nlp/src/main/java/com/markwatson/nlp/ComparableDocument.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.nlp; 2 | 3 | import com.markwatson.nlp.util.NoiseWords; 4 | import public_domain.Stemmer; 5 | 6 | import java.io.File; 7 | import java.io.FileNotFoundException; 8 | import java.util.*; 9 | 10 | /** 11 | * This class stores stem count data for words in a document and provides 12 | * an API to compare the similarity between this document and another. 13 | * 14 | * @author Mark Watson 15 | * 16 | *

17 | * Copyright 1998-2012 by Mark Watson. All rights reserved. 18 | *

19 | * This software is can be used under either of the following licenses: 20 | *

21 | * 1. LGPL v3
22 | * 2. Apache 2 23 | *

24 | * 25 | */ 26 | public class ComparableDocument { 27 | private ComparableDocument() { } // disable default constructor calls 28 | public ComparableDocument(File document) throws FileNotFoundException { 29 | this(new Scanner(document).useDelimiter("\\Z").next()); 30 | } 31 | public ComparableDocument(String text) { 32 | // System.out.println("text:\n\n" + text + "\n\n"); 33 | List stems = new Stemmer().stemString(text); 34 | for (String stem : stems) { 35 | if (!NoiseWords.checkFor(stem)) { 36 | stem_count++; 37 | if (stemCountMap.containsKey(stem)) { 38 | Integer count = stemCountMap.get(stem); 39 | stemCountMap.put(stem, 1 + count); 40 | } else { 41 | stemCountMap.put(stem, 1); 42 | } 43 | } 44 | // System.out.println(stem + " : " + stemCountMap.get(stem)); 45 | } 46 | } 47 | public Map getStemMap() { return stemCountMap; } 48 | public int getStemCount() { return stem_count; } 49 | public float compareTo(ComparableDocument otherDocument) { 50 | long count = 0; 51 | Map map2 = otherDocument.getStemMap(); 52 | Iterator iter = stemCountMap.keySet().iterator(); 53 | while (iter.hasNext()) { 54 | String key = iter.next(); 55 | Integer count1 = stemCountMap.get(key); 56 | Integer count2 = map2.get(key); 57 | 58 | if (count1!=null && count2!=null) { 59 | count += count1 + count2; 60 | //System.out.println(key); 61 | } 62 | } 63 | //System.out.println("stem_count="+stem_count); 64 | return (float) Math.sqrt(((float)(count*count) / (double)(stem_count * otherDocument.getStemCount()))) / 2f; 65 | } 66 | private Map stemCountMap = new HashMap(); 67 | private int stem_count = 0; 68 | // throw away test program: 69 | public static void main(String[] args) throws FileNotFoundException { 70 | ComparableDocument news1 = new ComparableDocument(new File("test_data/news_1.txt")); 71 | ComparableDocument news2 = new ComparableDocument(new File("test_data/news_2.txt")); 72 | ComparableDocument econ1 = new ComparableDocument(new File("test_data/economy_1.txt")); 73 | ComparableDocument econ2 = new ComparableDocument(new File("test_data/economy_2.txt")); 74 | System.out.println("news 1 - news1: " + news1.compareTo(news1)); 75 | System.out.println("news 1 - news2: " + news1.compareTo(news2)); 76 | System.out.println("news 2 - news2: " + news2.compareTo(news2)); 77 | System.out.println("news 1 - econ1: " + news1.compareTo(econ1)); 78 | System.out.println("econ 1 - econ1: " + econ1.compareTo(econ1)); 79 | System.out.println("news 1 - econ2: " + news1.compareTo(econ2)); 80 | System.out.println("news 2 - econ2: " + news2.compareTo(econ2)); 81 | System.out.println("econ 1 - econ2: " + econ1.compareTo(econ2)); 82 | System.out.println("econ 2 - econ2: " + econ2.compareTo(econ2)); 83 | } 84 | } 85 | 86 | 87 | -------------------------------------------------------------------------------- /nlp/src/main/java/com/markwatson/nlp/util/NameValue.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.nlp.util; 2 | 3 | public class NameValue { 4 | private K name = null; 5 | private V value = null; 6 | public NameValue(K k, V v) { 7 | this.name = k; 8 | this.value = v; 9 | } 10 | public K getName() { 11 | return this.name; 12 | } 13 | public V getValue() { 14 | return this.value; 15 | } 16 | public void setValue(V val) { 17 | this.value = val; 18 | } 19 | public String toString() { 20 | return "[NameValue: " + name + " : " + value + "]"; 21 | } 22 | } -------------------------------------------------------------------------------- /nlp/src/main/java/com/markwatson/nlp/util/NoiseWords.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.nlp.util; 2 | 3 | import java.util.HashSet; 4 | import java.util.Set; 5 | import public_domain.Stemmer; 6 | 7 | public class NoiseWords { 8 | private static String[] words = { 9 | "the", "a", "an", "it", "or", "and", "he", "she", 10 | "with", "often", "to", "do", "that", "this", "is", 11 | "are", "one", "two", "since", "just", "start", 12 | "beyond", "could", "not", "be", "from", "on", "could", 13 | "as", "say", "said", "will", "if", "by", "on", "often", 14 | "little", "big", "did", "do", "about", "any", "such", 15 | "up", "s", "already", "than", "now", "gave", "less", 16 | "more", "another", "for", "other", "goes", "would", 17 | "of", "her", "how", "told", "meet", "without", 18 | "few", "has", "ask", "run", "across", "rather", "me", 19 | "sometme", "want", "d", "look", "perhaps", "come", 20 | "o", "us", "m", "seem", "i", "u", "t", "what", 21 | "but", "last", "who", "toward", "when", "thing", 22 | "got", "can", "with", "at", "off", "in", "much", 23 | "under", "why", "also", "take", "am", "great", 24 | "in", "top" 25 | }; 26 | private static Set stems = new HashSet(); 27 | static { 28 | Stemmer stemmer = new Stemmer(); 29 | for (String word : words) { 30 | stems.add(stemmer.stemOneWord(word)); 31 | } 32 | } 33 | public static boolean checkFor(String stem) { 34 | return stems.contains(stem); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /nlp/src/main/java/com/markwatson/nlp/util/RunExternal.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.nlp.util; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.InputStreamReader; 5 | 6 | public class RunExternal { 7 | 8 | public static void main(String argv[]) { 9 | try { 10 | String line; 11 | Process p = Runtime.getRuntime().exec 12 | ("echo \"thhe dogg brked\" | /usr/local/bin/aspell -a list"); 13 | BufferedReader input = 14 | new BufferedReader 15 | (new InputStreamReader(p.getInputStream())); 16 | while ((line = input.readLine()) != null) { 17 | System.out.println(line); 18 | } 19 | input.close(); 20 | } 21 | catch (Exception err) { 22 | err.printStackTrace(); 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /nlp/test_data/dictionary/README: -------------------------------------------------------------------------------- 1 | 2 | This file contains the contents of the Ispell (ver 3.1.20) word list after 3 | being expand from there affix compressed form used by Ispell. 4 | 5 | Ispell can be found at http://fmg-www.cs.ucla.edu/geoff/ispell.html. 6 | This wordlist can be found at http://wordlist.sourceforge.net/ 7 | 8 | These word lists are under the same copyright as Ispell itself: 9 | 10 | Copyright 1993, Geoff Kuenning, Granada Hills, CA 11 | All rights reserved. 12 | 13 | Redistribution and use in source and binary forms, with or without 14 | modification, are permitted provided that the following conditions 15 | are met: 16 | 17 | 1. Redistributions of source code must retain the above copyright 18 | notice, this list of conditions and the following disclaimer. 19 | 2. Redistributions in binary form must reproduce the above copyright 20 | notice, this list of conditions and the following disclaimer in the 21 | documentation and/or other materials provided with the distribution. 22 | 3. All modifications to the source code must be clearly marked as 23 | such. Binary redistributions based on modified source code 24 | must be clearly marked as modified versions in the documentation 25 | and/or other materials provided with the distribution. 26 | 4. All advertising materials mentioning features or use of this software 27 | must display the following acknowledgment: 28 | This product includes software developed by Geoff Kuenning and 29 | other unpaid contributors. 30 | 5. The name of Geoff Kuenning may not be used to endorse or promote 31 | products derived from this software without specific prior 32 | written permission. 33 | 34 | THIS SOFTWARE IS PROVIDED BY GEOFF KUENNING AND CONTRIBUTORS ``AS 35 | IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 37 | FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GEOFF 38 | KUENNING OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 39 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 40 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 41 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 42 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 43 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 44 | ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 45 | POSSIBILITY OF SUCH DAMAGE. 46 | -------------------------------------------------------------------------------- /nlp/test_data/propername.ser: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mark-watson/Java-AI-Book-Code/f62e1231f4cf2de4641ed01518334696d4d07ff0/nlp/test_data/propername.ser -------------------------------------------------------------------------------- /ollama-llm-client/Makefile: -------------------------------------------------------------------------------- 1 | run: 2 | mvn test -q # run test in quiet mode 3 | 4 | -------------------------------------------------------------------------------- /ollama-llm-client/README.md: -------------------------------------------------------------------------------- 1 | # Ollama Local LLM REST API Example 2 | 3 | Make sure Ollama serve is running: 4 | 5 | ollama serve 6 | 7 | Use the make file default target to run the test: 8 | 9 | make -------------------------------------------------------------------------------- /ollama-llm-client/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.markwatson.openai 5 | ollama-llm-client 6 | jar 7 | 1.0-SNAPSHOT 8 | ollama-llm-client 9 | http://maven.apache.org 10 | 11 | 12 | junit 13 | junit 14 | 3.8.1 15 | test 16 | 17 | 18 | org.json 19 | json 20 | 20240303 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /ollama-llm-client/src/main/java/com/markwatson/ollama/OllamaLlmClient.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.ollama; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.IOException; 5 | import java.io.InputStreamReader; 6 | import java.io.OutputStream; 7 | import java.net.HttpURLConnection; 8 | import java.net.URI; 9 | import java.net.URL; 10 | import java.net.URLConnection; 11 | import java.nio.file.Files; 12 | import java.nio.file.Path; 13 | import java.nio.file.Paths; 14 | 15 | import org.json.JSONObject; 16 | 17 | public class OllamaLlmClient { 18 | 19 | public static void main(String[] args) throws Exception { 20 | String prompt = "Translate the following English text to French: 'Hello, how are you?'"; 21 | String completion = getCompletion(prompt, "mistral"); 22 | System.out.println("completion: " + completion); 23 | } 24 | 25 | public static String getCompletion(String prompt, String modelName) throws Exception { 26 | System.out.println("prompt: " + prompt + ", modelName: " + modelName); 27 | 28 | // New JSON message format 29 | JSONObject message = new JSONObject(); 30 | message.put("prompt", prompt); 31 | message.put("model", modelName); 32 | message.put("stream", false); 33 | URI uri = new URI("http://localhost:11434/api/generate"); 34 | URL url = uri.toURL(); 35 | //System.out.println("jsonBody: " + jsonBody); 36 | URLConnection connection = url.openConnection(); 37 | connection.setDoOutput(true); 38 | connection.setRequestProperty("Content-Type", "application/json"); 39 | // Send the JSON payload 40 | try (OutputStream os = connection.getOutputStream()) { 41 | byte[] input = message.toString().getBytes("utf-8"); 42 | os.write(input, 0, input.length); 43 | } 44 | 45 | StringBuilder response; 46 | // Read the response from the server 47 | try (BufferedReader br = new BufferedReader( 48 | new InputStreamReader(connection.getInputStream(), "utf-8"))) { 49 | response = new StringBuilder(); 50 | String responseLine; 51 | while ((responseLine = br.readLine()) != null) { 52 | response.append(responseLine.trim()); 53 | } 54 | System.out.println(response.toString()); 55 | } 56 | 57 | ((HttpURLConnection) connection).disconnect(); 58 | 59 | JSONObject jsonObject = new JSONObject(response.toString()); 60 | String s = jsonObject.getString("response"); 61 | return s; 62 | } 63 | 64 | /*** 65 | * Utilities for using the Ollama LLM APIs 66 | */ 67 | 68 | // read the contents of a file path into a Java string 69 | public static String readFileToString(String filePath) throws IOException { 70 | Path path = Paths.get(filePath); 71 | return new String(Files.readAllBytes(path)); 72 | } 73 | 74 | public static String replaceSubstring(String originalString, String substringToReplace, String replacementString) { 75 | return originalString.replace(substringToReplace, replacementString); 76 | } 77 | public static String promptVar(String prompt0, String varName, String varValue) { 78 | String prompt = replaceSubstring(prompt0, varName, varValue); 79 | return replaceSubstring(prompt, varName, varValue); 80 | } 81 | } 82 | 83 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /ollama-llm-client/src/test/java/com/markwatson/ollama/OllamaLlmClientTest.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.ollama; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class OllamaLlmClientTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public OllamaLlmClientTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( OllamaLlmClientTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | * @throws Exception 34 | */ 35 | public void testCompletion() throws Exception { 36 | String r = 37 | OllamaLlmClient.getCompletion("Translate the following English text to French: 'Hello, how are you?'", "llama3:instruct"); 38 | System.out.println("completion: " + r); 39 | assertTrue( true ); 40 | } 41 | 42 | public void testTwoShotTemplate() throws Exception { 43 | String input_text = "Mark Johnson enjoys living in Berkeley California at 102 Dunston Street and use mjess@foobar.com for contacting him."; 44 | String prompt0 = OllamaLlmClient.readFileToString("../prompts/two-shot-2-var.txt"); 45 | System.out.println("prompt0: " + prompt0); 46 | String prompt = OllamaLlmClient.promptVar(prompt0, "{input_text}", input_text); 47 | System.out.println("prompt: " + prompt); 48 | String r = 49 | OllamaLlmClient.getCompletion(prompt, "llama3:instruct"); 50 | System.out.println("two shot extraction completion: " + r); 51 | assertTrue( true ); 52 | } 53 | 54 | public void testSummarization() throws Exception { 55 | String input_text = "Jupiter is the fifth planet from the Sun and the largest in the Solar System. It is a gas giant with a mass one-thousandth that of the Sun, but two-and-a-half times that of all the other planets in the Solar System combined. Jupiter is one of the brightest objects visible to the naked eye in the night sky, and has been known to ancient civilizations since before recorded history. It is named after the Roman god Jupiter.[19] When viewed from Earth, Jupiter can be bright enough for its reflected light to cast visible shadows,[ and is on average the third-brightest natural object in the night sky after the Moon and Venus."; 56 | String prompt0 = OllamaLlmClient.readFileToString("../prompts/summarization_prompt.txt"); 57 | System.out.println("prompt0: " + prompt0); 58 | String prompt = OllamaLlmClient.promptVar(prompt0, "{input_text}", input_text); 59 | System.out.println("prompt: " + prompt); 60 | String r = 61 | OllamaLlmClient.getCompletion(prompt, "llama3:instruct"); 62 | System.out.println("summarization completion: " + r); 63 | assertTrue( true ); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /openai-llm-client/Makefile: -------------------------------------------------------------------------------- 1 | run: 2 | mvn test -q # run test in quiet mode 3 | 4 | -------------------------------------------------------------------------------- /openai-llm-client/README.md: -------------------------------------------------------------------------------- 1 | # OpenAI API Example 2 | 3 | run test using default Makefile target: 4 | 5 | make -------------------------------------------------------------------------------- /openai-llm-client/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.markwatson.openai 5 | openai-client-example 6 | jar 7 | 1.0-SNAPSHOT 8 | openai-client-example 9 | http://maven.apache.org 10 | 11 | 12 | junit 13 | junit 14 | 3.8.1 15 | test 16 | 17 | 18 | org.json 19 | json 20 | 20240303 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /openai-llm-client/src/main/java/com/markwatson/openai/OpenAICompletions.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.openai; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.IOException; 5 | import java.io.InputStreamReader; 6 | import java.io.OutputStream; 7 | import java.net.HttpURLConnection; 8 | import java.net.URI; 9 | import java.net.URL; 10 | import java.net.URLConnection; 11 | import java.nio.file.Files; 12 | import java.nio.file.Path; 13 | import java.nio.file.Paths; 14 | 15 | import org.json.JSONArray; 16 | import org.json.JSONObject; 17 | 18 | public class OpenAICompletions { 19 | 20 | public static void main(String[] args) throws Exception { 21 | String prompt = "Translate the following English text to French: 'Hello, how are you?'"; 22 | String completion = getCompletion(prompt); 23 | System.out.println("completion: " + completion); 24 | } 25 | 26 | public static String getCompletion(String prompt) throws Exception { 27 | System.out.println("prompt: " + prompt); 28 | String apiKey = System.getenv("OPENAI_API_KEY"); 29 | String model = "gpt-4o-mini"; // Replace with the desired model 30 | 31 | // New JSON message format 32 | JSONObject message = new JSONObject(); 33 | message.put("role", "user"); 34 | message.put("content", prompt); 35 | 36 | JSONArray messages = new JSONArray(); 37 | messages.put(message); 38 | //System.out.println("messages: " + messages.toString()); 39 | JSONObject jsonBody = new JSONObject(); 40 | jsonBody.put("messages", messages); 41 | jsonBody.put("model", model); 42 | URI uri = new URI("https://api.openai.com/v1/chat/completions"); 43 | URL url = uri.toURL(); 44 | //System.out.println("jsonBody: " + jsonBody); 45 | URLConnection connection = url.openConnection(); 46 | connection.setDoOutput(true); 47 | connection.setRequestProperty("Content-Type", "application/json"); 48 | connection.setRequestProperty("Authorization", "Bearer " + apiKey); 49 | // Send the JSON payload 50 | try (OutputStream os = connection.getOutputStream()) { 51 | byte[] input = jsonBody.toString().getBytes("utf-8"); 52 | os.write(input, 0, input.length); 53 | } 54 | 55 | StringBuilder response; 56 | // Read the response from the server 57 | try (BufferedReader br = new BufferedReader( 58 | new InputStreamReader(connection.getInputStream(), "utf-8"))) { 59 | response = new StringBuilder(); 60 | String responseLine; 61 | while ((responseLine = br.readLine()) != null) { 62 | response.append(responseLine.trim()); 63 | } 64 | System.out.println(response.toString()); 65 | } 66 | 67 | ((HttpURLConnection) connection).disconnect(); 68 | JSONObject jsonObject = new JSONObject(response.toString()); 69 | JSONArray choices = jsonObject.getJSONArray("choices"); 70 | JSONObject messageObject = choices.getJSONObject(0).getJSONObject("message"); 71 | String content = messageObject.getString("content"); 72 | //System.out.println("content: " + content); 73 | return content; 74 | } 75 | 76 | 77 | /*** 78 | * Utilities for using the OpenAI API 79 | */ 80 | 81 | // read the contents of a file path into a Java string 82 | public static String readFileToString(String filePath) throws IOException { 83 | Path path = Paths.get(filePath); 84 | return new String(Files.readAllBytes(path)); 85 | } 86 | 87 | public static String replaceSubstring(String originalString, String substringToReplace, String replacementString) { 88 | return originalString.replace(substringToReplace, replacementString); 89 | } 90 | public static String promptVar(String prompt0, String varName, String varValue) { 91 | String prompt = replaceSubstring(prompt0, varName, varValue); 92 | return replaceSubstring(prompt, varName, varValue); 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /openai-llm-client/src/test/java/com/markwatson/openai/OpenAICompletionsTest.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.openai; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class OpenAICompletionsTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public OpenAICompletionsTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( OpenAICompletionsTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | * @throws Exception 34 | */ 35 | public void testCompletion() throws Exception 36 | { 37 | String r = OpenAICompletions.getCompletion("Translate the following English text to French: 'Hello, how are you?'"); 38 | System.out.println("completion: " + r); 39 | assertTrue( true ); 40 | } 41 | 42 | public void testTwoShotTemplate() throws Exception { 43 | String input_text = "Mark Johnson enjoys living in Berkeley California at 102 Dunston Street and use mjess@foobar.com for contacting him."; 44 | String prompt0 = OpenAICompletions.readFileToString("../prompts/two-shot-2-var.txt"); 45 | System.out.println("prompt0: " + prompt0); 46 | String prompt = OpenAICompletions.promptVar(prompt0, "{input_text}", input_text); 47 | System.out.println("prompt: " + prompt); 48 | String r = 49 | OpenAICompletions.getCompletion(prompt); 50 | System.out.println("two shot extraction completion: " + r); 51 | assertTrue( true ); 52 | } 53 | 54 | public void testSummarization() throws Exception { 55 | String input_text = "Jupiter is the fifth planet from the Sun and the largest in the Solar System. It is a gas giant with a mass one-thousandth that of the Sun, but two-and-a-half times that of all the other planets in the Solar System combined. Jupiter is one of the brightest objects visible to the naked eye in the night sky, and has been known to ancient civilizations since before recorded history. It is named after the Roman god Jupiter.[19] When viewed from Earth, Jupiter can be bright enough for its reflected light to cast visible shadows,[ and is on average the third-brightest natural object in the night sky after the Moon and Venus."; 56 | String prompt0 = OpenAICompletions.readFileToString("../prompts/summarization_prompt.txt"); 57 | System.out.println("prompt0: " + prompt0); 58 | String prompt = OpenAICompletions.promptVar(prompt0, "{input_text}", input_text); 59 | System.out.println("prompt: " + prompt); 60 | String r = 61 | OpenAICompletions.getCompletion(prompt); 62 | System.out.println("summarization completion: " + r); 63 | assertTrue( true ); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /prompts/README.md: -------------------------------------------------------------------------------- 1 | #% Example prompts for OPenAI and Ollama examples 2 | -------------------------------------------------------------------------------- /prompts/extraction_prompt.txt: -------------------------------------------------------------------------------- 1 | Given the two examples below, extract the names, addresses, and email addresses of individuals mentioned later as Process Text. Format the extracted information in JSON, with keys for "name", "address", and "email". If any information is missing, use "null" for that field. 2 | 3 | Example 1: 4 | Text: "John Doe lives at 1234 Maple Street, Springfield. His email is johndoe@example.com." 5 | Output: 6 | { 7 | "name": "John Doe", 8 | "address": "1234 Maple Street, Springfield", 9 | "email": "johndoe@example.com" 10 | } 11 | 12 | Example 2: 13 | Text: "Jane Smith has recently moved to 5678 Oak Avenue, Anytown. She hasn't updated her email yet." 14 | Output: 15 | { 16 | "name": "Jane Smith", 17 | "address": "5678 Oak Avenue, Anytown", 18 | "email": null 19 | } 20 | 21 | Process Text: "{input_text}" 22 | Output: 23 | -------------------------------------------------------------------------------- /prompts/summarization_prompt.txt: -------------------------------------------------------------------------------- 1 | Summarize the following text: "{input_text}" 2 | Output: 3 | -------------------------------------------------------------------------------- /prompts/two-shot-2-var.txt: -------------------------------------------------------------------------------- 1 | Given the two examples below, extract the names, addresses, and email addresses of individuals mentioned later as Process Text. Format the extracted information in JSON, with keys for "name", "address", and "email". If any information is missing, use "null" for that field. Be very concise in your output by providing only the output JSON. 2 | 3 | Example 1: 4 | Text: "John Doe lives at 1234 Maple Street, Springfield. His email is johndoe@example.com." 5 | Output: 6 | { 7 | "name": "John Doe", 8 | "address": "1234 Maple Street, Springfield", 9 | "email": "johndoe@example.com" 10 | } 11 | 12 | Example 2: 13 | Text: "Jane Smith has recently moved to 5678 Oak Avenue, Anytown. She hasn't updated her email yet." 14 | Output: 15 | { 16 | "name": "Jane Smith", 17 | "address": "5678 Oak Avenue, Anytown", 18 | "email": null 19 | } 20 | 21 | Process Text: "{input_text}" 22 | Output: 23 | -------------------------------------------------------------------------------- /prompts/two-shot-2.txt: -------------------------------------------------------------------------------- 1 | Given the two examples below, extract the names, addresses, and email addresses of individuals mentioned later as Process Text. Format the extracted information in JSON, with keys for "name", "address", and "email". If any information is missing, use "null" for that field. Be very concise in your output by providing only the output JSON. 2 | 3 | Example 1: 4 | Text: "John Doe lives at 1234 Maple Street, Springfield. His email is johndoe@example.com." 5 | Output: 6 | { 7 | "name": "John Doe", 8 | "address": "1234 Maple Street, Springfield", 9 | "email": "johndoe@example.com" 10 | } 11 | 12 | Example 2: 13 | Text: "Jane Smith has recently moved to 5678 Oak Avenue, Anytown. She hasn't updated her email yet." 14 | Output: 15 | { 16 | "name": "Jane Smith", 17 | "address": "5678 Oak Avenue, Anytown", 18 | "email": null 19 | } 20 | 21 | Process Text: "Mark Johnson enjoys living in Berkeley California at 102 Dunston Street and use mjess@foobar.com for contacting him." 22 | Output: 23 | -------------------------------------------------------------------------------- /search/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | saved-model.bin 3 | conf.json 4 | *.iml 5 | target 6 | -------------------------------------------------------------------------------- /search/.settings/org.eclipse.jdt.apt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.apt.aptEnabled=false 3 | -------------------------------------------------------------------------------- /search/.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 3 | org.eclipse.jdt.core.compiler.compliance=1.8 4 | org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning 5 | org.eclipse.jdt.core.compiler.processAnnotations=disabled 6 | org.eclipse.jdt.core.compiler.release=disabled 7 | org.eclipse.jdt.core.compiler.source=1.8 8 | -------------------------------------------------------------------------------- /search/.settings/org.eclipse.m2e.core.prefs: -------------------------------------------------------------------------------- 1 | activeProfiles= 2 | eclipse.preferences.version=1 3 | resolveWorkspaceProjects=true 4 | version=1 5 | -------------------------------------------------------------------------------- /search/Makefile: -------------------------------------------------------------------------------- 1 | chess: 2 | mvn install -q 3 | mvn exec:java -Dexec.mainClass="search.game.Chess" -q 4 | 5 | tictactoe: 6 | mvn install -q 7 | mvn exec:java -Dexec.mainClass="search.game.TicTacToe" -q 8 | 9 | graph: 10 | mvn install -q 11 | mvn exec:java -Dexec.mainClass="search.graph.GraphDepthFirstSearch" -q 12 | 13 | maze: 14 | mvn install -q 15 | mvn exec:java -Dexec.mainClass="search.maze.MazeBreadthFirstSearch" -q 16 | 17 | 18 | install: 19 | mvn install -DskipTest -Dmaven.test.skip=true -q 20 | -------------------------------------------------------------------------------- /search/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.markwatson 5 | search_examples 6 | jar 7 | 1.0-SNAPSHOT 8 | search_examples 9 | http://maven.apache.org 10 | 11 | 1.8 12 | 13 | 14 | 15 | 16 | 17 | junit 18 | junit 19 | 4.13.1 20 | test 21 | 22 | 23 | org.slf4j 24 | slf4j-api 25 | 1.7.12 26 | 27 | 28 | org.apache.commons 29 | commons-io 30 | 1.3.2 31 | 32 | 33 | 34 | 35 | 36 | org.apache.maven.plugins 37 | maven-compiler-plugin 38 | 2.3.2 39 | 40 | 1.8 41 | 1.8 42 | 43 | 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /search/src/main/java/search/game/ChessMove.java: -------------------------------------------------------------------------------- 1 | package search.game; 2 | 3 | public class ChessMove extends Move { 4 | public int from; 5 | public int to; 6 | } 7 | -------------------------------------------------------------------------------- /search/src/main/java/search/game/ChessPosition.java: -------------------------------------------------------------------------------- 1 | package search.game; 2 | 3 | public class ChessPosition extends Position { 4 | final static public int BLANK = 0; 5 | final static public int HUMAN = 1; 6 | final static public int PROGRAM = -1; 7 | final static public int PAWN = 1; 8 | final static public int KNIGHT = 2; 9 | final static public int BISHOP = 3; 10 | final static public int ROOK = 4; 11 | final static public int QUEEN = 5; 12 | final static public int KING = 9; 13 | int [] board = new int[120]; 14 | public String toString() { 15 | StringBuffer sb = new StringBuffer("["); 16 | for (int i=22; i<100; i++) { 17 | sb.append(""+board[i]+","); 18 | } 19 | sb.append("]"); 20 | return sb.toString(); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /search/src/main/java/search/game/Move.java: -------------------------------------------------------------------------------- 1 | package search.game; 2 | 3 | abstract public class Move { 4 | } 5 | -------------------------------------------------------------------------------- /search/src/main/java/search/game/Position.java: -------------------------------------------------------------------------------- 1 | package search.game; 2 | 3 | abstract public class Position { 4 | } -------------------------------------------------------------------------------- /search/src/main/java/search/game/TicTacToeMove.java: -------------------------------------------------------------------------------- 1 | package search.game; 2 | 3 | public class TicTacToeMove extends Move { 4 | public int moveIndex; 5 | } 6 | -------------------------------------------------------------------------------- /search/src/main/java/search/game/TicTacToePosition.java: -------------------------------------------------------------------------------- 1 | package search.game; 2 | 3 | public class TicTacToePosition extends Position { 4 | final static public int BLANK = 0; 5 | final static public int HUMAN = 1; 6 | final static public int PROGRAM = -1; 7 | int [] board = new int[9]; 8 | public String toString() { 9 | StringBuffer sb = new StringBuffer("["); 10 | for (int i=0; i<9; i++) { 11 | sb.append(""+board[i]+","); 12 | } 13 | sb.append("]"); 14 | return sb.toString(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /search/src/main/java/search/game/out.txt: -------------------------------------------------------------------------------- 1 | Board position: 2 | 3 | BR BN BB BQ BK BB BN BR 4 | BP BP BP BP BP BP BP BP 5 | . . . . 6 | . . . . 7 | . . . . 8 | . . . . 9 | WP WP WP WP WP WP WP WP 10 | WR WN WB WQ WK WB WN WR 11 | Your move: 12 | enter a move like 'd2d4' or 'oo' 13 | s=d2d4 14 | From 35, to 55 15 | Board position: 16 | 17 | BR BN BB BQ BK BB BN BR 18 | BP BP BP BP BP BP BP BP 19 | . . . . 20 | . . . . 21 | . WP . . 22 | . . . . 23 | WP WP WP . WP WP WP WP 24 | WR WN WB WQ WK WB WN WR 25 | next element: 5.4 26 | next element: [4,2,3,5,9,3,2,4,7,7,1,1,1,0,1,1,1,1,7,7,0,0,0,0,0,0,0,0,7,7,0,0,0,1,0,0,0,0,7,7,0,0,0,0,0,0,0,0,7,7,0,0,0,0,-1,0,0,0,7,7,-1,-1,-1,-1,0,-1,-1,-1,7,7,-4,-2,-3,-5,-9,-3,-2,-4,] 27 | next element: [4,2,3,0,9,3,2,4,7,7,1,1,1,5,1,1,1,1,7,7,0,0,0,0,0,0,0,0,7,7,0,0,0,1,0,0,0,0,7,7,0,0,0,0,0,0,0,0,7,7,0,0,0,0,-1,0,0,0,7,7,-1,-1,-1,-1,0,-1,-1,-1,7,7,-4,-2,-3,-5,-9,-3,-2,-4,] 28 | next element: [4,2,3,0,9,3,2,4,7,7,1,1,1,5,1,1,1,1,7,7,0,0,0,0,0,0,0,0,7,7,0,0,0,1,0,0,0,0,7,7,0,0,0,0,0,0,0,0,7,7,0,0,0,0,-1,-5,0,0,7,7,-1,-1,-1,-1,0,-1,-1,-1,7,7,-4,-2,-3,0,-9,-3,-2,-4,] 29 | next element: [4,2,3,0,9,3,0,4,7,7,1,1,1,5,1,1,1,1,7,7,0,0,0,0,0,2,0,0,7,7,0,0,0,1,0,0,0,0,7,7,0,0,0,0,0,0,0,0,7,7,0,0,0,0,-1,-5,0,0,7,7,-1,-1,-1,-1,0,-1,-1,-1,7,7,-4,-2,-3,0,-9,-3,-2,-4,] 30 | next element: [4,2,3,0,9,3,0,4,7,7,1,1,1,5,1,1,1,1,7,7,0,0,0,0,0,2,0,0,7,7,0,0,0,1,0,0,0,0,7,7,-1,0,0,0,0,0,0,0,7,7,0,0,0,0,-1,-5,0,0,7,7,0,-1,-1,-1,0,-1,-1,-1,7,7,-4,-2,-3,0,-9,-3,-2,-4,] 31 | Board position: 32 | 33 | BR BN BB BQ BK BB BN BR 34 | BP BP BP BP . BP BP BP 35 | . . BP . . 36 | . . . . 37 | . WP . . 38 | . . . . 39 | WP WP WP . WP WP WP WP 40 | WR WN WB WQ WK WB WN WR 41 | Your move: 42 | enter a move like 'd2d4' or 'oo' 43 | s=g1f3 44 | From 28, to 47 45 | Board position: 46 | 47 | BR BN BB BQ BK BB BN BR 48 | BP BP BP BP . BP BP BP 49 | . . BP . . 50 | . . . . 51 | . WP . . 52 | . . . WN . 53 | WP WP WP . WP WP WP WP 54 | WR WN WB WQ WK WB . WR 55 | next element: 6.1999993 56 | next element: [4,2,3,5,9,3,0,4,7,7,1,1,1,0,1,1,1,1,7,7,0,0,0,0,0,2,0,0,7,7,0,0,0,1,0,0,0,0,7,7,0,0,0,0,0,0,0,0,7,7,0,0,0,0,-1,-5,0,0,7,7,-1,-1,-1,-1,0,-1,-1,-1,7,7,-4,-2,-3,0,-9,-3,-2,-4,] 57 | next element: [4,2,3,0,9,3,0,4,7,7,1,1,1,0,1,1,1,1,7,7,0,0,0,5,0,2,0,0,7,7,0,0,0,1,0,0,0,0,7,7,0,0,0,0,0,0,0,0,7,7,0,0,0,0,-1,-5,0,0,7,7,-1,-1,-1,-1,0,-1,-1,-1,7,7,-4,-2,-3,0,-9,-3,-2,-4,] 58 | next element: [4,2,3,0,9,3,0,4,7,7,1,1,1,0,1,1,1,1,7,7,0,0,0,5,0,2,0,0,7,7,0,-3,0,1,0,0,0,0,7,7,0,0,0,0,0,0,0,0,7,7,0,0,0,0,-1,-5,0,0,7,7,-1,-1,-1,-1,0,-1,-1,-1,7,7,-4,-2,-3,0,-9,0,-2,-4,] 59 | next element: [4,2,3,0,9,3,0,4,7,7,1,1,0,0,1,1,1,1,7,7,0,0,1,5,0,2,0,0,7,7,0,-3,0,1,0,0,0,0,7,7,0,0,0,0,0,0,0,0,7,7,0,0,0,0,-1,-5,0,0,7,7,-1,-1,-1,-1,0,-1,-1,-1,7,7,-4,-2,-3,0,-9,0,-2,-4,] 60 | next element: [4,2,3,0,9,3,0,4,7,7,1,1,0,0,1,1,1,1,7,7,0,0,1,5,0,2,0,0,7,7,0,-3,0,1,0,0,0,-5,7,7,0,0,0,0,0,0,0,0,7,7,0,0,0,0,-1,0,0,0,7,7,-1,-1,-1,-1,0,-1,-1,-1,7,7,-4,-2,-3,0,-9,0,-2,-4,] 61 | Board position: 62 | 63 | BR BN BB . BK BB BN BR 64 | BP BP BP BP . BP BP BP 65 | . . BP BQ . 66 | . . . . 67 | . WP . . 68 | . . . WN . 69 | WP WP WP . WP WP WP WP 70 | WR WN WB WQ WK WB . WR 71 | Your move: 72 | enter a move like 'd2d4' or 'oo' 73 | java.lang.NullPointerException -------------------------------------------------------------------------------- /search/src/main/java/search/graph/AbstractGraphSearch.java: -------------------------------------------------------------------------------- 1 | package search.graph; 2 | 3 | /** 4 | * Graph search 5 | * 6 | *

7 | * Copyright 1998-2012 by Mark Watson. All rights reserved. 8 | *

9 | * This software is can be used under either of the following licenses: 10 | *

11 | * 1. LGPL v3
12 | * 2. Apache 2 13 | *

14 | */ 15 | abstract public class AbstractGraphSearch { 16 | 17 | public void addNode(String name, int x, int y) { 18 | System.out.println("Adding node: " + name + ", " + x + ", " + y); 19 | nodeNames[numNodes] = name; 20 | node_x[numNodes] = x; 21 | node_y[numNodes] = y; 22 | numNodes++; 23 | } 24 | 25 | public int getNumNodes() { return numNodes; } 26 | public int getNumLinks() { return numLinks; } 27 | 28 | public String getNodeName(int index) { 29 | try { 30 | return nodeNames[index]; 31 | } catch (Exception e) { 32 | System.out.println("Error in getNodeName: " + e); 33 | } 34 | return "no name"; // error condition 35 | } 36 | 37 | public int getNodeX(int index) { 38 | try { 39 | return node_x[index]; 40 | } catch (Exception e) { 41 | System.out.println("Error in getNodePosition: " + e); 42 | } 43 | return 0; // error condition 44 | } 45 | 46 | 47 | public int getNodeY(int index) { 48 | try { 49 | return node_y[index]; 50 | } catch (Exception e) { 51 | System.out.println("Error in getNodePosition: " + e); 52 | } 53 | return 0; // error condition 54 | } 55 | 56 | public int getLink1(int index) { 57 | return link_1[index]; 58 | } 59 | 60 | public int getLink2(int index) { 61 | return link_2[index]; 62 | } 63 | 64 | public void addLink(int node1, int node2) { 65 | link_1[numLinks] = node1; 66 | link_2[numLinks] = node2; 67 | int dist_squared = 68 | (node_x[node1] - node_x[node2]) * (node_x[node1] - node_x[node2]) + 69 | (node_y[node1] - node_y[node2]) * (node_y[node1] - node_y[node2]); 70 | lengths[numLinks] = (int)Math.sqrt(dist_squared); 71 | numLinks++; 72 | } 73 | 74 | public void addLink(String name1, String name2) { 75 | int index1 = -1, index2 = -1; 76 | for (int i=0; i 7 | * Copyright 1998-2012 by Mark Watson. All rights reserved. 8 | *

9 | * This software is can be used under either of the following licenses: 10 | *

11 | * 1. LGPL v3
12 | * 2. Apache 2 13 | *

14 | */ 15 | public class BreadthFirstSearch extends AbstractGraphSearch { 16 | 17 | /** findPath - abstract method in super class */ 18 | public int [] findPath(int start_node, int goal_node) { // return an array of node indices 19 | System.out.println("Entered BreadthFirstSearch.findPath(" + 20 | start_node + ", " + goal_node + ")"); 21 | // data structures for depth first search: 22 | boolean [] alreadyVisitedFlag = new boolean[numNodes]; 23 | int [] predecessor = new int[numNodes]; 24 | IntQueue queue = new IntQueue(numNodes + 2); 25 | 26 | for (int i=0; i= (len - 1)) { 78 | tail = 0; 79 | } else { 80 | tail++; 81 | } 82 | } 83 | public int removeFromQueue() { 84 | int ret = queue[head]; 85 | if (head >= (len - 1)) { 86 | head = 0; 87 | } else { 88 | head++; 89 | } 90 | return ret; 91 | } 92 | public boolean isEmpty() { 93 | return head == (tail + 1); 94 | } 95 | public int peekAtFrontOfQueue() { 96 | return queue[head]; 97 | } 98 | } 99 | 100 | protected int [] connected_nodes(int node) { 101 | int [] ret = new int[AbstractGraphSearch.MAX]; 102 | int num = 0; 103 | 104 | for (int n=0; n 7 | * Copyright 1998-2012 by Mark Watson. All rights reserved. 8 | *

9 | * This software is can be used under either of the following licenses: 10 | *

11 | * 1. LGPL v3
12 | * 2. Apache 2 13 | *

14 | */ 15 | public class DepthFirstSearch extends AbstractGraphSearch { 16 | 17 | /** findPath - abstract method in super class */ 18 | public int [] findPath(int start_node, int goal_node) { // return an array of node indices 19 | System.out.println("Entered DepthFirstSearch.findPath(" + 20 | start_node + ", " + goal_node + ")"); 21 | path[0] = start_node; 22 | return findPathHelper(path, 1, goal_node); 23 | } 24 | 25 | public int [] findPathHelper(int [] path, int num_path, int goal_node) { 26 | System.out.println("Entered DepthFirstSearch.findPathHelper(...," + 27 | num_path + ", " + goal_node + ")"); 28 | if (goal_node == path[num_path - 1]) { 29 | int [] ret = new int[num_path]; 30 | for (int i=0; i 7 | * Copyright 1998-2012 by Mark Watson. All rights reserved. 8 | *

9 | * This software is can be used under either of the following licenses: 10 | *

11 | * 1. LGPL v3
12 | * 2. Apache 2 13 | *

14 | */ 15 | public class AbstractSearchEngine { 16 | public AbstractSearchEngine(int width, int height) { 17 | maze = new Maze(width, height); 18 | initSearch(); 19 | } 20 | public Maze getMaze() { return maze; } 21 | protected Maze maze; 22 | /** 23 | * We will use the Java type Location (fields width and height will 24 | * encode the coordinates in x and y directions) for the search path: 25 | */ 26 | protected Location [] searchPath = null; 27 | protected int pathCount; 28 | protected int maxDepth; 29 | protected Location startLoc, goalLoc, currentLoc; 30 | protected boolean isSearching = true; 31 | 32 | protected void initSearch() { 33 | if (searchPath == null) { 34 | searchPath = new Location[1000]; 35 | for (int i=0; i<1000; i++) { 36 | searchPath[i] = new Location(); 37 | } 38 | } 39 | pathCount = 0; 40 | startLoc = maze.startLoc; 41 | currentLoc = startLoc; 42 | goalLoc = maze.goalLoc; 43 | searchPath[pathCount++] = currentLoc; 44 | } 45 | 46 | protected boolean equals(Location d1, Location d2) { 47 | return d1.x == d2.x && d1.y == d2.y; 48 | } 49 | 50 | public Location [] getPath() { 51 | Location [] ret = new Location[maxDepth]; 52 | for (int i=0; i 7 | * Copyright 1998-2012 by Mark Watson. All rights reserved. 8 | *

9 | * This software is can be used under either of the following licenses: 10 | *

11 | * 1. LGPL v3
12 | * 2. Apache 2 13 | *

14 | */ 15 | public class DepthFirstSearchEngine extends AbstractSearchEngine { 16 | public DepthFirstSearchEngine(int width, int height) { 17 | super(width, height); 18 | iterateSearch(startLoc, 1); 19 | } 20 | 21 | private void iterateSearch(Location loc, int depth) { 22 | if (isSearching == false) return; 23 | maze.setValue(loc.x, loc.y, (short)depth); 24 | Location [] moves = getPossibleMoves(loc); 25 | for (int i=0; i<4; i++) { 26 | if (moves[i] == null) break; // out of possible moves from this location 27 | searchPath[depth] = moves[i]; 28 | if (equals(moves[i], goalLoc)) { 29 | System.out.println("Found the goal at " + moves[i].x + 30 | ", " + moves[i].y); 31 | isSearching = false; 32 | maxDepth = depth; 33 | return; 34 | } else { 35 | iterateSearch(moves[i], depth + 1); 36 | if (isSearching == false) return; 37 | } 38 | } 39 | return; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /search/src/main/java/search/maze/Location.java: -------------------------------------------------------------------------------- 1 | package search.maze; 2 | 3 | /** 4 | * Copyright Mark Watson 2008-2012. All Rights Reserved. 5 | */ 6 | 7 | public class Location { 8 | int x, y; 9 | public Location(int x, int y) { this.x = x; this.y = y; } 10 | 11 | public Location() { 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /search/src/main/java/search/maze/Maze.java: -------------------------------------------------------------------------------- 1 | package search.maze; 2 | 3 | /** 4 | * Class Maze - class for representing search space as a two-Locational maze 5 | */ 6 | public class Maze { 7 | public static short OBSTICLE = -1; 8 | public static short START_LOC_VALUE = -2; 9 | public static short GOAL_LOC_VALUE = -3; 10 | private int width = 0; 11 | private int height = 0; 12 | public Location startLoc = new Location(); 13 | public Location goalLoc = new Location(); 14 | /** 15 | * The maze (or search space) data is stored as a short integer rather than 16 | * as a boolean so that bread-first style searches can use the array to store 17 | * search depth. A value of -1 indicates a barrier in the maze. 18 | */ 19 | private short [][]maze; 20 | public Maze(int width, int height) { 21 | System.out.println("New maze of size " + width + " by " + height); 22 | this.width = width; 23 | this.height = height; 24 | maze = new short[width+2][height+2]; 25 | for (int i=0; i . 2 | . 3 | . 4 | . 5 | . 6 | . 7 | . 8 | "Her Majesty's Secret Service" . 9 | "HMSS" . 10 | . 11 | . 12 | . 13 | . 14 | . 15 | "M" . 16 | . 17 | . 18 | . 19 | . 20 | "Q" . 21 | . 22 | . 23 | . 24 | "James Bond" . 25 | . 26 | . 27 | . 28 | "Darko Kerim" . 29 | . 30 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/data/inferencing_1.txt: -------------------------------------------------------------------------------- 1 | # triples to add to a new repository: 2 | 3 | @prefix rdf: . 4 | @prefix rdfs: . 5 | @prefix kb: . 6 | @prefix person: . 7 | 8 | kb:Sibling rdfs:subClassOf rdfs:Class . 9 | 10 | kb:Brother rdfs:subClassOf kb:Sibling . 11 | 12 | person:mark rdf:type kb:Brother . 13 | 14 | # sample queries to demonstrate inferencing: 15 | 16 | SELECT DISTINCT ?s 17 | WHERE { ?s rdf:type kb:Brother } 18 | 19 | SELECT DISTINCT ?s 20 | WHERE { ?s rdf:type kb:Sibling } -------------------------------------------------------------------------------- /semantic_web_apache_jena/data/inferencing_2.txt: -------------------------------------------------------------------------------- 1 | # triples to add to a new repository: 2 | 3 | @prefix rdf: . 4 | @prefix rdfs: . 5 | @prefix kb: . 6 | @prefix person: . 7 | 8 | kb:mother rdfs:subPropertyOf kb:parent . 9 | kb:father rdfs:subPropertyOf kb:parent . 10 | 11 | person:ron kb:father person:anthony . 12 | 13 | # sample queries to demonstrate inferencing: 14 | 15 | SELECT DISTINCT ?s ?o 16 | WHERE { ?s kb:father ?o } 17 | 18 | SELECT DISTINCT ?s ?o 19 | WHERE { ?s kb:parent ?o } -------------------------------------------------------------------------------- /semantic_web_apache_jena/data/inferencing_3.txt: -------------------------------------------------------------------------------- 1 | # triples to add to a new repository: 2 | 3 | @prefix rdf: . 4 | @prefix rdfs: . 5 | @prefix kb: . 6 | @prefix person: . 7 | 8 | kb:mother rdfs:domain kb:Female . 9 | kb:father rdfs:domain kb:Male . 10 | 11 | person:kate rdf:type kb:Female . 12 | person:kate kb:father person:bill . 13 | 14 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/data/inferencing_4.txt: -------------------------------------------------------------------------------- 1 | # triples to add to a new repository: 2 | 3 | @prefix rdf: . 4 | @prefix rdfs: . 5 | @prefix kb: . 6 | @prefix person: . 7 | 8 | kb:mother rdfs:domain kb:Female . 9 | kb:father rdfs:domain kb:Male . 10 | 11 | person:mary kb:mother person:susan . -------------------------------------------------------------------------------- /semantic_web_apache_jena/data/news.n3: -------------------------------------------------------------------------------- 1 | @prefix kb: . 2 | @prefix rdfs: . 3 | 4 | kb:containsCity rdfs:subPropertyOf kb:containsPlace . 5 | 6 | kb:containsCountry rdfs:subPropertyOf kb:containsPlace . 7 | 8 | kb:containsState rdfs:subPropertyOf kb:containsPlace . 9 | 10 | kb:containsCity "Burlington" , "Denver" , "St. Paul" , "Chicago" , "Quincy" , "CHICAGO" , "Iowa City" ; 11 | kb:containsRegion "U.S. Midwest" , "Midwest" ; 12 | kb:containsCountry "United States" , "Japan" ; 13 | kb:containsState "Minnesota" , "Illinois" , "Mississippi" , "Iowa" ; 14 | kb:containsOrganization "National Guard" , "U.S. Department of Agriculture" , "White House" , "Chicago Board of Trade" , "Department of Transportation" ; 15 | kb:containsPerson "Dena Gray-Fisher" , "Donald Miller" , "Glenn Hollander" , "Rich Feltes" , "George W. Bush" ; 16 | kb:containsIndustryTerm "food inflation" , "food" , "finance ministers" , "oil" . 17 | 18 | kb:containsCity "Washington" , "FLINT" , "Baghdad" , "Arlington" , "Flint" ; 19 | kb:containsCountry "United States" , "Afghanistan" , "Iraq" ; 20 | kb:containsState "Illinois" , "Virginia" , "Arizona" , "Michigan" ; 21 | kb:containsOrganization "White House" , "Obama administration" , "Iraqi government" ; 22 | kb:containsPerson "David Petraeus" , "John McCain" , "Hoshiyar Zebari" , "Barack Obama" , "George W. Bush" , "Carly Fiorina" ; 23 | kb:containsIndustryTerm "oil prices" . 24 | 25 | kb:containsCity "WASHINGTON" ; 26 | kb:containsCountry "United States" , "Pakistan" , "Islamic Republic of Iran" ; 27 | kb:containsState "Maryland" ; 28 | kb:containsOrganization "University of Maryland" , "United Nations" ; 29 | kb:containsPerson "Ban Ki-moon" , "Gordon Brown" , "Hu Jintao" , "George W. Bush" , "Pervez Musharraf" , "Vladimir Putin" , "Steven Kull" , "Mahmoud Ahmadinejad" . 30 | 31 | kb:containsCity "Sao Paulo" , "Kuala Lumpur" ; 32 | kb:containsRegion "Midwest" ; 33 | kb:containsCountry "United States" , "Britain" , "Saudi Arabia" , "Spain" , "Italy" , "India" , "France" , "Canada" , "Russia" , "Germany" , "China" , "Japan" , "South Korea" ; 34 | kb:containsOrganization "Federal Reserve Bank" , "European Union" , "European Central Bank" , "European Commission" ; 35 | kb:containsPerson "Lee Myung-bak" , "Rajat Nag" , "Luiz Inacio Lula da Silva" , "Jeffrey Lacker" ; 36 | kb:containsCompany "Development Bank Managing" , "Reuters" , "Richmond Federal Reserve Bank" ; 37 | kb:containsIndustryTerm "central bank" , "food" , "energy costs" , "finance ministers" , "crude oil prices" , "oil prices" , "oil shock" , "food prices" , "Finance ministers" , "Oil prices" , "oil" . 38 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/data/rdfs_business.n3: -------------------------------------------------------------------------------- 1 | @prefix : . 2 | @prefix owl: . 3 | @prefix rdfs: . 4 | @prefix rdfs_sample_1: . 5 | @prefix xsd: . 6 | 7 | a owl:Ontology . 8 | 9 | :FreshVeggies a :Business . 10 | :HanksHardware a :Business . 11 | :MarkWatsonSoftware a :Business . 12 | :Amazon a :Business . 13 | :MarkWatsonSoftware :Customer :Amazon . 14 | 15 | :Business a owl:Class; 16 | rdfs:label "Business"; 17 | rdfs:subClassOf :Organization . 18 | 19 | :Customer a owl:Class; 20 | rdfs:label "Customer"; 21 | rdfs:subClassOf :Business . 22 | 23 | :Organization a owl:Class; 24 | rdfs:label "Organization" . 25 | 26 | :organizationName a owl:ObjectProperty; 27 | rdfs:domain :Organization; 28 | rdfs:range xsd:stringstring . 29 | 30 | xsd:stringstring a owl:Class . 31 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/data/rdfs_business.nt: -------------------------------------------------------------------------------- 1 | . 2 | 3 | . 4 | 5 | "Business" . 6 | 7 | . 8 | 9 | . 10 | 11 | "Customer" . 12 | 13 | . 14 | 15 | . 16 | 17 | . 18 | 19 | . 20 | 21 | . 22 | 23 | . 24 | 25 | "Organization" . 26 | 27 | . 28 | 29 | . 30 | 31 | . 32 | 33 | . 34 | 35 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/data/rdfs_sample_1.n3: -------------------------------------------------------------------------------- 1 | @prefix : . 2 | @prefix kb: . 3 | @prefix rdf: . 4 | 5 | kb:Business a :Class; 6 | :label "Business"; 7 | :subClassOf kb:Organization . 8 | 9 | kb:Customer a :Class; 10 | :label "Customer"; 11 | :subClassOf kb:Business . 12 | 13 | kb:Organization a :Class; 14 | :label "Organization" . 15 | 16 | kb:organizationName a rdf:Property; 17 | :domain kb:Organization; 18 | :range . 19 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/data/rdfs_sample_1.owl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/data/rdfs_sample_2.n3: -------------------------------------------------------------------------------- 1 | #Processed by Id: cwm.py,v 1.197 2007/12/13 15:38:39 syosi Exp 2 | # using base file:///Users/markw/Documents/WORK/ruby_scripting_book/src/part2/data/zzzzz.rdf 3 | 4 | # Notation3 generation by 5 | # notation3.py,v 1.200 2007/12/11 21:18:08 syosi Exp 6 | 7 | # Base was: file:///Users/markw/Documents/WORK/ruby_scripting_book/src/part2/data/zzzzz.rdf 8 | @prefix : . 9 | @prefix owl: . 10 | @prefix rdfs: . 11 | @prefix rdfs_sample_1: . 12 | @prefix xsd: . 13 | 14 | a owl:Ontology . 15 | 16 | rdfs_sample_1:FreshVeggies a :Business . 17 | 18 | :Business a owl:Class; 19 | rdfs:label "Business"; 20 | rdfs:subClassOf :Organization . 21 | 22 | :Customer a owl:Class; 23 | rdfs:label "Customer"; 24 | rdfs:subClassOf :Business . 25 | 26 | :Organization a owl:Class; 27 | rdfs:label "Organization" . 28 | 29 | :organizationName a owl:ObjectProperty; 30 | rdfs:domain :Organization; 31 | rdfs:range xsd:stringstring . 32 | 33 | xsd:stringstring a owl:Class . 34 | 35 | #ENDS 36 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/data/rdfs_sample_2.owl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | ]> 13 | 14 | 15 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | Business 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | Customer 44 | 45 | 46 | 47 | 48 | 49 | Organization 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/data/sample_news.n3: -------------------------------------------------------------------------------- 1 | @prefix kb: . 2 | 3 | kb:oak_creek_flooding kb:storyType kb:disaster ; 4 | kb:summary "Oak Creek flooded last week affecting 5 businesses" ; 5 | kb:title "Oak Creek Flood" . 6 | 7 | kb:bear_mountain_fire kb:storyType kb:disaster ; 8 | kb:summary "The fire on Bear Mountain was caused by lightning" ; 9 | kb:title "Bear Mountain Fire" . 10 | 11 | kb:trout_season kb:storyType kb:sports , kb:recreation ; 12 | kb:summary "Fishing was good the first day of trout season" ; 13 | kb:title "Trout Season Starts" . 14 | 15 | kb:jc_basketball kb:storyType kb:sports ; 16 | kb:summary "Local JC Basketball team lost by 12 points last night" ; 17 | kb:title "Local JC Lost Last Night" . 18 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/data/sample_news.nt: -------------------------------------------------------------------------------- 1 | . 2 | 3 | "Oak Creek flooded last week affecting 5 businesses" . 4 | 5 | . 6 | 7 | "The fire on Bear Mountain was caused by lightening" . 8 | 9 | . 10 | 11 | . 12 | 13 | . 14 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/data/sparql_ask_test.txt: -------------------------------------------------------------------------------- 1 | PREFIX kb: 2 | ASK 3 | WHERE { ?article_uri ?predicate "Trout Season Starts" } 4 | 5 | PREFIX kb: 6 | ASK 7 | WHERE { ?article_uri kb:copyright ?copyright_value } 8 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/data/sparql_combine_rdfs_test.txt: -------------------------------------------------------------------------------- 1 | @prefix kb: . 2 | @prefix testnews: . 3 | @prefix rdfs: . 4 | 5 | kb:title rdfs:subPropertyOf testnews:title . 6 | testnews:title rdfs:subPropertyOf kb:title . 7 | 8 | kb:oak_creek_flooding kb:storyType kb:disaster ; 9 | kb:summary "Oak Creek flooded last week affecting 5 businesses" ; 10 | kb:title "Oak Creek Flood" . 11 | 12 | PREFIX kb: 13 | PREFIX kb: 14 | 15 | SELECT DISTINCT ?article_uri1 ?object 16 | WHERE { 17 | ?article_uri1 kb:title ?object . 18 | } 19 | 20 | SELECT DISTINCT ?article_uri1 ?object 21 | WHERE { 22 | ?article_uri1 testnews:title ?object . 23 | } 24 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/data/sparql_combine_test.txt: -------------------------------------------------------------------------------- 1 | @prefix kb: . 2 | @prefix testnews: . 3 | @prefix owl: . 4 | 5 | kb:title owl:sameAs testnews:title . 6 | 7 | kb:oak_creek_flooding kb:storyType kb:disaster ; 8 | kb:summary "Oak Creek flooded last week affecting 5 businesses" ; 9 | kb:title "Oak Creek Flood" . 10 | 11 | PREFIX kb: 12 | PREFIX kb: 13 | 14 | SELECT DISTINCT ?article_uri1 ?object 15 | WHERE { 16 | ?article_uri1 kb:title ?object . 17 | } 18 | 19 | # works if OWL inferencing is supported: 20 | SELECT DISTINCT ?article_uri1 ?object 21 | WHERE { 22 | ?article_uri1 testnews:title ?object . 23 | } 24 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/data/sparql_construct_test.txt: -------------------------------------------------------------------------------- 1 | 2 | PREFIX kb: 3 | CONSTRUCT { ?story_type kb:subject_of ?article1 . } 4 | WHERE { 5 | { ?article1 ?story_type kb:sports } UNION { ?article1 ?story_type kb:recreation } . 6 | } 7 | 8 | PREFIX kb: 9 | CONSTRUCT { ?story_type kb:subject_of ?article1 . } 10 | WHERE { 11 | { ?article1 ?story_type kb:sports } UNION { ?article1 ?story_type kb:recreation } . 12 | { ?article2 ?story_type kb:sports } UNION { ?article2 ?story_type kb:recreation } . 13 | FILTER (?article1 != ?article2) 14 | } 15 | 16 | PREFIX kb: 17 | CONSTRUCT { ?story_type kb:subject_of ?article1 . } 18 | WHERE { 19 | ?article1 ?story_type ?story_type_value . 20 | ?article2 ?story_type ?story_type_value . 21 | FILTER ((?article1 != ?article2) && ((?story_type_value = kb:sports) || (?story_type_value = kb:recreation))) 22 | } 23 | 24 | PREFIX kb: 25 | CONSTRUCT { ?story_type_value kb:subject_of ?article1 . } 26 | WHERE { 27 | ?article1 ?story_type ?story_type_value . 28 | ?article2 ?story_type ?story_type_value . 29 | FILTER ((?article1 != ?article2) && ((?story_type_value = kb:sports) || (?story_type_value = kb:recreation))) 30 | } 31 | 32 | PREFIX kb: 33 | CONSTRUCT { ?story_type_value kb:subject_of ?article1 . 34 | ?article2 kb:same_topic ?article1 .} 35 | WHERE { 36 | ?article1 ?story_type ?story_type_value . 37 | ?article2 ?story_type ?story_type_value . 38 | FILTER ((?article1 != ?article2) && ((?story_type_value = kb:sports) || (?story_type_value = kb:recreation))) 39 | } 40 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/data/sparql_describe_test.txt: -------------------------------------------------------------------------------- 1 | PREFIX kb: 2 | DESCRIBE ?article_uri 3 | WHERE { ?article_uri kb:title "Trout Season Starts" } 4 | 5 | PREFIX kb: 6 | DESCRIBE ?article_uri ?predicate 7 | WHERE { ?article_uri ?predicate "Trout Season Starts" } 8 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/data/sparql_select_test.txt: -------------------------------------------------------------------------------- 1 | @prefix kb: . 2 | 3 | kb:oak_creek_flooding kb:storyType kb:disaster ; 4 | kb:summary "Oak Creek flooded last week affecting 5 businesses" ; 5 | kb:title "Oak Creek Flood" . 6 | 7 | kb:bear_mountain_fire kb:storyType kb:disaster ; 8 | kb:summary "The fire on Bear Mountain was caused by lightning" ; 9 | kb:title "Bear Mountain Fire" . 10 | 11 | kb:trout_season kb:storyType kb:sports , kb:recreation ; 12 | kb:summary "Fishing was good the first day of trout season" ; 13 | kb:title "Trout Season Starts" . 14 | 15 | kb:jc_basketball kb:storyType kb:sports ; 16 | kb:summary "Local JC Basketball team lost by 12 points last night" ; 17 | kb:title "Local JC Lost Last Night" . 18 | 19 | 20 | 21 | PREFIX kb: 22 | SELECT DISTINCT ?article_uri1 ?article_uri2 ?predicate1 ?predicate2 23 | WHERE { 24 | ?article_uri1 ?predicate1 ?same_object . 25 | ?article_uri2 ?predicate2 ?same_object . 26 | FILTER (sameTerm(?predicate1, ?predicate2) && !sameTerm(?article_uri1, ?article_uri2)) . 27 | } 28 | 29 | 30 | PREFIX kb: 31 | SELECT DISTINCT ?article_uri1 ?article_uri2 ?predicate1 ?predicate2 32 | WHERE { 33 | ?article_uri1 ?predicate1 "Trout Season Starts" . 34 | ?article_uri2 ?predicate2 "Trout Season Starts" . 35 | FILTER (!sameTerm(?article_uri1, ?article_uri2)) . 36 | } 37 | 38 | PREFIX kb: 39 | SELECT DISTINCT ?article_uri1 ?article_uri2 ?predicate1 ?predicate2 40 | WHERE { 41 | ?article_uri1 ?predicate1 ?o1 . 42 | ?article_uri2 ?predicate2 ?o2 . 43 | FILTER (!sameTerm(?article_uri1, ?article_uri2) && sameTerm(?predicate1, ?predicate2)) . 44 | FILTER regex(?o1, "Season") . 45 | FILTER regex(?o2, "Season") . 46 | } 47 | 48 | PREFIX kb: 49 | SELECT ?article_uri ?title ?summary 50 | WHERE { 51 | { ?article_uri kb:storyType kb:sports } UNION { ?article_uri kb:storyType kb:recreation } . 52 | ?article_uri kb:title ?title . 53 | ?article_uri kb:summary ?summary . 54 | } 55 | 56 | PREFIX kb: 57 | SELECT DISTINCT ?article_uri ?title ?summary 58 | WHERE { 59 | { ?article_uri kb:storyType kb:sports } UNION { ?article_uri kb:storyType kb:recreation } . 60 | ?article_uri kb:title ?title . 61 | ?article_uri kb:summary ?summary . 62 | } 63 | 64 | 65 | # new triple, without a summary: 66 | kb:jc_bowling kb:storyType kb:sports ; 67 | kb:title "JC Bowling Team to Open Season" . 68 | 69 | PREFIX kb: 70 | SELECT DISTINCT ?title ?summary 71 | WHERE { ?article_uri kb:title ?title . 72 | OPTIONAL { ?article_uri kb:summary ?summary } 73 | } 74 | 75 | PREFIX kb: 76 | SELECT DISTINCT ?title ?summary ?page_count 77 | WHERE { ?article_uri kb:title ?title . 78 | OPTIONAL { ?article_uri kb:summary ?summary } . 79 | OPTIONAL { ?article_uri kb:page_count ?page_count . FILTER (?page_count > 1) } . 80 | } 81 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/jenaApisLog.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mark-watson/Java-AI-Book-Code/f62e1231f4cf2de4641ed01518334696d4d07ff0/semantic_web_apache_jena/jenaApisLog.log -------------------------------------------------------------------------------- /semantic_web_apache_jena/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.markwatson 5 | semanticweb 6 | 1.0.3-SNAPSHOT 7 | 8 | 11.0 9 | UTF-8 10 | UTF-8 11 | 12 | 13 | 14 | 15 | 16 | org.apache.jena 17 | apache-jena-libs 18 | 3.17.0 19 | pom 20 | 21 | 22 | org.apache.derby 23 | derby 24 | 10.17.1.0 25 | 26 | 27 | org.apache.derby 28 | derbytools 29 | 10.15.2.0 30 | 31 | 32 | org.apache.derby 33 | derbyclient 34 | 10.15.2.0 35 | 36 | 37 | 38 | commons-io 39 | commons-io 40 | 2.14.0 41 | 42 | 43 | junit 44 | junit 45 | 4.13.1 46 | test 47 | 48 | 49 | org.slf4j 50 | slf4j-api 51 | 1.7.30 52 | 53 | 54 | org.slf4j 55 | slf4j-log4j12 56 | 1.7.30 57 | test 58 | 59 | 60 | 61 | 62 | 63 | maven-compiler-plugin 64 | 3.8.0 65 | 66 | 11 67 | 68 | 69 | 70 | org.apache.maven.plugins 71 | maven-jar-plugin 72 | 3.1.2 73 | 74 | 75 | 76 | com.markwatson.semanticweb.JenaApis 77 | 78 | 79 | 80 | 81 | 82 | 83 | org.apache.maven.plugins 84 | maven-shade-plugin 85 | 2.4.3 86 | 87 | 88 | package 89 | 90 | shade 91 | 92 | 93 | 94 | 95 | 96 | com.markwatson.semanticweb.JenaApis 97 | 1.0 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/src/main/java/com/markwatson/semanticweb/Cache.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.semanticweb; 2 | 3 | import java.sql.*; 4 | 5 | import org.apache.commons.lang3.SerializationUtils; 6 | import org.apache.derby.client.*; 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import java.util.Properties; 10 | 11 | public class Cache { 12 | public Cache() throws SQLException, ClassNotFoundException { 13 | Properties props = new Properties(); 14 | String dbName = "./sparqlCache.db"; 15 | conn = DriverManager.getConnection(protocol + dbName 16 | + ";create=true", props); 17 | conn.setAutoCommit(true); 18 | try { 19 | Statement s = conn.createStatement(); 20 | s.execute("CREATE TABLE cache (query varchar(3000) PRIMARY KEY, result blob)"); 21 | System.out.println("Created table 'cache'"); 22 | } catch (Exception ex) { 23 | //System.out.println("Error (Cache()): " + ex.getMessage()); 24 | //System.out.println("Table 'cache' already exists"); 25 | } 26 | } 27 | public void saveQueryResultInCache (String query, byte [] result) { 28 | try { 29 | if (fetchResultFromCache(query) != null) { 30 | //System.out.println("Query is already in the cache"); 31 | return; 32 | } 33 | Statement s = conn.createStatement(); 34 | PreparedStatement ps = conn.prepareStatement( 35 | "insert into cache (query, result) values (?, ?)"); 36 | ps.setString(1, query); 37 | ps.setBytes(2, result); 38 | ps.executeUpdate(); 39 | } catch (Exception ex) { 40 | System.out.println("Error (saveQueryResultInCache): " + ex.getMessage()); 41 | } 42 | } 43 | 44 | public byte [] fetchResultFromCache (String query) { 45 | try { 46 | Statement s = conn.createStatement(); 47 | PreparedStatement ps = conn.prepareStatement( 48 | "select result from cache where query = ?"); 49 | ps.setString(1, query); 50 | ResultSet rs = ps.executeQuery(); 51 | if (!rs.next()) return null; 52 | return rs.getBytes(1); 53 | } catch (Exception ex) { 54 | System.out.println("Error (fetchResultFromCache): " + ex.getMessage()); 55 | return null; 56 | } 57 | } 58 | 59 | private Connection conn = null; 60 | private String protocol = "jdbc:derby:"; 61 | 62 | } 63 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/src/main/java/com/markwatson/semanticweb/QueryResult.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.semanticweb; 2 | 3 | import java.io.Serializable; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | public class QueryResult implements Serializable { 8 | private QueryResult() { } 9 | public QueryResult(List variableList) { 10 | this.variableList = variableList; 11 | } 12 | public List variableList; 13 | public List> rows = new ArrayList(); 14 | public String toString() { 15 | StringBuilder sb = new StringBuilder("[QueryResult vars:" + variableList + "\nRows:\n"); 16 | for (List row : rows) { 17 | sb.append(" " + row + "\n"); 18 | } 19 | return sb.toString(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/src/main/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /semantic_web_apache_jena/src/test/java/com/markwatson/semanticweb/JenaApisTest.java: -------------------------------------------------------------------------------- 1 | package com.markwatson.semanticweb; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | public class JenaApisTest extends TestCase { 8 | /** 9 | * Create the test case 10 | * 11 | * @param testName name of the test case 12 | */ 13 | public JenaApisTest(String testName) 14 | { 15 | super( testName ); 16 | } 17 | 18 | /** 19 | * @return the suite of tests being tested 20 | */ 21 | public static Test suite() 22 | { 23 | return new TestSuite( JenaApisTest.class ); 24 | } 25 | 26 | /** 27 | * Test that is just for side effect printouts: 28 | */ 29 | public void test1() throws Exception { 30 | JenaApis jenaApis = new JenaApis(); 31 | // test remote SPARQL queries against DBPedia SPARQL endpoint 32 | QueryResult qrRemote = jenaApis.queryRemote( 33 | "https://dbpedia.org/sparql", 34 | "select ?p where { ?p . } limit 10"); 35 | // "select distinct ?s { ?s ?p } LIMIT 10"); 36 | System.out.println("qrRemote:" + qrRemote); 37 | System.out.println("Repeat query to test caching:"); 38 | qrRemote = jenaApis.queryRemote( 39 | "https://dbpedia.org/sparql", 40 | "select distinct ?s { ?s ?p } LIMIT 10"); 41 | System.out.println("qrRemote (hopefully from cache):" + qrRemote); 42 | 43 | jenaApis.loadRdfFile("data/rdfs_business.nt"); 44 | jenaApis.loadRdfFile("data/sample_news.nt"); 45 | jenaApis.loadRdfFile("data/sample_news.n3"); 46 | 47 | QueryResult qr = jenaApis.query( 48 | "select ?s ?o where { ?s ?o } limit 15"); 49 | System.out.println("qr:" + qr); 50 | 51 | jenaApis.saveModelToTurtleFormat("model_save.nt"); 52 | jenaApis.saveModelToN3Format("model_save.n3"); 53 | } 54 | 55 | /** 56 | * Test that is just for side effect printouts: 57 | */ 58 | public void testOwlReasoning() throws Exception { 59 | JenaApis jenaApis = new JenaApis(); 60 | jenaApis.loadRdfFile("data/news.n3"); 61 | 62 | QueryResult qr = jenaApis.query( 63 | "prefix kb: \n" + 64 | "select ?s ?o where { ?s kb:containsCity ?o } "); 65 | System.out.println("qr:" + qr); 66 | 67 | qr = jenaApis.query( 68 | "prefix kb: \n" + 69 | "select ?s ?o where { ?s kb:containsPlace ?o }"); 70 | System.out.println("qr:" + qr); 71 | 72 | qr = jenaApis.query( // count for each place name 73 | "prefix kb: \n" + 74 | "select ?o (count(*) as ?count) where { ?s kb:containsPlace ?o } " + 75 | "group by ?o"); 76 | System.out.println("qr:" + qr); 77 | } 78 | 79 | } --------------------------------------------------------------------------------