├── .gitignore ├── LICENSE.txt ├── README.md ├── anonymization ├── allowedToolNames ├── allowedUserAgents ├── datatypesWhitelist └── stringWhitelist ├── exampleMonthsFolder └── exampleMonth │ ├── anonymousRawData │ ├── AnonymousQueryCnt01.tsv.gz │ ├── AnonymousQueryCnt02.tsv.gz │ └── AnonymousQueryCnt03.tsv.gz │ └── rawLogData │ ├── QueryCnt01.tsv.gz │ ├── QueryCnt02.tsv.gz │ └── QueryCnt03.tsv.gz ├── geosoft_checks.xml ├── parserSettings └── standardPrefixes.tsv ├── pom.xml ├── preBuildQueryTypeFiles ├── AllSubjectsAllPredicatesEntity.preBuildQueryType ├── AllSubjectsAllPredicatesEntityStatement.preBuildQueryType ├── AllSubjectsAllPredicatesReference.preBuildQueryType ├── GNDIDThisWithID.preBuildQueryType ├── InCHIKeyThis.preBuildQueryType ├── aboutThisGNDIDWithFilterLimitOffset.preBuildQueryType ├── aboutThisGNDIDWithFilterLimitOffsetDistinct.preBuildQueryType ├── aboutThisGNDIDWithFilterLimitOffsetDistinctOptionalsBinds.preBuildQueryType ├── aboutThisGNDIDWithFilterLimitOffsetDistinctOptionalsBinds2.preBuildQueryType ├── acidDissociationConstantThis.preBuildQueryType ├── allRelationsThis.preBuildQueryType ├── askWhereInstaceOf.preBuildQueryType ├── askWhereLocatedInTheAdministrativeTerritorialEntity.preBuildQueryType ├── bot2.preBuildQueryType ├── bot222.preBuildQueryType ├── coordianteLocationThis.preBuildQueryType ├── coordinateLocationThisViaBind.preBuildQueryType ├── distinctInstanceOfThis.preBuildQueryType ├── doesFootballerHasAEnglishWikipediaArticle.preBuildQueryType ├── entityAllOptionalLabelImageCoordinatesArticleInLanguage.preBuildQueryType ├── entityAllPropertiesAllObjectsWithInstanceAndLabel.preBuildQueryType ├── entityAllPropertiesAndObjects.preBuildQueryType ├── entityLabelImageAbstractLocationEnglishWikipediaArticle.preBuildQueryType ├── entityLabelImageDescriptionAllOptional.preBuildQueryType ├── entityLabelInSpecificLanguage.preBuildQueryType ├── entityMultipleOptionalProperties.preBuildQueryType ├── entityMultipleOptionalProperties2.preBuildQueryType ├── entityMultipleOptionalProperties3.preBuildQueryType ├── entityMultipleOptionalProperties4.preBuildQueryType ├── entityOptionalInstanceOfOptionalIMDb_ID.preBuildQueryType ├── entityStatementAllPredicatesAndObjects.preBuildQueryType ├── equivalentClassThis.preBuildQueryType ├── findWikidataId.preBuildQueryType ├── gasProgramSSSP.preBuildQueryType ├── geneAndInformationWithConstraints.preBuildQueryType ├── generalInformationAboutPerson.preBuildQueryType ├── hasLabel.preBuildQueryType ├── hasLabelInstanceOfThis.preBuildQueryType ├── hasLabelThisIsSubclass.preBuildQueryType ├── imageThis.preBuildQueryType ├── imageToSocialNetworksID.preBuildQueryType ├── infoAboutHuman.preBuildQueryType ├── instanceOfSubclassOfMultipleFiltersAndOptional.preBuildQueryType ├── instanceOfSubclassOfTVSeries.preBuildQueryType ├── instanceOfSubclassRelations.preBuildQueryType ├── instanceOfSubclassRelations2.preBuildQueryType ├── instanceOfSubclassRelations3.preBuildQueryType ├── instanceOfThis.preBuildQueryType ├── instanceOfThisAnySubclassOfWithLabelAndFilter.preBuildQueryType ├── itemsWithPicturesFromCatalogue.preBuildQueryType ├── linkedDataPilotSwitzerlandCoordinateLocationThis.preBuildQueryType ├── prevAndNextFilmInSeries.preBuildQueryType ├── qt:1027_6765.preBuildQueryType ├── qt:1049_0.preBuildQueryType ├── qt:1059_6997.preBuildQueryType ├── qt:1088_0.preBuildQueryType ├── qt:1088_15.preBuildQueryType ├── qt:1145_24.preBuildQueryType ├── qt:1173_2.preBuildQueryType ├── qt:1181_6600.preBuildQueryType ├── qt:1191_6821.preBuildQueryType ├── qt:1203_1.preBuildQueryType ├── qt:1203_11.preBuildQueryType ├── qt:1244_152.preBuildQueryType ├── qt:1279_2.preBuildQueryType ├── qt:127_15.preBuildQueryType ├── qt:1289_21.preBuildQueryType ├── qt:1309_21.preBuildQueryType ├── qt:1310_0.preBuildQueryType ├── qt:1406_20.preBuildQueryType ├── qt:1407_10.preBuildQueryType ├── qt:1454_1.preBuildQueryType ├── qt:1454_51.preBuildQueryType ├── qt:1454_999.preBuildQueryType ├── qt:1466_0.preBuildQueryType ├── qt:1485_13.preBuildQueryType ├── qt:1489_6788.preBuildQueryType ├── qt:1512_79.preBuildQueryType ├── qt:152_181.preBuildQueryType ├── qt:1583_28.preBuildQueryType ├── qt:1593_1098.preBuildQueryType ├── qt:175_2695.preBuildQueryType ├── qt:197_437.preBuildQueryType ├── qt:217_2527.preBuildQueryType ├── qt:218_18.preBuildQueryType ├── qt:249_6798.preBuildQueryType ├── qt:263_6718.preBuildQueryType ├── qt:266_211.preBuildQueryType ├── qt:270_22.preBuildQueryType ├── qt:281_9725.preBuildQueryType ├── qt:282_199.preBuildQueryType ├── qt:290_169.preBuildQueryType ├── qt:300_56.preBuildQueryType ├── qt:316_160.preBuildQueryType ├── qt:323_6697.preBuildQueryType ├── qt:350_248.preBuildQueryType ├── qt:361_18.preBuildQueryType ├── qt:383_361.preBuildQueryType ├── qt:438_212.preBuildQueryType ├── qt:448_13.preBuildQueryType ├── qt:474_86.preBuildQueryType ├── qt:479_0.preBuildQueryType ├── qt:47_6975.preBuildQueryType ├── qt:498_0.preBuildQueryType ├── qt:499_6736.preBuildQueryType ├── qt:509_16.preBuildQueryType ├── qt:517_9743.preBuildQueryType ├── qt:518_41.preBuildQueryType ├── qt:521_13.preBuildQueryType ├── qt:522_6.preBuildQueryType ├── qt:562_0.preBuildQueryType ├── qt:566_220.preBuildQueryType ├── qt:609_6710.preBuildQueryType ├── qt:61_505.preBuildQueryType ├── qt:625_2507.preBuildQueryType ├── qt:625_6703.preBuildQueryType ├── qt:632_1.preBuildQueryType ├── qt:639_1.preBuildQueryType ├── qt:64_7.preBuildQueryType ├── qt:650_45.preBuildQueryType ├── qt:655_9683.preBuildQueryType ├── qt:740_0.preBuildQueryType ├── qt:740_16.preBuildQueryType ├── qt:749_6696.preBuildQueryType ├── qt:754_15.preBuildQueryType ├── qt:764_189.preBuildQueryType ├── qt:768_1226.preBuildQueryType ├── qt:768_54.preBuildQueryType ├── qt:76_108.preBuildQueryType ├── qt:770_244.preBuildQueryType ├── qt:773_6824.preBuildQueryType ├── qt:774_20.preBuildQueryType ├── qt:788_0.preBuildQueryType ├── qt:798_291.preBuildQueryType ├── qt:828_152.preBuildQueryType ├── qt:882_183.preBuildQueryType ├── qt:896_183.preBuildQueryType ├── qt:898_189.preBuildQueryType ├── qt:911_37.preBuildQueryType ├── qt:921_9879.preBuildQueryType ├── qt:934_15.preBuildQueryType ├── qt:935_0.preBuildQueryType ├── qt:936_1150.preBuildQueryType ├── qt:974_188.preBuildQueryType ├── readme.md ├── referenceAllPredicatesAndObjects.preBuildQueryType ├── sameAsThis.preBuildQueryType ├── schemaAboutEntityLabelImageCoordinatesAllOptional.preBuildQueryType ├── signatureThis.preBuildQueryType ├── subclassOfThis.preBuildQueryType ├── taxonomyInformations.preBuildQueryType ├── thisAFI_Catalog_of_Feature_Films_ID.preBuildQueryType ├── thisAllMovie_movie_ID.preBuildQueryType ├── thisAllcinema_film_ID.preBuildQueryType ├── thisAlloCine_film_ID.preBuildQueryType ├── thisAnimator.ru_film_ID.preBuildQueryType ├── thisAnime_News_Network_anime_ID.preBuildQueryType ├── thisArticleAboutInLanguageIsPartOfWikiGroupWithLimit.preBuildQueryType ├── thisAustralianStatisticalGeography2011IDOptionalDescriptionWithLabel.preBuildQueryType ├── thisBFIFilmAndTVIDID.preBuildQueryType ├── thisBFI_work_ID.preBuildQueryType ├── thisBangla_Movie_Database_ID.preBuildQueryType ├── thisBasedOn.preBuildQueryType ├── thisBox_Office_Mojo_film_ID.preBuildQueryType ├── thisCBSMunicipalityCodeThisWithLabel.preBuildQueryType ├── thisCBSMunicipalityCodeWithLabel.preBuildQueryType ├── thisCBSMunicipalityCodeWithLabelWithPopulation.preBuildQueryType ├── thisCSFD_film_ID.preBuildQueryType ├── thisCiNetMag_film_ID.preBuildQueryType ├── thisCine-Ressources_film_ID.preBuildQueryType ├── thisCinema_ID.preBuildQueryType ├── thisCinema_of_Israel_ID.preBuildQueryType ├── thisCinemagiaFilmID.preBuildQueryType ├── thisCinemathequeQuebecoiseWorkIdentifier.preBuildQueryType ├── thisCineplex_film_ID.preBuildQueryType ├── thisCollectionDepictsPositionHeld.preBuildQueryType ├── thisComposer.preBuildQueryType ├── thisDNF_film_ID.preBuildQueryType ├── thisDOIThatWithFilter.preBuildQueryType ├── thisDaumMovieID.preBuildQueryType ├── thisDerivedFromStatedInPaper.preBuildQueryType ├── thisDeutsche_Synchronkartei_film_ID.preBuildQueryType ├── thisDoubanMovieID.preBuildQueryType ├── thisEDb_film_ID.preBuildQueryType ├── thisEIDR_identifier.preBuildQueryType ├── thisEditionOrTranslationOf.preBuildQueryType ├── thisEldoblaje_Movie_ID.preBuildQueryType ├── thisElonet_movie_ID.preBuildQueryType ├── thisEntrezGeneID.preBuildQueryType ├── thisEntrezGeneIDFoundInTaxon.preBuildQueryType ├── thisExactMatchWithFilter.preBuildQueryType ├── thisExplotationVisaNumber.preBuildQueryType ├── thisFilmAffinity_ID.preBuildQueryType ├── thisFilmPolski.pl_ID.preBuildQueryType ├── thisFilmingLocation.preBuildQueryType ├── thisFilmportal_ID.preBuildQueryType ├── thisFilmweb.pl_ID.preBuildQueryType ├── thisFreebaseIDWithCompanyTickerExchange.preBuildQueryType ├── thisFreebaseIDWithLabelOptionalInstanceOf.preBuildQueryType ├── thisFreebaseIDWithLabelOptionalInstanceOf2.preBuildQueryType ├── thisGECD_film_ID.preBuildQueryType ├── thisGND_ID.preBuildQueryType ├── thisGeneOntologyID.preBuildQueryType ├── thisGenre.preBuildQueryType ├── thisGenreWithEnglishLabel.preBuildQueryType ├── thisHKMDb_film_ID.preBuildQueryType ├── thisHasLabelLocatedInCoordinatesThat.preBuildQueryType ├── thisHasLabel_cityOrRegion.preBuildQueryType ├── thisHasLabel_existingCountries.preBuildQueryType ├── thisHasLabel_geographicalObjects.preBuildQueryType ├── thisIMDb_ID.preBuildQueryType ├── thisISAN.preBuildQueryType ├── thisISSNThatWithFilter.preBuildQueryType ├── thisISzDb_film_ID.preBuildQueryType ├── thisInstanceOf.preBuildQueryType ├── thisInstanceOfGenderDescription.preBuildQueryType ├── thisInstanceOfWithSubclass.preBuildQueryType ├── thisIshim_ID.preBuildQueryType ├── thisJMDb_film_ID.preBuildQueryType ├── thisKINENOTE_film_ID.preBuildQueryType ├── thisKMDb_documentary_ID.preBuildQueryType ├── thisKMDb_film_ID.preBuildQueryType ├── thisKinopoisk_film_ID.preBuildQueryType ├── thisKinopolis_film_ID.preBuildQueryType ├── thisKvikmyndir_film_ID.preBuildQueryType ├── thisLUMIEREFilmID.preBuildQueryType ├── thisLdiF_ID.preBuildQueryType ├── thisLocatedInTheAdministrativeTerritorialEntityArbitraryLengthInstanceOfArbitraryLengthAdministrativeTerritoralEntity.preBuildQueryType ├── thisLocatedInTheAdministrativeTerritorialEntityWithCoordinateLocation.preBuildQueryType ├── thisMegogo_ID.preBuildQueryType ├── thisMetacritic_ID.preBuildQueryType ├── thisMinkultury_film_ID.preBuildQueryType ├── thisMovieMeter_Movie_ID.preBuildQueryType ├── thisMovie_Walker_ID.preBuildQueryType ├── thisMultipleCatalogProperties.preBuildQueryType ├── thisMyAnimeList_anime_ID.preBuildQueryType ├── thisMyDramaList_title_ID.preBuildQueryType ├── thisNCBILocusTagFoundInTaxonNCBITaxonomyIDWithEncodes.preBuildQueryType ├── thisNCBITaxonomyIDSubclassOfOperon.preBuildQueryType ├── thisNKCRAUTIDAboutAndLanguage.preBuildQueryType ├── thisNRHPReferenceNumber.preBuildQueryType ├── thisNamedAfter.preBuildQueryType ├── thisNetflix_ID.preBuildQueryType ├── thisNorsk_filmografi_ID.preBuildQueryType ├── thisOFDb_ID.preBuildQueryType ├── thisOpen_Media_Database_film_ID.preBuildQueryType ├── thisPMCID.preBuildQueryType ├── thisPMCIDthatWithFilter.preBuildQueryType ├── thisPORT_film_ID.preBuildQueryType ├── thisPrismaID.preBuildQueryType ├── thisPubMedID.preBuildQueryType ├── thisPubMedIDthatWithFilter.preBuildQueryType ├── thisRotten_Tomatoes_ID.preBuildQueryType ├── thisRussiancinema.ru_film_ID.preBuildQueryType ├── thisScope.dk_film_ID.preBuildQueryType ├── thisSomePropertyStringWithMultipleOtherProperties.preBuildQueryType ├── thisSourehCinema_film_ID.preBuildQueryType ├── thisSratim_ID.preBuildQueryType ├── thisStreamIDOptionalArticeInLanguage.preBuildQueryType ├── thisSubclassOfWithLabelOrderByLabel.preBuildQueryType ├── thisSwedish_Film_Database_film_ID.preBuildQueryType ├── thisTV.Com_ID.preBuildQueryType ├── thisTV_Guide_Show_ID.preBuildQueryType ├── thisTaxonNameTaxonRank.preBuildQueryType ├── thisTaxonRankParentTaxon.preBuildQueryType ├── thisThe_Numbers_movie_ID.preBuildQueryType ├── thisTransfermarkManagerID.preBuildQueryType ├── thisTransfermarktFootballerID.preBuildQueryType ├── thisTurner_Classic_Movies_film_ID.preBuildQueryType ├── thisULANID.preBuildQueryType ├── thisUniProtID.preBuildQueryType ├── thisUniProtIDThatExactMatchFoundInTaxon.preBuildQueryType ├── thisUniProtIDWithFilterIsLiteral.preBuildQueryType ├── thisUnifrance_film_ID.preBuildQueryType ├── thisVIAFIDBVMCpersonID.preBuildQueryType ├── thisVIAFIDWithLabel.preBuildQueryType ├── thisWikiLovesMonumentsID.preBuildQueryType ├── thiscine.gr_film_ID.preBuildQueryType ├── thiscinenacional.com_movie_ID.preBuildQueryType ├── thisdanskefilm_film_ID.preBuildQueryType ├── thisdanskefilm_silent_film_ID.preBuildQueryType ├── thiselCinema_film_ID.preBuildQueryType ├── thiselFilm_film_ID.preBuildQueryType ├── thisfilm-documentaire.fr_film_ID.preBuildQueryType ├── toolCheckingInstanceOfInEnglishUsingScalaj-http.preBuildQueryType ├── toolListingAllClubsAndTheRespectiveYearsAFootballPlayerWasInATeam.preBuildQueryType ├── toolLookingForEnglishNameOfHistoricalPerson.preBuildQueryType ├── toolLookingForFootballPlayerBirths.preBuildQueryType ├── toolLookingForFootballPlayerDeaths.preBuildQueryType ├── toolLookingForFootballPlayerGender.preBuildQueryType ├── toolLookingForFottballPlayerEnglishWikipediaPage.preBuildQueryType ├── toolLookingForGeoInformation.preBuildQueryType ├── toolLookingForGeoInformation2.preBuildQueryType ├── toolLookingForGeoInformationClasses.preBuildQueryType ├── toolLookingForNNDBIdForAllFootballPlayers.preBuildQueryType ├── toolLookingForWikidataIdBasedOnTwitterUsername.preBuildQueryType ├── toolMapping.tsv ├── totalRevenueWithTotalRevenuePointInTimeLabel.preBuildQueryType ├── transfermarktPlayerIDtransfermarktManagerIDthis.preBuildQueryType ├── transitiveSubclassOfThis.preBuildQueryType └── wikidataLastModified.preBuildQueryType ├── propertyClassification └── propertyGroupMapping.tsv ├── src └── main │ └── java │ ├── anonymize │ ├── Anonymizer.java │ ├── Test.java │ └── package-info.java │ ├── general │ ├── Main.java │ ├── ParseOneDayWorker.java │ └── package-info.java │ ├── input │ ├── InputHandler.java │ ├── InputHandlerTSV.java │ ├── SparkHadoopExperiment.java │ ├── factories │ │ ├── InputHandlerFactory.java │ │ ├── InputHandlerTSVFactory.java │ │ └── package-info.java │ └── package-info.java │ ├── logging │ ├── LoggingHandler.java │ ├── TimestampFileAppender.java │ └── package-info.java │ ├── openrdffork │ ├── IRIShorteningRenderVisitor.java │ ├── RenderVisitor.java │ ├── StandardizingPrefixDeclProcessor.java │ ├── StandardizingSPARQLParser.java │ ├── TupleExprWrapper.java │ └── package-info.java │ ├── output │ ├── OutputHandler.java │ ├── OutputHandlerAnonymizer.java │ ├── OutputHandlerTSV.java │ ├── factories │ │ ├── OutputHandlerAnonymizerFactory.java │ │ ├── OutputHandlerFactory.java │ │ ├── OutputHandlerTSVFactory.java │ │ └── package-info.java │ └── package-info.java │ ├── query │ ├── Cache.java │ ├── OpenRDFQueryHandler.java │ ├── QueryHandler.java │ ├── factories │ │ ├── OpenRDFQueryHandlerFactory.java │ │ ├── QueryHandlerFactory.java │ │ └── package-info.java │ ├── package-info.java │ └── statistics │ │ ├── NonSimplePropertyPathVisitor.java │ │ ├── OpenRDFQuerySizeCalculatorVisitor.java │ │ ├── QueryContainerSparqlStatisticsCollector.java │ │ ├── TupleExprSparqlStatisticsCollector.java │ │ └── package-info.java │ └── utility │ ├── NoURIException.java │ └── package-info.java ├── tools ├── Anonymize.py ├── QueryAnalysis.py ├── __init__.py ├── automatedBotClassification.py ├── botClassificationHelper.py ├── config.py ├── countAnything.py ├── countRdfProperties.py ├── countTools.py ├── countValid.py ├── createTestData.py ├── exampleDatasetGenerator.py ├── extract.py ├── extractCachedData.py ├── extractMonth.sh ├── extractQueryTypeDataRanking.py ├── extractQueryTypeDataset.py ├── featureVectors.py ├── fieldEntriesDaysApart.py ├── fieldRanking.py ├── generalStat.py ├── geoHeatMap.py ├── getDriveStatistics.py ├── getDriveStatistics.sh ├── getHourlyMetricCount.py ├── getSparqlStatistic.py ├── joinMonth.py ├── mergeOneColumnTsv.py ├── oldScripts │ ├── fieldRanking.py │ └── getProperties.py ├── operatorUsageStatistic.py ├── plotHourlyMetricCount.py ├── postprocess │ ├── __init__.py │ └── processdata.py ├── rankDataTypes.py ├── showData.py ├── sortByTime.py ├── utility │ ├── __init__.py │ └── utility.py ├── visualisePropertyTree │ ├── bower.json │ ├── createHtml.py │ ├── index.html │ ├── requirements.txt │ ├── script.js │ └── style.css └── xyMapping.py └── userAgentClassification ├── toolNameForUserCategory.tsv └── userAgentRegex.dat /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | 3 | # Mobile Tools for Java (J2ME) 4 | .mtj.tmp/ 5 | 6 | # Package Files # 7 | *.jar 8 | *.war 9 | *.ear 10 | 11 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 12 | hs_err_pid* 13 | 14 | # IntelliJ 15 | /out/ 16 | /.idea/ 17 | /*.iml 18 | /*.iws 19 | 20 | # mpeltonen/sbt-idea plugin 21 | .idea_modules/ 22 | 23 | # JIRA plugin 24 | atlassian-ide-plugin.xml 25 | 26 | # Crashlytics plugin (for Android Studio and IntelliJ) 27 | com_crashlytics_export_strings.xml 28 | crashlytics.properties 29 | crashlytics-build.properties 30 | fabric.properties 31 | /bin/ 32 | 33 | # Maven 34 | /target/ 35 | 36 | # Eclipse 37 | .classpath 38 | .project 39 | .checkstyle 40 | .settings 41 | 42 | #wikidata logs 43 | *.tsv 44 | test/ 45 | !test/java* 46 | 47 | #but allow preBuildQueryConfig and prefixConfig 48 | !toolMapping.tsv 49 | !standardPrefixes.tsv 50 | !propertyGroupMapping.tsv 51 | !toolNameForUserCategory.tsv 52 | 53 | #error logs 54 | /logs/ 55 | /logsExperimental/ 56 | /inputData/ 57 | 58 | /.pydevproject 59 | /benchmark/ 60 | tools/classifiedBotsData/ 61 | /outputData/ 62 | tools/visualisePropertyTree/ranking.json 63 | 64 | *.venv 65 | *.pyc 66 | 67 | 68 | tools/visualisePropertyTree/.venv 69 | tools/visualisePropertyTree/bower_components/ 70 | 71 | *.venv 72 | *.pyc 73 | 74 | 75 | # example log files 76 | exampleMonthsFolder/exampleMonth/processedLogData/ 77 | 78 | exampleMonthsFolder/ 79 | !exampleMonthsFolder/exampleMonth/rawLogData/*.tsv.gz 80 | !exampleMonthsFolder/exampleMonth/anonymousRawData/*.tsv.gz 81 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # QueryAnalysis 2 | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 3 | 4 | ## Getting Started 5 | ### Prerequisites 6 | You need to have `Maven`, `OpenJDK 8` and `Python 2` installed. 7 | 8 | ### Installing 9 | ```shell 10 | $ mvn clean package 11 | ``` 12 | 13 | ### Running the main Java log analyser 14 | ```shell 15 | # Processes the example SPARQL log files into exampleMonthsFolder/exampleMonth/processedLogData 16 | $ mvn exec:java@QueryAnalysis -Dexec.args="-w exampleMonthsFolder/exampleMonth -logging" 17 | 18 | # There are more (useful) CLI parameter available, you can list them with: 19 | $ mvn exec:java@QueryAnalysis -Dexec.args="--help" 20 | 21 | ``` 22 | 23 | **Important:** In order to not flush the command line with error messages all uncatched Runtime Exceptions are being written to the log files, residing in the logs/ folder, so please have a look at those regularly. The logs are not generated by default, so you should enable them using the `-l` option. 24 | 25 | ### Running the QueryAnalysis-Script 26 | The QueryAnalysis-Script handles both steps: Extraction using hive and processing using the java application. Extraction using hive only works on the server, but is ignored if the month exists in the months folder. To run the QueryAnalysis-Script locally, you need to provide the local months folder. 27 | ```shell 28 | $ python QueryAnalysis exampleMonth -m ../exampleMonthsFolder 29 | 30 | # You can also specify multiple months in the same directory by separating them using commas 31 | $ python QueryAnalysis exampleMonth,otherMonth -m ../exampleMonthsFolder 32 | ``` 33 | 34 | 35 | ## License 36 | The code in this repository is released under the [Apache 2.0](LICENSE.txt) license. External libraries used may have their own licensing terms. 37 | -------------------------------------------------------------------------------- /anonymization/allowedToolNames: -------------------------------------------------------------------------------- 1 | allowed ToolName 2 | -------------------------------------------------------------------------------- /anonymization/allowedUserAgents: -------------------------------------------------------------------------------- 1 | new key original key 2 | -------------------------------------------------------------------------------- /anonymization/datatypesWhitelist: -------------------------------------------------------------------------------- 1 | http://www.w3.org/2001/XMLSchema#dateTime 2 | http://www.w3.org/2001/XMLSchema#duration 3 | http://www.w3.org/2001/XMLSchema#time 4 | http://www.w3.org/2001/XMLSchema#date 5 | http://www.w3.org/2001/XMLSchema#gYearMonth 6 | http://www.w3.org/2001/XMLSchema#gYear 7 | http://www.w3.org/2001/XMLSchema#gMonthDay 8 | http://www.w3.org/2001/XMLSchema#gDay 9 | http://www.w3.org/2001/XMLSchema#gMonth 10 | http://www.w3.org/2001/XMLSchema#boolean 11 | http://www.w3.org/2001/XMLSchema#float 12 | http://www.w3.org/2001/XMLSchema#double 13 | http://www.w3.org/2001/XMLSchema#decimal 14 | http://www.w3.org/2001/XMLSchema#integer 15 | http://www.w3.org/2001/XMLSchema#nonPositiveInteger 16 | http://www.w3.org/2001/XMLSchema#negativeInteger 17 | http://www.w3.org/2001/XMLSchema#long 18 | http://www.w3.org/2001/XMLSchema#int 19 | http://www.w3.org/2001/XMLSchema#short 20 | http://www.w3.org/2001/XMLSchema#byte 21 | http://www.w3.org/2001/XMLSchema#nonNegativeInteger 22 | http://www.w3.org/2001/XMLSchema#positiveInteger 23 | http://www.w3.org/2001/XMLSchema#unsignedLong 24 | http://www.w3.org/2001/XMLSchema#unsignedInt 25 | http://www.w3.org/2001/XMLSchema#unsignedShort 26 | http://www.w3.org/2001/XMLSchema#unsignedByte 27 | -------------------------------------------------------------------------------- /anonymization/stringWhitelist: -------------------------------------------------------------------------------- 1 | com.bigdata.rdf.graph.analytics.SSSP 2 | com.bigdata.rdf.graph.analytics.BFS 3 | com.bigdata.rdf.graph.analytics.CC 4 | com.bigdata.rdf.graph.analytics.PR 5 | None 6 | Static 7 | Runtime 8 | Managed 9 | Native 10 | -------------------------------------------------------------------------------- /exampleMonthsFolder/exampleMonth/anonymousRawData/AnonymousQueryCnt01.tsv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikidata/QueryAnalysis/2ae7b6226a3ce6a7ebccd3deff0ddd9460d4aa57/exampleMonthsFolder/exampleMonth/anonymousRawData/AnonymousQueryCnt01.tsv.gz -------------------------------------------------------------------------------- /exampleMonthsFolder/exampleMonth/anonymousRawData/AnonymousQueryCnt02.tsv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikidata/QueryAnalysis/2ae7b6226a3ce6a7ebccd3deff0ddd9460d4aa57/exampleMonthsFolder/exampleMonth/anonymousRawData/AnonymousQueryCnt02.tsv.gz -------------------------------------------------------------------------------- /exampleMonthsFolder/exampleMonth/anonymousRawData/AnonymousQueryCnt03.tsv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikidata/QueryAnalysis/2ae7b6226a3ce6a7ebccd3deff0ddd9460d4aa57/exampleMonthsFolder/exampleMonth/anonymousRawData/AnonymousQueryCnt03.tsv.gz -------------------------------------------------------------------------------- /exampleMonthsFolder/exampleMonth/rawLogData/QueryCnt01.tsv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikidata/QueryAnalysis/2ae7b6226a3ce6a7ebccd3deff0ddd9460d4aa57/exampleMonthsFolder/exampleMonth/rawLogData/QueryCnt01.tsv.gz -------------------------------------------------------------------------------- /exampleMonthsFolder/exampleMonth/rawLogData/QueryCnt02.tsv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikidata/QueryAnalysis/2ae7b6226a3ce6a7ebccd3deff0ddd9460d4aa57/exampleMonthsFolder/exampleMonth/rawLogData/QueryCnt02.tsv.gz -------------------------------------------------------------------------------- /exampleMonthsFolder/exampleMonth/rawLogData/QueryCnt03.tsv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikidata/QueryAnalysis/2ae7b6226a3ce6a7ebccd3deff0ddd9460d4aa57/exampleMonthsFolder/exampleMonth/rawLogData/QueryCnt03.tsv.gz -------------------------------------------------------------------------------- /parserSettings/standardPrefixes.tsv: -------------------------------------------------------------------------------- 1 | prefix uri complexity 2 | hint http://www.bigdata.com/queryHints# simple 3 | hint http://www.bigdata.com/queryHints# simple 4 | gas http://www.bigdata.com/rdf/gas# simple 5 | bds http://www.bigdata.com/rdf/search# simple 6 | bd http://www.bigdata.com/rdf# simple 7 | schema http://schema.org/ simple 8 | cc http://creativecommons.org/ns# simple 9 | geo http://www.opengis.net/ont/geosparql# simple 10 | prov http://www.w3.org/ns/prov# complex 11 | xsd http://www.w3.org/2001/XMLSchema# simple 12 | skos http://www.w3.org/2004/02/skos/core# simple 13 | owl http://www.w3.org/2002/07/owl# simple 14 | rdf http://www.w3.org/1999/02/22-rdf-syntax-ns# simple 15 | rdfs http://www.w3.org/2000/01/rdf-schema# simple 16 | wdata http://www.wikidata.org/wiki/Special:EntityData/ simple 17 | wdno http://www.wikidata.org/prop/novalue/ simple 18 | prn http://www.wikidata.org/prop/reference/value-normalized/ complex 19 | prv http://www.wikidata.org/prop/reference/value/ complex 20 | pr http://www.wikidata.org/prop/reference/ complex 21 | pqn http://www.wikidata.org/prop/qualifier/value-normalized/ complex 22 | pqv http://www.wikidata.org/prop/qualifier/value/ complex 23 | pq http://www.wikidata.org/prop/qualifier/ complex 24 | psn http://www.wikidata.org/prop/statement/value-normalized/ complex 25 | psv http://www.wikidata.org/prop/statement/value/ complex 26 | ps http://www.wikidata.org/prop/statement/ complex 27 | wdv http://www.wikidata.org/value/ complex 28 | wdref http://www.wikidata.org/reference/ simple 29 | p http://www.wikidata.org/prop/ simple 30 | wds http://www.wikidata.org/entity/statement/ complex 31 | wdt http://www.wikidata.org/prop/direct/ simple 32 | wd http://www.wikidata.org/entity/ simple 33 | wikibase http://wikiba.se/ontology# simple 34 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/AllSubjectsAllPredicatesEntity.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?s ?p 2 | WHERE { 3 | ?s ?p 4 | } 5 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/AllSubjectsAllPredicatesEntityStatement.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?s ?p 2 | WHERE { 3 | ?s ?p 4 | } 5 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/AllSubjectsAllPredicatesReference.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?s ?p 2 | WHERE { 3 | ?s ?p 4 | } 5 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/GNDIDThisWithID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT * 2 | WHERE { 3 | ?gnd . 4 | ?gnd ?id 5 | } 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/InCHIKeyThis.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?inchikey 2 | WHERE { 3 | ?inchikey . 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/aboutThisGNDIDWithFilterLimitOffset.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?cid ?p ?o 2 | WHERE { 3 | ?cid wdt:P227 ?gnd. 4 | ?article schema:about ?cid . 5 | filter( regex(str(?article), "en.wikipedia.org" )) . 6 | ?cid ?p ?o 7 | } 8 | limit 10000 9 | offset 1069000 10 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/aboutThisGNDIDWithFilterLimitOffsetDistinct.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?s ?article ?pLabel ?oLabel 2 | WHERE { 3 | ?s wdt:P227 ?gnd. 4 | ?article schema:about ?s . 5 | filter(regex(str(?article), "en.wikipedia.org" )) . 6 | ?s ?pLabel ?oLabel . 7 | } 8 | limit 1000 9 | offset 565000 10 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/aboutThisGNDIDWithFilterLimitOffsetDistinctOptionalsBinds.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT * 2 | WHERE { 3 | ?s wdt:P227 ?gnd. 4 | ?article schema:about ?s . 5 | filter(regex(str(?article), "en.wikipedia.org" )) . 6 | ?s ?p ?o . 7 | optional { 8 | ?wd wikibase:directClaim ?p . 9 | ?wd rdfs:label ?pLabelTemp . 10 | filter langMatches(lang(?pLabelTemp), "en") . 11 | } . 12 | BIND ( IF (BOUND (?pLabelTemp), ?pLabelTemp, ?p ) as ?pLabel ) . 13 | optional { 14 | ?o rdfs:label ?oLabelTemp . 15 | filter langMatches(lang(?oLabelTemp), "en") . 16 | } . 17 | BIND ( IF (BOUND (?oLabelTemp), ?oLabelTemp, ?o ) as ?oLabel ) . 18 | } 19 | limit 1000 20 | offset 227000 21 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/aboutThisGNDIDWithFilterLimitOffsetDistinctOptionalsBinds2.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT * 2 | WHERE { 3 | ?s wdt:P227 ?gnd. 4 | ?article schema:about ?s . 5 | filter(regex(str(?article), "en.wikipedia.org" )) . 6 | ?s ?p ?o . 7 | optional { 8 | ?wd wikibase:directClaim ?p . 9 | ?wd rdfs:label ?pLabelTemp . 10 | filter (lang(?pLabelTemp) = "en") . 11 | } . 12 | BIND ( IF (BOUND (?pLabelTemp), ?pLabelTemp, ?p ) as ?pLabel ) . 13 | optional { 14 | ?o rdfs:label ?oLabelTemp .filter (lang(?oLabelTemp) = "en") . 15 | } . 16 | BIND ( IF (BOUND (?oLabelTemp), ?oLabelTemp, ?o ) as ?oLabel ) . 17 | } 18 | limit 1000 19 | offset 885000 20 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/acidDissociationConstantThis.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?pKa2 2 | WHERE { 3 | wd:Q349391 wdt:P1117 ?pKa2 . 4 | } 5 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/allRelationsThis.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX bd: 2 | PREFIX wdt: 3 | PREFIX hint: 4 | PREFIX wikibase: 5 | PREFIX skos: 6 | PREFIX wd: 7 | 8 | SELECT ?relation ?itemLabel ?lang 9 | WHERE { 10 | wd:Q5133110 ?relation ?item 11 | SERVICE wikibase:label { bd:serviceParam wikibase:language "en" } 12 | } 13 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/askWhereInstaceOf.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wd: 2 | PREFIX wdt: 3 | Ask Where { 4 | wd:Q1781699 wdt:P31 wd:Q2481479. 5 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/askWhereLocatedInTheAdministrativeTerritorialEntity.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wd: 2 | PREFIX wdt: 3 | 4 | Ask Where { 5 | wd:Q478031 wdt:P131 wd:Q2574808. 6 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/bot2.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX v: 2 | PREFIX q: 3 | PREFIX ps: 4 | PREFIX pq: 5 | SELECT ?q ?x { wd:Q2557017 wdt:P131* ?q . ?q wdt:P300 ?x } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/bot222.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX v: 2 | PREFIX q: 3 | PREFIX ps: 4 | PREFIX pq: 5 | SELECT ?item WHERE { 6 | ?item wdt:P31 wd:Q5 . 7 | ?item wdt:P21 wd:Q6581072 . 8 | ?item wdt:P106 wd:Q1028181 . 9 | ?item wdt:P569 ?time0 . 10 | FILTER ( ?time0 >= "1900-01-01T00:00:00Z"^^xsd:dateTime && ?time0 <= "1920-01-01T00:00:00Z"^^xsd:dateTime )} -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/coordianteLocationThis.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?Mayor 2 | WHERE { 3 | ?Mayor . 4 | } 5 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/coordinateLocationThisViaBind.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?itemLabel ?coord 2 | WHERE { 3 | BIND (wd:Q534914 AS ?item). 4 | ?item wdt:P625 ?coord 5 | SERVICE wikibase:label { 6 | bd:serviceParam wikibase:language "de". 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/distinctInstanceOfThis.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?type ?typeLabel 2 | WHERE { 3 | wd:Q349391 wdt:P31 ?type. 4 | SERVICE wikibase:label { bd:serviceParam wikibase:language "en". } 5 | } 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/doesFootballerHasAEnglishWikipediaArticle.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX schema: 2 | PREFIX wd: 3 | 4 | SELECT * 5 | WHERE 6 | { ?wikipediaLink schema:about wd:Q11948 . 7 | ?wikipediaLink schema:inLanguage "en" . 8 | ?wikipediaLink schema:isPartOf 9 | } 10 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/entityAllOptionalLabelImageCoordinatesArticleInLanguage.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?label ?image ?coordinates ?wikilink 2 | where { 3 | OPTIONAL{ 4 | rdfs:label ?label . 5 | FILTER (lang(?label)=?lang) . 6 | values (?lang ?lang_) { ("it" 1) ("en" 2) ("de" 3) ("fr" 4) ("it" 5)} 7 | } 8 | OPTIONAL{ 9 | wdt:P18 ?image . 10 | } 11 | OPTIONAL{ 12 | wdt:P625 ?coordinates . 13 | } 14 | OPTIONAL{ 15 | ?wikilink a schema:Article ; 16 | schema:about ; 17 | schema:inLanguage "it" ; 18 | schema:isPartOf . 19 | } 20 | } 21 | order by ?lang_ 22 | 23 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/entityAllPropertiesAllObjectsWithInstanceAndLabel.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX entity: 2 | PREFIX wdt: 3 | PREFIX rdfs: 4 | 5 | select ?property ?propertyLabel ?item ?itemLabel ?itemInstanceOf ?itemInstanceOfLabe 6 | where { 7 | hint:Query hint:optimizer "None" . 8 | entity:Q402770 ?property ?item . 9 | ?prop wikibase:directClaim ?property . 10 | ?item wdt:P31 ?itemInstanceOf . 11 | 12 | ?item rdfs:label ?itemLabel . 13 | ?itemInstanceOf rdfs:label ?itemInstanceOfLabel . 14 | ?prop rdfs:label ?propertyLabel . 15 | 16 | FILTER (lang(?propertyLabel) = "en") 17 | FILTER (lang(?itemLabel) = "en") 18 | FILTER (lang(?itemInstanceOfLabel) = "en") 19 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/entityAllPropertiesAndObjects.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX bd: 2 | PREFIX ps: 3 | PREFIX wd: 4 | PREFIX wdt: 5 | PREFIX rdfs: 6 | PREFIX pq: 7 | PREFIX wikibase: 8 | PREFIX p: 9 | select * 10 | where { 11 | ?p0 ?e1 12 | } 13 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/entityLabelImageAbstractLocationEnglishWikipediaArticle.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX dbo: 2 | select ?label ?abstract ?image ?lat ?long ?wikilink 3 | where { 4 | OPTIONAL{ 5 | rdfs:label ?label . 6 | FILTER (lang(?label)=?lang) . 7 | values (?lang ?lang_) { ("en" 1) ("en" 2) ("de" 3) ("fr" 4) ("it" 5)} 8 | } 9 | OPTIONAL{ 10 | wdt:P18 ?image . 11 | } 12 | OPTIONAL{ 13 | dbo:abstract ?abstract . 14 | FILTER (lang(?abstract)="en" ) 15 | } 16 | OPTIONAL{ 17 | wdt:P625 ?lat . 18 | } 19 | OPTIONAL{ 20 | ?wikilink a schema:Article; 21 | schema:about ; 22 | schema:inLanguage "en"; 23 | schema:isPartOf . 24 | } 25 | } 26 | order by ?lang_ 27 | 28 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/entityLabelImageDescriptionAllOptional.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX dbo: 2 | select ?label ?desc ?image 3 | where { 4 | OPTIONAL { wd:Q6669738 rdfs:label ?label . 5 | FILTER (lang(?label)="en") . } 6 | OPTIONAL{ wd:Q6669738 wdt:P18 ?image . } 7 | OPTIONAL{ wd:Q6669738 schema:description ?desc. FILTER (lang(?desc)="en") . } 8 | } 9 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/entityLabelInSpecificLanguage.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?serieLabel 2 | WHERE { 3 | wd:Q2655470 rdfs:label ?serieLabel . 4 | FILTER(LANG(?serieLabel) = "pt") 5 | } 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/entityMultipleOptionalProperties.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?class ?birth ?death ?gender ?country ?imdb ?image ?occupation 2 | WHERE { 3 | OPTIONAL {wd:Q214601 wdt:P31 ?class.} 4 | OPTIONAL {wd:Q214601 wdt:P569 ?birth.} 5 | OPTIONAL {wd:Q214601 wdt:P570 ?death.} 6 | OPTIONAL {wd:Q214601 wdt:P21 ?gender.} 7 | OPTIONAL {wd:Q214601 wdt:P27 ?country.} 8 | OPTIONAL {wd:Q214601 wdt:P345 ?imdb.} 9 | OPTIONAL {wd:Q214601 wdt:P18 ?image.} 10 | OPTIONAL {wd:Q214601 wdt:P106 ?occupation.} 11 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/entityMultipleOptionalProperties2.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?type ?typeLabel ?lieu ?location ?time ?starttime ?endtime ?inception ?birthdate 2 | WHERE { 3 | OPTIONAL{wd:Q349391 wdt:P31 ?type}. 4 | OPTIONAL{wd:Q349391 wdt:P131 ?lieu}. 5 | OPTIONAL{wd:Q349391 wdt:P625 ?location}. 6 | OPTIONAL{wd:Q349391 wdt:P585 ?time}. 7 | OPTIONAL{wd:Q349391 wdt:P580 ?starttime}. 8 | OPTIONAL{wd:Q349391 wdt:P582 ?endtime}. 9 | OPTIONAL{wd:Q349391 wdt:P571 ?inception}. 10 | OPTIONAL{wd:Q349391 wdt:P569 ?birthdate}. 11 | SERVICE wikibase:label { bd:serviceParam wikibase:language 'en'. } 12 | } 13 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/entityMultipleOptionalProperties3.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?type ?typeLabel ?lieu ?location ?time WHERE { 2 | OPTIONAL{wd:Q349391 wdt:P31 ?type}. 3 | OPTIONAL{wd:Q349391 wdt:P131 ?lieu}. 4 | OPTIONAL{wd:Q349391 wdt:P625 ?location}. 5 | OPTIONAL{wd:Q349391 wdt:P585 ?time}. 6 | SERVICE wikibase:label { bd:serviceParam wikibase:language 'en'. } 7 | } 8 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/entityMultipleOptionalProperties4.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?type ?typeLabel ?lieu ?location ?time ?starttime ?inception WHERE { 2 | OPTIONAL{wd:Q349391 wdt:P31 ?type}. 3 | OPTIONAL{wd:Q349391 wdt:P131 ?lieu}. 4 | OPTIONAL{wd:Q349391 wdt:P625 ?location}. 5 | OPTIONAL{wd:Q349391 wdt:P585 ?time}. 6 | OPTIONAL{wd:Q349391 wdt:P580 ?starttime}. 7 | OPTIONAL{wd:Q349391 wdt:P571 ?inception}. 8 | SERVICE wikibase:label { bd:serviceParam wikibase:language 'en'. } 9 | 10 | } 11 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/entityOptionalInstanceOfOptionalIMDb_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?class ?imdb 2 | WHERE { 3 | OPTIONAL {wd:Q349391 wdt:P31 ?class.} 4 | OPTIONAL {wd:Q349391 wdt:P345 ?imdb.} 5 | } 6 | 7 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/entityStatementAllPredicatesAndObjects.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?p ?o 2 | WHERE { 3 | ?p ?o 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/equivalentClassThis.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT * 2 | WHERE { 3 | ?same 4 | } 5 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/findWikidataId.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX rdfs: 2 | 3 | SELECT ?wikidataId 4 | WHERE 5 | { ?wikidataId rdfs:label "Rob van Dijk"@en } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/gasProgramSSSP.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX gas: 2 | SELECT ?item ?linkTo ?depth { 3 | SERVICE gas:service { 4 | gas:program gas:gasClass "com.bigdata.rdf.graph.analytics.SSSP"; 5 | gas:in wd:Q1133482; 6 | gas:traversalDirection "Forward"; 7 | gas:out ?item; 8 | gas:out1 ?depth; 9 | gas:linkType wdt:P279 . 10 | } 11 | OPTIONAL { 12 | ?item wdt:P279 ?linkTo 13 | } 14 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/geneAndInformationWithConstraints.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wdt: 2 | PREFIX wd: 3 | PREFIX qualifier: 4 | SELECT ?start ?end ?uniqueID ?strand ?uri ?entrezGeneID ?name ?description ?refSeq 5 | WHERE { 6 | ?gene wdt:P279 wd:Q7187; 7 | wdt:P703 ?strain; 8 | wdt:P351 ?uniqueID; 9 | wdt:P351 ?entrezGeneID; 10 | wdt:P2393 ?name; 11 | rdfs:label ?description; 12 | wdt:P644 ?start; 13 | wdt:P645 ?end; 14 | wdt:P2548 ?wdstrand; 15 | p:P644 ?chr. 16 | OPTIONAL { 17 | ?chr qualifier:P2249 ?refSeq. 18 | } 19 | FILTER(?refSeq = "NC_001133.9") ?strain wdt:P685 '559292'. 20 | bind( 21 | IF(?wdstrand = wd:Q22809680, '1', '-1') as ?strand). bind(str(?gene) as ?uri). 22 | filter ( !(xsd:integer(?start) > 57755 || xsd:integer(?end) < 57355)) 23 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/generalInformationAboutPerson.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?person ?sportLabel ?dateOfBirth ?personLabel ?nationalityLabel ?nationalityStart ?nationalityEnd ?placeOfBirthLabel ?countryOfBirthLabel ?image (CONCAT("_",str(?gallery)) as ?_gallery)?occupationLabel 2 | WHERE { 3 | ?person wdt:P569 ?dateOfBirth; 4 | p:P27 ?nationalityStmt; 5 | wdt:P18 ?image; 6 | wdt:P19 ?placeOfBirth. 7 | ?nationalityStmt ps:P27 ?nationality. 8 | ?placeOfBirth wdt:P17 ?countryOfBirth. 9 | schema:about ?person. 10 | OPTIONAL{ ?person wdt:P641 ?sport } 11 | OPTIONAL{ ?person wdt:P106 ?occupation } 12 | OPTIONAL{ ?person wdt:P2416 ?discCompetedIn } 13 | OPTIONAL{ ?person wdt:P373 ?gallery } 14 | OPTIONAL{ ?nationalityStmt pq:P580 ?nationalityStart } 15 | OPTIONAL{ ?nationalityStmt pq:P582 ?nationalityEnd } 16 | SERVICE wikibase:label { bd:serviceParam wikibase:language "fr" } 17 | } 18 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/hasLabel.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX ff: 2 | PREFIX bd: 3 | PREFIX owl: 4 | PREFIX wdt: 5 | PREFIX wikibase: 6 | PREFIX skos: 7 | PREFIX rdfs: 8 | PREFIX ogc: 9 | PREFIX dbpedia: 10 | PREFIX lexinfo: 11 | PREFIX geom: 12 | PREFIX wd: 13 | PREFIX dbo: 14 | PREFIX lemon: 15 | PREFIX lgdo: 16 | PREFIX rdf: 17 | PREFIX geo-pos: 18 | PREFIX lgdr: 19 | PREFIX omgeo: 20 | PREFIX foaf: 21 | PREFIX bn-lemon: 22 | PREFIX om: 23 | 24 | SELECT DISTINCT ?class ?classLabel 25 | WHERE 26 | { ?class rdfs:label "Test"@de 27 | SERVICE wikibase:label 28 | { bd:serviceParam wikibase:language "de" } 29 | } 30 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/hasLabelInstanceOfThis.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wd: 2 | PREFIX wdt: 3 | 4 | SELECT distinct ?cat WHERE { 5 | 6 | { 7 | Optional{ 8 | ?cid wdt:P31 ?cat . 9 | ?cid rdfs:label "cat"@en .} 10 | } 11 | UNION 12 | { Optional{ 13 | ?cid wdt:P31 ?cat . 14 | ?cid rdfs:label "CAT"@en . } 15 | } 16 | UNION{ 17 | Optional{ 18 | ?cid wdt:P31 ?cat . 19 | ?cid rdfs:label "Cat"@en . } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/hasLabelThisIsSubclass.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX ff: 2 | PREFIX bd: 3 | PREFIX owl: 4 | PREFIX wdt: 5 | PREFIX wikibase: 6 | PREFIX skos: 7 | PREFIX rdfs: 8 | PREFIX ogc: 9 | PREFIX dbpedia: 10 | PREFIX lexinfo: 11 | PREFIX geom: 12 | PREFIX wd: 13 | PREFIX dbo: 14 | PREFIX lemon: 15 | PREFIX lgdo: 16 | PREFIX rdf: 17 | PREFIX geo-pos: 18 | PREFIX lgdr: 19 | PREFIX omgeo: 20 | PREFIX foaf: 21 | PREFIX bn-lemon: 22 | PREFIX om: 23 | 24 | SELECT DISTINCT ?class ?classLabel 25 | WHERE 26 | { ?class rdfs:label "Cat"@en . 27 | ?class wdt:P279 ?super 28 | SERVICE wikibase:label 29 | { bd:serviceParam wikibase:language "de" } 30 | } 31 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/imageThis.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?image 2 | WHERE { 3 | wdt:P18 ?image 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/imageToSocialNetworksID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?_image 2 | WHERE { 3 | ?q wdt:P3430 "w6wt1qhz". 4 | OPTIONAL { 5 | ?q wdt:P18 ?_image. 6 | } 7 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/infoAboutHuman.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?s ?prof ?viaf ?gnd ?repec WHERE { 2 | ?s . 3 | ?s ?label "Tim Berners-Lee"@en . 4 | OPTIONAL { 5 | ?s ?label "Tim Berners-Lee"@de. 6 | } 7 | OPTIONAL { 8 | ?s ?label "Tim Berners-Lee". 9 | } 10 | OPTIONAL { ?s ?prof .} 11 | OPTIONAL { ?s ?viaf. } 12 | OPTIONAL { ?s ?gnd. } 13 | OPTIONAL { ?s ?repec. } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/instanceOfSubclassOfMultipleFiltersAndOptional.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX schema: 2 | 3 | SELECT DISTINCT ?c2 ?c5 ?x ?y WHERE { 4 | { 5 | OPTIONAL {?item ?x wd:Q532.} 6 | OPTIONAL {?item ?y wd:Q486972.} 7 | ?item (wdt:P31) ?c. 8 | ?c (wdt:P279+) ?c2. 9 | ?c2 rdfs:label ?c5. 10 | FILTER(?item = wd:Q875800) 11 | FILTER((LANG(?c5)) = "en") 12 | FILTER(?c2 = wd:Q43229 || ?c2 = wd:Q17334923 ||?c2 = wd:Q215627 ) 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/instanceOfSubclassOfTVSeries.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wdt: 2 | PREFIX wd: 3 | 4 | SELECT DISTINCT ?tvSeries 5 | WHERE 6 | { ?tvSeries wdt:P31 ?instance . 7 | ?instance (wdt:P279)* wd:Q5398426 8 | } 9 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/instanceOfSubclassRelations2.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?UnitClass ?UnitClassLabel ?wayPoint ?wayPointLabel ?wayPointParent ?wayPointParentLabel 2 | WHERE 3 | { 4 | Values ?startPoint { 5 | wd:Q47574 wd:Q309314 6 | 7 | } 8 | BIND(wd:Q4462543 as ?endPoint) 9 | ?endPoint wdt:P31|wdt:P279 ?UnitClass. 10 | ?UnitClass wdt:P279* ?startPoint. 11 | ?wayPoint ^wdt:P279* ?UnitClass. 12 | ?wayPointParent ^wdt:P279 ?wayPoint. 13 | ?wayPointParent wdt:P279* ?startPoint. 14 | SERVICE wikibase:label { 15 | bd:serviceParam wikibase:language "de, en" 16 | } 17 | } 18 | 19 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/instanceOfSubclassRelations3.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?UnitClass ?UnitClassLabel ?wayPoint ?wayPointLabel ?wayPointParent ?wayPointParentLabel 2 | WHERE { 3 | Values ?startPoint { 4 | wd:Q47574 5 | wd:Q309314 6 | } 7 | BIND(wd:Q453224 as ?endPoint) 8 | ?endPoint wdt:P31?|wdt:P279? ?UnitClass. 9 | ?UnitClass wdt:P31*/wdt:P279* ?startPoint. 10 | ?UnitClass wdt:P31*/wdt:P279* ?wayPoint. 11 | ?wayPoint wdt:P31|wdt:P279 ?wayPointParent. 12 | ?wayPointParent wdt:P279* ?startPoint. 13 | SERVICE wikibase:label { 14 | bd:serviceParam wikibase:language "de, en" 15 | } 16 | } 17 | 18 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/instanceOfThis.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item ?itemLabel 2 | WHERE { 3 | wd:Q129987 wdt:P31 ?item . 4 | SERVICE wikibase:label { bd:serviceParam wikibase:language "en" } 5 | } 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/instanceOfThisAnySubclassOfWithLabelAndFilter.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX schema: 2 | 3 | SELECT DISTINCT ?c2 ?c5 4 | WHERE { 5 | { 6 | ?item (wdt:P31) ?c. 7 | ?c (wdt:P279+) ?c2. 8 | ?c2 rdfs:label ?c5. 9 | FILTER(?item = wd:Q5956491) 10 | FILTER((LANG(?c5)) = "en") 11 | FILTER(?c2 = wd:Q43229 || ?c2 = wd:Q17334923 ||?c2 = wd:Q215627 ) 12 | } 13 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/itemsWithPicturesFromCatalogue.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item ?picture 2 | WHERE { 3 | { 4 | SELECT DISTINCT ?item ?itemLabel ?numero (SAMPLE( ?pic) AS ?picture) 5 | WHERE { 6 | ?item ?catalogStatement . 7 | ?catalogStatement ?numero ; 8 | . 9 | OPTIONAL { 10 | ?item ?pic . 11 | } 12 | SERVICE { "en" . } 13 | } 14 | GROUP BY ?item ?itemLabel ?numero 15 | ORDER BY ASC ( ?numero) 16 | } 17 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/linkedDataPilotSwitzerlandCoordinateLocationThis.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?Mayor 2 | WHERE { 3 | ?Mayor . 4 | } 5 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/prevAndNextFilmInSeries.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?next ?nextLabel ?prev ?prevLabel 2 | WHERE{ 3 | ?film wdt:P31 wd:Q11424; 4 | rdfs:label "Transformers"@fr. 5 | OPTIONAL{ 6 | ?film p:P179 ?serieStmt . 7 | OPTIONAL{ 8 | ?serieStmt pq:P155 ?prev 9 | } . 10 | OPTIONAL{ 11 | ?serieStmt pq:P156 ?next 12 | } 13 | } 14 | OPTIONAL{ 15 | ?film wdt:P156 ?next 16 | } 17 | OPTIONAL{ 18 | ?film wdt:P155 ?prev 19 | } 20 | SERVICE wikibase:label { bd:serviceParam wikibase:language "fr" }} -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1027_6765.preBuildQueryType: -------------------------------------------------------------------------------- 1 | ASK 2 | WHERE { 3 | ?var1 "114841100". 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1049_0.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 ?var3 ?var4 2 | WHERE { 3 | ?var4 . 4 | ?var3 ?var4 ; 5 | ?var2 ; 6 | ?var1 . 7 | ?var2 . 8 | FILTER ( ( !( isLITERAL ( ) ) ) 9 | ) . 10 | } 11 | 12 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1059_6997.preBuildQueryType: -------------------------------------------------------------------------------- 1 | ASK 2 | WHERE { 3 | ?var1 "string1". 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1088_0.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 . 4 | } 5 | LIMIT 10000 6 | 7 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1088_15.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 2 | WHERE { 3 | ?var1 ?var2 ; 4 | ?var3 . 5 | VALUES ( ?var2 ) { 6 | ( "string1" ) 7 | ( "string2" ) 8 | } 9 | } 10 | ORDER BY ASC( ?var3 ) 11 | 12 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1145_24.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1173_2.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT( COUNT ( ?var1 ) AS ?var2 ) 2 | WHERE { 3 | ?var1 ( / *) . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1181_6600.preBuildQueryType: -------------------------------------------------------------------------------- 1 | ASK 2 | WHERE { 3 | ?var1 "string1". 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1191_6821.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1Label ?var1 2 | WHERE { 3 | ?var1 . 4 | ?var1 ?var1Label . 5 | FILTER ( ( ( "string1" = LANG ( ?var1Label ) ) ) 6 | ) . 7 | } 8 | 9 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1203_1.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 . 4 | } 5 | LIMIT 10000 6 | 7 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1203_11.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 . 4 | } 5 | LIMIT 10000 6 | 7 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1244_152.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?var1 2 | WHERE { 3 | ?var1 "string1"@fr . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1279_2.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:127_15.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1289_21.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1309_21.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1310_0.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 2 | WHERE { 3 | ?var2 ?var1 . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1406_20.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 ?var3 2 | WHERE { 3 | { 4 | ?var1 "string1". 5 | } 6 | ?var2 ?var1 . 7 | { 8 | ?var2 "string2". 9 | } 10 | UNION 11 | { 12 | ?var2 "string3". 13 | } 14 | ?var1 ?var3 . 15 | SERVICE { 16 | "en". 17 | } 18 | } 19 | 20 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1407_10.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var1Label ?var2 ?var2Label ?var3 ?var3Label ?var4 ?var4Label 2 | WHERE { 3 | ?var3 ?var5 . 4 | ?var3 ?var6 . 5 | ?var3 ?var7 . 6 | ?var3 . 7 | ?var4 ?var8 . 8 | ?var4 ?var9 . 9 | ?var4 . 10 | ?var1 ?var5 ?var10 . 11 | ?var10 ?var6 ?var2 . 12 | ?var10 ?var9 . 13 | SERVICE { 14 | "en". 15 | } 16 | } 17 | 18 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1454_1.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 ?var3 ( MIN ( ?var4 ) AS ?var4 ) 2 | WHERE { 3 | ?var1 . 4 | SERVICE { 5 | "[AUTO_LANGUAGE], en". 6 | } 7 | OPTIONAL { 8 | ?var1 ?var2 . 9 | } 10 | OPTIONAL { 11 | ?var1 ?var3 . 12 | } 13 | OPTIONAL { 14 | ?var1 ?var4 . 15 | } 16 | } 17 | GROUP BY ?var1 ?var2 ?var3 18 | LIMIT 200 19 | OFFSET 75400 20 | 21 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1454_51.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?var1 ?var1Label ?var2 ?var3 ?var4 ?var4Label ?var5 ?var6 ?var7 ?var8 ?var9 ?var9Label ?var10 ?var11 2 | WHERE { 3 | ?var1 ?var12Label "string1"@it . 4 | ?var13 ?var1 . 5 | ?var13 "string2". 6 | ?var13 . 7 | SERVICE { 8 | "it". 9 | } 10 | OPTIONAL { 11 | ?var1 ?var3 . 12 | } 13 | OPTIONAL { 14 | ?var1 ?var4 . 15 | } 16 | OPTIONAL { 17 | ?var1 ?var5 . 18 | } 19 | OPTIONAL { 20 | ?var1 ?var6 . 21 | } 22 | OPTIONAL { 23 | ?var1 ?var7 . 24 | } 25 | OPTIONAL { 26 | ?var1 ?var8 . 27 | } 28 | OPTIONAL { 29 | ?var1 ?var9 . 30 | } 31 | OPTIONAL { 32 | ?var1 ?var10 . 33 | } 34 | OPTIONAL { 35 | ?var1 ?var11 . 36 | } 37 | OPTIONAL { 38 | ?var1 ?var14 . 39 | } 40 | OPTIONAL { 41 | ?var1 ?var15 . 42 | } 43 | OPTIONAL { 44 | ?var1 ?var16 . 45 | } 46 | OPTIONAL { 47 | ?var1 ?var17 . 48 | } 49 | OPTIONAL { 50 | ?var1 ?var18 . 51 | } 52 | } 53 | 54 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1454_999.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 "string1". 4 | FILTER ( ( isLITERAL ( "string1" ) ) 5 | ) . 6 | } 7 | 8 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1466_0.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1Label ?var2 ?var3 ?var4 ?var5 2 | WHERE { 3 | ?var5 ?var4 ; 4 | ?var3 ; 5 | ; 6 | ; 7 | ?var6 . 8 | ?var6 ?var3 ; 9 | ?var7 . 10 | ?var3 ?var2 ; 11 | ?var1Label . 12 | FILTER ( ( !( isLITERAL ( ) ) ) 13 | ) . 14 | FILTER ( ( ( ?var7 = ) ) 15 | ) . 16 | FILTER ( ( ( "string1" = LANG ( ?var1Label ) ) ) 17 | ) . 18 | } 19 | 20 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1485_13.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var2 ?var1 . 4 | VALUES ( ?var2 ) { 5 | ( ) 6 | } 7 | } 8 | 9 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1489_6788.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?var1 ?var1Label 2 | WHERE { 3 | ?var2 . 4 | ?var2 ?var1 . 5 | ?var1 . 6 | SERVICE { 7 | "nl". 8 | } 9 | } 10 | ORDER BY ASC( ?var1Label ) 11 | 12 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1512_79.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1Label ?var2 ?var3 ?var4 ?var5 2 | WHERE { 3 | ?var5 ?var4 ; 4 | ?var3 ; 5 | ?var6 ; 6 | ; 7 | ?var7 . 8 | ?var7 ?var3 ; 9 | ?var8 . 10 | ?var3 ?var2 ; 11 | ?var1Label . 12 | FILTER ( ( ( ?var8 = ) ) 13 | ) . 14 | FILTER ( ( ( "string1" = LANG ( ?var1Label ) ) ) 15 | ) . 16 | } 17 | 18 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:152_181.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1Label ?var2Label 2 | WHERE { 3 | SERVICE { 4 | "string1". 5 | ?var1Label . 6 | ?var2Label . 7 | } 8 | } 9 | 10 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1583_28.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:1593_1098.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 ?var3 2 | WHERE { 3 | ?var3 "string1". 4 | ?var2 ?var3 . 5 | ?var1 ?var2 ; 6 | * . 7 | FILTER ( ( isLITERAL ( "string1" ) ) 8 | ) . 9 | } 10 | 11 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:175_2695.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 ?var2Label 2 | WHERE { 3 | ?var1 . 4 | ?var1 ?var2 . 5 | SERVICE { 6 | "[AUTO_LANGUAGE], en". 7 | } 8 | } 9 | LIMIT 200 10 | OFFSET 30600 11 | 12 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:197_437.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:217_2527.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?var1 ?var2Label ?var3 ?var4 2 | WHERE { 3 | ?var1 "string1"@ja ; 4 | ?var2Label ; 5 | ?var4 ; 6 | ?var3 . 7 | FILTER ( ( ( LANG ( ?var2Label ) = "string2" ) ) 8 | ) . 9 | } 10 | ORDER BY DESC( ?var4 ) 11 | 12 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:218_18.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 2 | WHERE { 3 | ?var1 . 4 | ?var1 ?var2 . 5 | FILTER ( ( ( ?var2 > "0"^^ ) ) 6 | ) . 7 | } 8 | ORDER BY DESC( ?var2 ) 9 | LIMIT 1000 10 | 11 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:249_6798.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 ?var3 ?var4 2 | WHERE { 3 | ?var4 ?var3 . 4 | ?var3 ; 5 | ?var2 ; 6 | ?var1 . 7 | } 8 | 9 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:263_6718.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1Label ?var1 2 | WHERE { 3 | ?var1 . 4 | ?var1 ?var1Label . 5 | FILTER ( ( EXISTS { 6 | . 7 | } 8 | ) 9 | ) . 10 | FILTER ( ( ( "string1" = LANG ( ?var1Label ) ) ) 11 | ) . 12 | } 13 | 14 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:266_211.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?var1 ?var1Label ?var2 2 | WHERE { 3 | BIND ( AS ?var1 ). 4 | SERVICE { 5 | "de". 6 | } 7 | } 8 | 9 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:270_22.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 2 | WHERE { 3 | ?var1 . 4 | ?var1 ?var2 . 5 | FILTER ( ( ( ?var2 > "0"^^ ) ) 6 | ) . 7 | } 8 | ORDER BY DESC( ?var2 ) 9 | LIMIT 250 10 | 11 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:281_9725.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:282_199.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 ?var3 ?var4 ?var5 ( STR ( ?var2 ) AS ?var6 ) 2 | WHERE { 3 | ?var1 ?var2 ; 4 | ?var3 ; 5 | ?var4 . 6 | OPTIONAL { 7 | ?var1 ?var5 . 8 | } 9 | } 10 | OFFSET 0 11 | 12 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:290_169.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 ?var3 ?var4 2 | WHERE { 3 | { ?var2 ; 4 | ?var3 ; 5 | ?var4 . } 6 | OPTIONAL { ?var1 . } } 7 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:300_56.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var2 . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:316_160.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 2 | WHERE { 3 | ?var2 ?var1 . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:323_6697.preBuildQueryType: -------------------------------------------------------------------------------- 1 | ASK 2 | WHERE { 3 | ?var1 "string1". 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:350_248.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT( ?var1 AS ?var2 )( ?var1Label AS ?var3 )( COUNT ( ?var4 ) AS ?var5 ) 2 | WHERE { 3 | ?var4 ; 4 | ( / ) ?var1 . 5 | SERVICE { 6 | "en". 7 | } 8 | } 9 | GROUP BY ?var1 ?var1Label 10 | ORDER BY DESC( ?var5 ) 11 | 12 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:361_18.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:383_361.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT( "1"^^ AS ?var1 ) 2 | WHERE { 3 | ?var2 . 4 | { 5 | ?var2 ( / *) . 6 | } 7 | UNION 8 | { 9 | ?var2 * . 10 | } 11 | } 12 | LIMIT 1 13 | 14 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:438_212.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | { ?var3 ?var2 ; 4 | ?var5 ; 5 | "string1"^^ . } 6 | OPTIONAL { ?var3 ?var4 . } } 7 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:448_13.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 2 | WHERE { 3 | ?var1 . 4 | ?var1 ?var2 . 5 | FILTER ( ( CONTAINS ( ?var2 , "string1" ) ) 6 | ) . 7 | OPTIONAL { 8 | ?var1 ?var3 . 9 | FILTER ( ( ( LANG ( ?var3 ) = "string2" ) ) 10 | ) . 11 | } 12 | FILTER ( ( !( BOUND ( ?var3 ) ) ) 13 | ) . 14 | } 15 | 16 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:474_86.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 ?var3 ?var4 2 | WHERE { 3 | ?var4 "string1". 4 | ?var3 ?var4 . 5 | ?var2 ?var3 ; 6 | * ; 7 | ?var1 . 8 | FILTER ( ( isLITERAL ( "string1" ) ) 9 | ) . 10 | } 11 | 12 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:479_0.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 ?var2Label 2 | WHERE { 3 | ?var1 . 4 | ?var1 ?var2 . 5 | SERVICE { 6 | "[AUTO_LANGUAGE], en". 7 | } 8 | } 9 | LIMIT 200 10 | OFFSET 128400 11 | 12 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:47_6975.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 2 | WHERE { 3 | ?var3 ?var2 ; 4 | ?var1 . 5 | } 6 | 7 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:498_0.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var1Label 2 | WHERE { 3 | ?var1 . 4 | SERVICE { 5 | "[AUTO_LANGUAGE],en". 6 | } 7 | } 8 | LIMIT 200 9 | OFFSET 39600 10 | 11 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:499_6736.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1Label ?var2 2 | WHERE { 3 | ?var2 . 4 | ?var2 ?var1Label . 5 | FILTER ( ( ( "string1" = LANG ( ?var1Label ) ) ) 6 | ) . 7 | } 8 | 9 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:509_16.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:517_9743.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 ?var2 . 4 | ?var3 ?var1 . 5 | VALUES ( ?var2 ) { 6 | ( ) 7 | } 8 | } 9 | 10 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:518_41.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1Label 2 | WHERE { 3 | ( / *) ; 4 | ?var1Label . 5 | FILTER ( ( ( "string1" = LANG ( ?var1Label ) ) ) 6 | ) . 7 | } 8 | 9 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:521_13.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:522_6.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 "string1". 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:562_0.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 . 4 | ?var1 . 5 | FILTER ( ( !( isLITERAL ( ) ) ) 6 | ) . 7 | } 8 | 9 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:566_220.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT * 2 | WHERE { 3 | ?var1 . 4 | OPTIONAL { 5 | ?var1 ?var2 . 6 | } 7 | OPTIONAL { 8 | ?var1 ?var3 . 9 | } 10 | OPTIONAL { 11 | ?var1 ?var4 . 12 | } 13 | OPTIONAL { 14 | ?var1 ?var5 . 15 | } 16 | OPTIONAL { 17 | ?var1 ?var6 . 18 | } 19 | OPTIONAL { 20 | ?var6 ?var7 . 21 | } 22 | SERVICE { 23 | "[AUTO_LANGUAGE],en". 24 | } 25 | FILTER ( ( ?var1 IN ( ) ) 26 | ) . 27 | } 28 | 29 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:609_6710.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX entity: 2 | PREFIX rdf: 3 | PREFIX hint: 4 | SELECT ?var1 ?var2Label ?var3 ?var4Label ?var5 5 | WHERE { 6 | "None". 7 | { 8 | BIND ( AS ?var3 ). 9 | BIND ( "string1" AS ?var1 ). 10 | BIND ( "string2"@ru AS ?var2Label ). 11 | } 12 | UNION 13 | { 14 | ?var1 ?var3 . 15 | ?var6 ?var7 ?var1 . 16 | ?var6 . 17 | ?var6 ?var2Label . 18 | } 19 | ?var3 ?var4Label . 20 | OPTIONAL { 21 | ?var3 ?var5 . 22 | } 23 | FILTER ( ( ( LANG ( ?var4Label ) = "string3" ) ) 24 | ) . 25 | FILTER ( ( ( LANG ( ?var2Label ) = "string3" ) ) 26 | ) . 27 | } 28 | LIMIT 1000 29 | 30 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:61_505.preBuildQueryType: -------------------------------------------------------------------------------- 1 | ASK 2 | WHERE { 3 | ( / *) . 4 | "string1". 5 | } 6 | 7 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:625_2507.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 ?var2Label 2 | WHERE { 3 | ?var1 . 4 | ?var1 ?var2 . 5 | SERVICE { 6 | "[AUTO_LANGUAGE], en". 7 | } 8 | } 9 | LIMIT 200 10 | OFFSET 32200 11 | 12 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:625_6703.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 "18343778". 4 | SERVICE { 5 | "[AUTO_LANGUAGE],en". 6 | } 7 | } 8 | LIMIT 1 9 | 10 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:632_1.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var2 . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:639_1.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 ?var2Label 2 | WHERE { 3 | ?var1 . 4 | ?var1 ?var2 . 5 | SERVICE { 6 | "[AUTO_LANGUAGE], en". 7 | } 8 | } 9 | ORDER BY ASC( ?var1 ) 10 | LIMIT 200 11 | OFFSET 111200 12 | 13 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:64_7.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1Label 2 | WHERE { 3 | SERVICE { 4 | "string1". 5 | ?var1Label . 6 | } 7 | } 8 | 9 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:650_45.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT( "1"^^ AS ?var1 ) 2 | WHERE { 3 | ?var2 . 4 | FILTER ( ( EXISTS { 5 | . 6 | } 7 | ) 8 | ) . 9 | } 10 | 11 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:655_9683.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:740_0.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var2 . 4 | OPTIONAL { 5 | ?var2 ?var1 . 6 | } 7 | OPTIONAL { 8 | ?var2 ?var3 . 9 | } 10 | FILTER ( ( REGEX ( STR ( ?var3 ) , "string1", "string2" ) ) 11 | ) . 12 | } 13 | 14 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:740_16.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 2 | WHERE { 3 | ?var1 ?var2 ; 4 | ?var3 . 5 | VALUES ( ?var2 ) { 6 | ( "string1" ) 7 | ( "string2" ) 8 | ( "string3" ) 9 | } 10 | } 11 | ORDER BY ASC( ?var3 ) 12 | 13 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:749_6696.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX schema: 2 | SELECT DISTINCT ?var1 ?var2 ?var3 ?var4 ?var5 ?var6 ( COUNT ( DISTINCT ?var7 ) AS ?var8 ) 3 | WHERE { 4 | ?var9 ?var1 . 5 | ?var9 "string1"@fr . 6 | ?var9 ?var6 . 7 | OPTIONAL { 8 | ?var9 ?var10 . 9 | } 10 | OPTIONAL { 11 | ?var9 ?var11 . 12 | } 13 | OPTIONAL { 14 | ?var9 ?var3 . 15 | } 16 | OPTIONAL { 17 | ?var9 ?var4 . 18 | } 19 | OPTIONAL { 20 | ?var9 ?var5 . 21 | } 22 | ?var7 ?var9 . 23 | OPTIONAL { 24 | ?var11 ?var2 . 25 | FILTER ( ( ( LANG ( ?var2 ) = "string2" ) ) 26 | ) . 27 | } 28 | FILTER ( ( ( LANG ( ?var1 ) = "string3" ) ) 29 | ) . 30 | } 31 | GROUP BY ?var2 ?var1 ?var3 ?var4 ?var5 ?var6 32 | ORDER BY ASC( ?var4 ) 33 | 34 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:754_15.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX gas: 2 | SELECT ?var1 ( ( ?var2 + ?var3 ) AS ?var4 ) 3 | WHERE { 4 | SERVICE { 5 | "com.bigdata.rdf.graph.analytics.SSSP"; 6 | ; 7 | "string1"; 8 | ?var1 ; 9 | ?var2 ; 10 | "10"^^ ; 11 | . 12 | } 13 | SERVICE { 14 | "com.bigdata.rdf.graph.analytics.SSSP"; 15 | ; 16 | "string1"; 17 | ?var1 ; 18 | ?var3 ; 19 | "10"^^ ; 20 | . 21 | } 22 | } 23 | ORDER BY ASC( ?var4 ) 24 | LIMIT 1 25 | 26 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:764_189.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 2 | WHERE { 3 | ?var2 ?var1 . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:768_1226.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 ?var3 2 | WHERE { 3 | ?var3 "string1". 4 | ?var2 ?var3 . 5 | ?var1 ?var2 ; 6 | * ; 7 | ?var4 . 8 | FILTER ( ( isLITERAL ( "string1" ) ) 9 | ) . 10 | } 11 | 12 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:768_54.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?var1 ?var1Label ?var2 ?var2Label ?var3 ?var4 2 | WHERE { 3 | ?var2 ( / *) . 4 | ?var1 . 5 | ?var1 ?var5 ?var2 . 6 | ?var1 ?var3 . 7 | SERVICE { 8 | ?var2 ?var6 . 9 | "string1"^^ . 10 | "200". 11 | ?var7 . 12 | } 13 | SERVICE { 14 | "en". 15 | } 16 | OPTIONAL { 17 | ?var2 ?var4 . 18 | } 19 | } 20 | ORDER BY ASC( ?var7 ) 21 | LIMIT 1 22 | 23 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:76_108.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 "string1". 4 | } 5 | LIMIT 1 6 | 7 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:770_244.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 . 4 | { 5 | ?var1 ( / *) . 6 | } 7 | UNION 8 | { 9 | ?var1 * . 10 | } 11 | SERVICE { 12 | "id". 13 | } 14 | } 15 | 16 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:773_6824.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var2 "string1"; 4 | ?var1 . 5 | FILTER ( ( isLITERAL ( "string1" ) ) 6 | ) . 7 | } 8 | 9 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:774_20.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var2 ?var1 . 4 | VALUES ( ?var2 ) { 5 | ( ) 6 | } 7 | } 8 | 9 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:788_0.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var2 . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:798_291.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 . 4 | { 5 | ?var1 ( / *) . 6 | } 7 | UNION 8 | { 9 | ?var1 * . 10 | } 11 | } 12 | 13 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:828_152.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT( SAMPLE ( ?var1 ) AS ?var2 ) 2 | WHERE { 3 | ?var3 ?var4 . 4 | FILTER ( ( ( ?var4 = ) ) 5 | ) . 6 | OPTIONAL { 7 | ?var4 ?var1 . 8 | } 9 | } 10 | 11 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:882_183.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?var1 2 | WHERE { 3 | ?var1 "string1"@fr . 4 | ?var1 . 5 | } 6 | 7 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:896_183.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?var1Label 2 | WHERE { 3 | ?var2 . 4 | ?var2 ?var1 . 5 | ?var1 . 6 | SERVICE { 7 | "nl". 8 | } 9 | } 10 | 11 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:898_189.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | { ?var1 ; 4 | ?var2 ; 5 | ?var3 . } 6 | OPTIONAL { ?var4 . } } 7 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:911_37.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:921_9879.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 ( / ) . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:934_15.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var1Label ?var2 ?var2Label 2 | WHERE { 3 | ?var1 ?var3 . 4 | ?var2 ?var3 . 5 | SERVICE { 6 | "en". 7 | } 8 | } 9 | LIMIT 100 10 | 11 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:935_0.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 . 4 | ?var1 . 5 | } 6 | 7 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:936_1150.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 2 | WHERE { 3 | ?var2 "string1". 4 | ?var1 ?var2 . 5 | FILTER ( ( isLITERAL ( "string1" ) ) 6 | ) . 7 | } 8 | 9 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/qt:974_188.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 ?var2 2 | WHERE { 3 | ?var2 ?var1 . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/readme.md: -------------------------------------------------------------------------------- 1 | - name.preBuildQueryType-files in this folder will be imported as queryTypes and name will be used as the query type 2 | - queries in .preBuildQueryType-files do not need to be normalized, they will be normalized during loading 3 | - the file toolMapping.tsv can be filled with mappings of the following form: 4 | queryType userAgent toolName toolVersion (comment) (using tab as a delimiter, ignoring the first row) 5 | 6 | NOTE: Since the OpenRDF-Renderer misses out on a lot of features it is recommended to supply an example query for query type rather than using the data from the .queryType-files. 7 | 8 | NOTES ON QUERY TYPES: 9 | user agents that are probably too broad: 10 | - Apache-Jena-ARQ 11 | - Java 12 | - [sparqlwrapper](http://rdflib.github.io/sparqlwrapper/) 13 | - Ruby 14 | 15 | NOTES ON POSSIBLY SEMI-AUTOMATED SOURCES: 16 | - wikidataLastModified could be semi-automated (lots of different browser/os combinations, always the same query) 17 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/referenceAllPredicatesAndObjects.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?p ?o 2 | WHERE { 3 | ?p ?o 4 | } 5 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/sameAsThis.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT * 2 | WHERE { 3 | ?s 4 | } 5 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/schemaAboutEntityLabelImageCoordinatesAllOptional.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?label ?image ?coordinates ?wikilink 2 | where { 3 | OPTIONAL{ rdfs:label ?label . 4 | FILTER (lang(?label)=?lang) . 5 | values (?lang ?lang_) { ("en" 1) ("en" 2) ("de" 3) ("fr" 4) ("it" 5)} } 6 | OPTIONAL{ wdt:P18 ?image . } 7 | OPTIONAL{ wdt:P625 ?coordinates . } 8 | OPTIONAL{ ?wikilink a schema:Article ; schema:about ; 9 | schema:inLanguage "en" ; schema:isPartOf .} 10 | } 11 | order by ?lang_ 12 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/signatureThis.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?image 2 | WHERE { 3 | wdt:P109 ?image 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/subclassOfThis.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT * 2 | WHERE { 3 | ?c 4 | } 5 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/taxonomyInformations.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wd: 2 | PREFIX wdt: 3 | PREFIX wikibase: 4 | PREFIX p: 5 | PREFIX ps: 6 | PREFIX pq: 7 | PREFIX rdfs: 8 | PREFIX bd: 9 | PREFIX pr: 10 | PREFIX prov: 11 | 12 | SELECT ?item ?taxonName ?taxonRank ?parentName ?parentRank ?higherParent WHERE { 13 | BIND("Nocardioides sp. 11683779" AS ?taxonName) . 14 | BIND(wd:Q7432 AS ?taxonRank) . 15 | BIND(wd:Q10876 AS ?higherParent) . 16 | ?item wdt:P225 ?taxonName . 17 | ?item wdt:P105 ?taxonRank . 18 | ?item wdt:P171/wdt:P225 ?parentName . 19 | ?item wdt:P171/wdt:P105 ?parentRank . 20 | ?item (wdt:P171)* ?higherParent . 21 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisAFI_Catalog_of_Feature_Films_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3593 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisAllMovie_movie_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P1562 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisAllcinema_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2465 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisAlloCine_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P1265 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisAnimator.ru_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P1934 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisAnime_News_Network_anime_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P1985 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisArticleAboutInLanguageIsPartOfWikiGroupWithLimit.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wikibase: 2 | PREFIX wd: 3 | PREFIX schema: 4 | select ?linkWikiversity 5 | where { 6 | ?linkWikiversity a schema:Article; 7 | schema:about wd:Q14914970; 8 | schema:inLanguage "en"; 9 | schema:isPartOf ?wikiversity . 10 | ?wikiversity wikibase:wikiGroup "wikiversity" . 11 | } 12 | limit 1 -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisAustralianStatisticalGeography2011IDOptionalDescriptionWithLabel.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?q ?qLabel ?description 2 | WHERE { 3 | ?q wdt:P4014 "SSC40203" 4 | OPTIONAL { 5 | ?q schema:description ?description 6 | filter(lang(?description)="en") 7 | } 8 | SERVICE wikibase:label { bd:serviceParam wikibase:language "en" } 9 | } 10 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisBFIFilmAndTVIDID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 "string1". 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisBFI_work_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2703 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisBangla_Movie_Database_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3821 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisBasedOn.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wikibase: 2 | PREFIX wdt: 3 | PREFIX wd: 4 | PREFIX bd: 5 | 6 | SELECT ?propertyIdLabel 7 | WHERE { 8 | wd:Q27057144 wdt:P144 ?propertyId 9 | SERVICE wikibase:label 10 | { bd:serviceParam wikibase:language "en" } 11 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisBox_Office_Mojo_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P1237 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisCBSMunicipalityCodeThisWithLabel.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?label ?cbsMunicipalityCode ?municipality_label 2 | WHERE { 3 | ?OrgnBNanon_9_0 ?municipality_label ; 4 | ?cbsMunicipalityCode . 5 | FILTER ( "en" = lang( ?label) ) 6 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisCBSMunicipalityCodeWithLabel.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?label ?municipality_label 2 | WHERE { 3 | ?OrgnBNanon_24_30 ?municipality_label ; 4 | "0106" . 5 | FILTER ( "en" = lang( ?label) ) 6 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisCBSMunicipalityCodeWithLabelWithPopulation.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?label ?population ?cbsMunicipalityCode ?municipality_label 2 | WHERE { 3 | ?OrgnBNanon_8_0 ?municipality_label; 4 | ?cbsMunicipalityCode; 5 | ?population . 6 | FILTER ( "en" = lang( ?label) ) 7 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisCSFD_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2529 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisCiNetMag_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3128 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisCine-Ressources_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3203 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisCinema_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3933 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisCinema_of_Israel_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3445 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisCinemagiaFilmID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 "373909". 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisCinemathequeQuebecoiseWorkIdentifier.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 "11142". 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisCineplex_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3077 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisCollectionDepictsPositionHeld.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT (1 AS ?stubvar18) 2 | WHERE { 3 | { 4 | SELECT DISTINCT ?item 5 | WHERE { 6 | ?item ; 7 | ?monarque . 8 | ?monarque . 9 | SERVICE { "en" . } 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisComposer.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wikibase: 2 | PREFIX wdt: 3 | PREFIX wd: 4 | PREFIX bd: 5 | 6 | SELECT ?propertyIdLabel 7 | WHERE { 8 | wd:Q5968033 wdt:P86 ?propertyId 9 | SERVICE wikibase:label 10 | { bd:serviceParam wikibase:language "en" } 11 | } 12 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisDNF_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P1804 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisDOIThatWithFilter.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX v: 2 | PREFIX q: 3 | PREFIX ps: 4 | PREFIX pq: 5 | SELECT ?q { 6 | ?q wdt:P356 ?o FILTER (lcase(str(?o)) = lcase("10.1529/BIOPHYSJ.106.088203")) 7 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisDaumMovieID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 "71391". 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisDerivedFromStatedInPaper.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?entry 2 | WHERE { 3 | ?entry ?p ?statement . 4 | ?statement prov:wasDerivedFrom//wdt:P356 '10.1093/bfgp/elp050' . 5 | } 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisDeutsche_Synchronkartei_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3844 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisDoubanMovieID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 "1304998". 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisEDb_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3141 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisEIDR_identifier.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2704 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisEditionOrTranslationOf.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?subject 2 | WHERE { 3 | ?subject wdt:P629 wd:Q1048916 . 4 | } 5 | LIMIT 1000 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisEldoblaje_Movie_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2897 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisElonet_movie_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2346 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisEntrezGeneID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wdt: 2 | SELECT * 3 | WHERE { 4 | ?gene wdt:P351 "3092928" 5 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisEntrezGeneIDFoundInTaxon.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?article 2 | WHERE { 3 | ?cid wdt:P351 '83942'. 4 | ?cid wdt:P703 wd:Q15978631 . 5 | OPTIONAL { 6 | ?cid rdfs:label ?label filter (lang(?label) = "en") . 7 | } 8 | ?article schema:about ?cid . 9 | ?article schema:inLanguage "en" . 10 | FILTER (SUBSTR(str(?article), 1, 25) = "https://en.wikipedia.org/") . 11 | FILTER (SUBSTR(str(?article), 1, 38) != "https://en.wikipedia.org/wiki/Template") 12 | } 13 | limit 1 -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisExactMatchWithFilter.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item . 4 | FILTER (!isLITERAL ( )) 5 | } 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisExplotationVisaNumber.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2755 "106385" 4 | } 5 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisFilmAffinity_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P480 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisFilmPolski.pl_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3495 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisFilmingLocation.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wikibase: 2 | PREFIX wdt: 3 | PREFIX wd: 4 | PREFIX bd: 5 | 6 | SELECT ?propertyIdLabel 7 | WHERE 8 | { wd:Q16917929 wdt:P915 ?propertyId 9 | SERVICE wikibase:label 10 | { bd:serviceParam wikibase:language "en" } 11 | } 12 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisFilmportal_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2639 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisFilmweb.pl_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3995 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisFreebaseIDWithCompanyTickerExchange.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?companyLabel ?rootLabel ?tickerLabel ?exchangeNameLabel 2 | WHERE { 3 | ?entity wdt:P646 "/m/05s8k" . 4 | ?entity wdt:P176* ?manufacturer . 5 | ?manufacturer wdt:P156* ?company . { 6 | ?company p:P414 ?exchange 7 | } UNION { 8 | ?company wdt:P127+ / wdt:P156* ?root . 9 | ?root p:P414 ?exchange 10 | } UNION { 11 | ?company wdt:P749+ / wdt:P156* ?root . 12 | ?root p:P414 ?exchange 13 | } . 14 | VALUES ?exchanges { 15 | wd:Q13677 wd:Q82059 16 | } . 17 | ?exchange ps:P414 ?exchanges . 18 | ?exchange pq:P249 ?ticker . 19 | ?exchange ps:P414 ?exchangeName . 20 | FILTER NOT EXISTS { 21 | ?company wdt:P31 / wdt:P279* wd:Q1616075 22 | } . 23 | FILTER NOT EXISTS { 24 | ?company wdt:P31 / wdt:P279* wd:Q11032 25 | } . 26 | SERVICE wikibase:label { 27 | bd:serviceParam wikibase:language "en" . 28 | } 29 | } GROUP BY ?companyLabel ?rootLabel ?tickerLabel ?exchangeNameLabel -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisFreebaseIDWithLabelOptionalInstanceOf.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item ?itemLabel ?instance_of WHERE { 2 | ?item wdt:P646 "/m/07d3x". 3 | SERVICE wikibase:label { bd:serviceParam wikibase:language "en". } 4 | OPTIONAL { ?item wdt:P31 ?instance_of. } 5 | } 6 | LIMIT 1 -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisFreebaseIDWithLabelOptionalInstanceOf2.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item ?itemLabel ?instance_of ?instance_ofLabel WHERE { 2 | ?item wdt:P646 "/m/07d3x". 3 | OPTIONAL { 4 | ?item wdt:P31 ?instance_of. 5 | } 6 | SERVICE wikibase:label { bd:serviceParam wikibase:language "en". } 7 | } 8 | LIMIT 1 -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisGECD_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3367 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisGND_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P227 "124909183" . 4 | } 5 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisGeneOntologyID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wdt: 2 | SELECT * 3 | WHERE {?gene wdt:P686 "GO:0044341"} 4 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisGenre.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX rdfs: 2 | PREFIX wikibase: 3 | PREFIX p: 4 | PREFIX wdt: 5 | PREFIX wd: 6 | PREFIX bd: 7 | 8 | SELECT ?genre_name 9 | WHERE 10 | { wd:Q936900 wdt:P136 ?genre 11 | SERVICE wikibase:label 12 | { bd:serviceParam wikibase:language "en" . 13 | ?genre rdfs:label ?genre_name 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisGenreWithEnglishLabel.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wd: 2 | PREFIX wdt: 3 | 4 | SELECT ?ID ?Genre WHERE { 5 | wd:Q664787 wdt:P136 ?ID. 6 | ?ID rdfs:label ?Genre filter (lang(?Genre) = "en") 7 | } 8 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisHKMDb_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2883 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisHasLabelLocatedInCoordinatesThat.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?loc 2 | WHERE { 3 | ?p "Trafalgar Square"@en; 4 | ; 5 | ?loc . 6 | } 7 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisHasLabel_cityOrRegion.preBuildQueryType: -------------------------------------------------------------------------------- 1 | select Distinct ?entity ?country ?coords ?image ?entityDescription ?type ?website ?article 2 | where { 3 | {{?entity skos:altLabel "Ryogoku"@en } UNION { ?entity rdfs:label "Ryogoku"@en }} . 4 | ?entity wdt:P31 ?type . 5 | {{ ?type wdt:P279* wd:Q515 } UNION {?type wdt:P279* wd:Q82794}} . 6 | ?entity wdt:P625 ?coords . 7 | ?entity wdt:P17 ?country . 8 | OPTIONAL { 9 | ?article schema:about ?entity . 10 | ?article schema:inLanguage "en" . 11 | ?article schema:isPartOf . 12 | } 13 | OPTIONAL { ?entity wdt:P18 ?image } . 14 | OPTIONAL { ?entity wdt:P856 ?website } . 15 | SERVICE wikibase:label { 16 | bd:serviceParam wikibase:language "en" . 17 | } 18 | } LIMIT 200 -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisHasLabel_existingCountries.preBuildQueryType: -------------------------------------------------------------------------------- 1 | select DISTINCT ?country ?countryLabel ?coords ?countryDescription ?article 2 | where { 3 | ?type wdt:P279* wd:Q6256 . 4 | ?country wdt:P31 ?type . 5 | MINUS { ?country wdt:P31 wd:Q3024240 } . 6 | {?country skos:altLabel "Eger"@en} UNION { ?country rdfs:label "Eger"@en } . 7 | ?country wdt:P625 ?coords . 8 | OPTIONAL { 9 | ?article schema:about ?country . 10 | ?article schema:inLanguage "en" . 11 | ?article schema:isPartOf . 12 | } 13 | SERVICE wikibase:label { 14 | bd:serviceParam wikibase:language "en" . 15 | } 16 | } LIMIT 50 -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisHasLabel_geographicalObjects.preBuildQueryType: -------------------------------------------------------------------------------- 1 | select Distinct ?entity ?country ?coords ?image ?entityDescription ?type ?website ?article where { 2 | { {?entity skos:altLabel "Mangart"@en } UNION { ?entity rdfs:label "Mangart"@en }} . 3 | ?entity wdt:P31 ?type . 4 | ?type wdt:P279* wd:Q618123 . 5 | ?entity wdt:P625 ?coords . 6 | ?entity wdt:P17 ?country . 7 | OPTIONAL { 8 | ?article schema:about ?entity . 9 | ?article schema:inLanguage "en" . 10 | ?article schema:isPartOf . 11 | } 12 | OPTIONAL { ?entity wdt:P18 ?image } . 13 | OPTIONAL { ?entity wdt:P856 ?website } . 14 | SERVICE wikibase:label { 15 | bd:serviceParam wikibase:language "en" . 16 | } 17 | } LIMIT 200 18 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisIMDb_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P345 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisISAN.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3212 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisISSNThatWithFilter.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX v: 2 | PREFIX q: 3 | PREFIX ps: 4 | PREFIX pq: 5 | SELECT ?q { 6 | ?q wdt:P236 ?o FILTER (lcase(str(?o)) = lcase("0733-5210")) 7 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisISzDb_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3110 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisInstanceOf.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisInstanceOfGenderDescription.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P31 wd:Q5 . #instanceof 4 | ?item wdt:P21 wd:Q6581097 . #gender 5 | ?item schema:description "Honduran choreographer"@en. #description 6 | OPTIONAL { ?item schema:description ?itemDescription. FILTER(LANG(?itemDescription) = "gl"). } 7 | FILTER (!BOUND(?itemDescription)) 8 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisInstanceOfWithSubclass.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item ?itemLabel ?linkTo { 2 | wd:Q1668643 wdt:P31 ?item 3 | OPTIONAL { ?item wdt:P279 ?linkTo } 4 | SERVICE wikibase:label {bd:serviceParam wikibase:language "en" } 5 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisIshim_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3906 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisJMDb_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2400 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisKINENOTE_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2508 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisKMDb_documentary_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3750 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisKMDb_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3704 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisKinopoisk_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2603 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisKinopolis_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2970 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisKvikmyndir_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3340 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisLUMIEREFilmID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 "21052". 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisLdiF_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3107 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisLocatedInTheAdministrativeTerritorialEntityArbitraryLengthInstanceOfArbitraryLengthAdministrativeTerritoralEntity.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P131 ?sub0 . 4 | ?sub0 (wdt:P131)* wd:Q2090 . 5 | ?item wdt:P31 ?sub1 . 6 | ?sub1 (wdt:P279)* wd:Q56061 . 7 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisLocatedInTheAdministrativeTerritorialEntityWithCoordinateLocation.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?item ?itemLabel ?coor ?range WHERE { 2 | ?item wdt:P131 wd:Q1617681. 3 | ?item ?range wd:Q1617681. 4 | ?item wdt:P625 ?coor. 5 | SERVICE wikibase:label { bd:serviceParam wikibase:language "fr". } 6 | } 7 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisMegogo_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2826 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisMetacritic_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P1712 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisMinkultury_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2636 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisMovieMeter_Movie_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P1970 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisMovie_Walker_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2509 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisMultipleCatalogProperties.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item ?picture 2 | WHERE { 3 | { 4 | SELECT DISTINCT ?item ?numero (SAMPLE( ?pic) AS ?picture) 5 | WHERE { 6 | ?item ?catalogStatement . 7 | ?catalogStatement ?numero; 8 | . 9 | OPTIONAL { 10 | ?item ?pic 11 | } 12 | SERVICE { "en" . } } 13 | GROUP BY ?item ?numero 14 | ORDER BY ASC ( ?numero) 15 | } 16 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisMyAnimeList_anime_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P4086 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisMyDramaList_title_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3868 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisNCBILocusTagFoundInTaxonNCBITaxonomyIDWithEncodes.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT distinct ?gene ?protein 2 | WHERE{ 3 | ?strain wdt:P685 '272561'. 4 | ?gene wdt:P2393 'CT_003'; 5 | wdt:P703 ?strain; 6 | wdt:P688 ?protein. 7 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisNCBITaxonomyIDSubclassOfOperon.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wdt: 2 | PREFIX wd: 3 | PREFIX qualifier: 4 | SELECT ?uniqueID ?description ?strand (MIN(?gstart) AS ?start) (MAX(?gend) AS ?end) ?uri 5 | WHERE { 6 | ?strain wdt:P685 '295405'. 7 | ?operon wdt:P279 wd:Q139677; 8 | wdt:P703 ?strain; 9 | rdfs:label ?description; 10 | wdt:P2548 ?wdstrand; 11 | wdt:P527 ?genes. 12 | ?genes wdt:P644 ?gstart; 13 | wdt:P645 ?gend. 14 | bind( IF(?wdstrand = wd:Q22809680, '1', '-1') as ?strand). 15 | bind(str(?operon) as ?uri) 16 | bind( strafter( str(?operon), "entity/" ) as ?uniqueID ). 17 | } 18 | GROUP BY ?uniqueID ?description ?strand ?uri ?prefix -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisNKCRAUTIDAboutAndLanguage.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?entityLabel ?wikiLink ?linkLanguage 2 | WHERE 3 | { 4 | ?entity wdt:P691 "ph119753" . 5 | 6 | ?wikiLink a schema:Article ; 7 | schema:about ?entity ; 8 | schema:inLanguage ?linkLanguage . 9 | 10 | SERVICE wikibase:label { 11 | bd:serviceParam wikibase:language "cs" . 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisNRHPReferenceNumber.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P649 "76002248" . 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisNamedAfter.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P138 wd:Q676555 . 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisNetflix_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P1874 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisNorsk_filmografi_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P1439 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisOFDb_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3138 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisOpen_Media_Database_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3302 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisPMCID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX v: 2 | PREFIX q: 3 | PREFIX ps: 4 | PREFIX pq: 5 | SELECT ?q { 6 | ?q wdt:P932 "2174230" 7 | } 8 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisPMCIDthatWithFilter.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX v: 2 | PREFIX q: 3 | PREFIX ps: 4 | PREFIX pq: 5 | SELECT ?q { 6 | ?q wdt:P932 ?o FILTER (lcase(str(?o)) = lcase("3379633")) 7 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisPORT_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P905 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisPrismaID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 "542208". 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisPubMedID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX v: 2 | PREFIX q: 3 | PREFIX ps: 4 | PREFIX pq: 5 | SELECT ?q { ?q wdt:P698 "12771101" } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisPubMedIDthatWithFilter.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX v: 2 | PREFIX q: 3 | PREFIX ps: 4 | PREFIX pq: 5 | SELECT ?q { 6 | ?q wdt:P698 ?o FILTER (lcase(str(?o)) = lcase("27500320")) 7 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisRotten_Tomatoes_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P1258 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisRussiancinema.ru_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2678 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisScope.dk_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2518 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisSomePropertyStringWithMultipleOtherProperties.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item ?itemLabel ?_image ?prenom ?dob ?dod ?yob ?yod ?link 2 | WHERE 3 | { 4 | ?item ?link "David Tennant"@fr. 5 | ?item wdt:P735 ?prenom. 6 | ?item wdt:P569 ?dob. 7 | ?item wdt:P570 ?dod. 8 | BIND(YEAR(?dob) as ?yob) . 9 | BIND(YEAR(?dod) as ?yod) . 10 | SERVICE wikibase:label { bd:serviceParam wikibase:language "fr". } 11 | OPTIONAL { ?item wdt:P18 ?_image. } 12 | } 13 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisSourehCinema_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3139 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisSratim_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3145 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisStreamIDOptionalArticeInLanguage.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item ?itemLabel ?itemAltLabel ?itemDescription ?wpzh_page_title 2 | WHERE { 3 | ?item wdt:P1733 "344410". 4 | SERVICE wikibase:label { bd:serviceParam wikibase:language "zh". } 5 | OPTIONAL { 6 | ?article schema:about ?item . 7 | ?article schema:inLanguage "zh" . 8 | ?article schema:about ?item; schema:isPartOf ; schema:name ?wpzh_page_title . 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisSubclassOfWithLabelOrderByLabel.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?s 2 | WHERE { 3 | ?s wdt:P279 wd:Q26944999 . 4 | SERVICE wikibase:label { bd:serviceParam wikibase:language "en" } 5 | } 6 | ORDER BY ?sLabel -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisSwedish_Film_Database_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2334 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisTV.Com_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2638 "string" 4 | } 5 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisTV_Guide_Show_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3804 "string" 4 | } 5 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisTaxonNameTaxonRank.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wd: 2 | PREFIX wdt: 3 | PREFIX wikibase: 4 | PREFIX p: 5 | PREFIX ps: 6 | PREFIX pq: 7 | PREFIX rdfs: 8 | PREFIX bd: 9 | PREFIX pr: 10 | PREFIX prov: 11 | 12 | SELECT ?item ?taxonName ?taxonRank ?parentName ?parentRank ?higherParent 13 | WHERE { 14 | BIND("Proasellus coxalis lucifugus" AS ?taxonName) . 15 | BIND(wd:Q68947 AS ?taxonRank) . 16 | ?item wdt:P225 ?taxonName . 17 | ?item wdt:P105 ?taxonRank . 18 | } 19 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisTaxonRankParentTaxon.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item ?itemLabel 2 | WHERE { 3 | ?item wdt:P105 wd:Q7432 . 4 | ?item wdt:P171 wd:Q228283 . 5 | SERVICE wikibase:label { bd:serviceParam wikibase:language "en" } 6 | } 7 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisThe_Numbers_movie_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3808 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisTransfermarkManagerID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wdt: 2 | PREFIX wd: 3 | 4 | SELECT * 5 | WHERE { 6 | wd:Q15575352 wdt:P2447 ?req_id 7 | } 8 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisTransfermarktFootballerID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT * 2 | WHERE 3 | { wd:Q1085325 wdt:P2446 ?req_id } 4 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisTurner_Classic_Movies_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P2631 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisULANID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P245 "500095196" . 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisUniProtID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wdt: 2 | SELECT * 3 | WHERE { 4 | ?gene wdt:P352 "Q9KMJ6" 5 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisUniProtIDThatExactMatchFoundInTaxon.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?stubvar22 2 | WHERE { 3 | ?item ?uniprot ; 4 | ; 5 | . 6 | FILTER (!isLITERAL ( )) 7 | } 8 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisUniProtIDWithFilterIsLiteral.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?wikidata 2 | WHERE { 3 | ?wikidata "P04440" . 4 | FILTER (isLITERAL ("P04440" )) 5 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisUnifrance_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3961 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisVIAFIDBVMCpersonID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wdt: 2 | 3 | SELECT * 4 | WHERE 5 | { { ?agent wdt:P214 "" .} 6 | UNION 7 | { ?agent wdt:P2799 "11893" .} 8 | OPTIONAL 9 | { ?agent wdt:P20 ?deathplace .} 10 | OPTIONAL 11 | { ?agent wdt:P19 ?birthplace .} 12 | OPTIONAL 13 | { ?agent wdt:P103 ?language .} 14 | OPTIONAL 15 | { ?agent wdt:P18 ?image .} 16 | OPTIONAL 17 | { ?agent wdt:P27 ?nationality .} 18 | } 19 | LIMIT 100 20 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisVIAFIDWithLabel.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?author ?label 2 | WHERE { 3 | ?author wdt:P214 '90305895' . 4 | SERVICE wikibase:label { bd:serviceParam wikibase:language 'it' } 5 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisWikiLovesMonumentsID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?s 2 | WHERE { 3 | ?s wdt:P2186 "0630583456" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thiscine.gr_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3129 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thiscinenacional.com_movie_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3851 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisdanskefilm_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3785 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisdanskefilm_silent_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3787 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thiselCinema_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3135 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thiselFilm_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3143 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/thisfilm-documentaire.fr_film_ID.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?item 2 | WHERE { 3 | ?item wdt:P3673 "string" 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/toolCheckingInstanceOfInEnglishUsingScalaj-http.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?x ?y WHERE{ wd:Q15265463 wdt:P31 ?x. SERVICE wikibase:label { bd:serviceParam wikibase:language "en". ?x rdfs:label ?y. }} -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/toolListingAllClubsAndTheRespectiveYearsAFootballPlayerWasInATeam.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wikibase: 2 | PREFIX p: 3 | PREFIX wd: 4 | PREFIX pq: 5 | PREFIX bd: 6 | PREFIX ps: 7 | 8 | SELECT (year(?start_time) AS ?start_year) (year(?end_time) AS ?end_year) ?team ?teamLabel 9 | WHERE 10 | { wd:Q11948 p:P54 ?team_prop . 11 | ?team_prop ps:P54 ?team . 12 | ?team_prop pq:P580 ?start_time 13 | OPTIONAL 14 | { ?team_prop pq:P582 ?end_time } 15 | SERVICE wikibase:label 16 | { bd:serviceParam wikibase:language "en" } 17 | } 18 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/toolLookingForEnglishNameOfHistoricalPerson.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX rdfs: 2 | PREFIX wd: 3 | PREFIX wdt: 4 | 5 | SELECT DISTINCT ?Enlabel 6 | WHERE 7 | { wd:Q52936 rdfs:label ?Enlabel 8 | FILTER langMatches(lang(?Enlabel), "en") 9 | } 10 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/toolLookingForFootballPlayerBirths.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/toolLookingForFootballPlayerDeaths.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT ?var1 2 | WHERE { 3 | ?var1 . 4 | } 5 | 6 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/toolLookingForFootballPlayerGender.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wikibase: 2 | PREFIX wdt: 3 | PREFIX wd: 4 | PREFIX bd: 5 | 6 | SELECT ?propertyIdLabel 7 | WHERE 8 | { wd:Q11948 wdt:P21 ?propertyId 9 | SERVICE wikibase:label 10 | { bd:serviceParam wikibase:language "en" } 11 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/toolLookingForFottballPlayerEnglishWikipediaPage.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX schema: 2 | PREFIX wd: 3 | 4 | SELECT * 5 | WHERE 6 | { OPTIONAL 7 | { ?wikipediaLink schema:about wd:Q11948 . 8 | ?wikipediaLink schema:inLanguage "en" . 9 | ?wikipediaLink schema:isPartOf 10 | } 11 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/toolLookingForGeoInformation.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX ff: 2 | PREFIX bd: 3 | PREFIX owl: 4 | PREFIX wdt: 5 | PREFIX wikibase: 6 | PREFIX skos: 7 | PREFIX rdfs: 8 | PREFIX ogc: 9 | PREFIX dbpedia: 10 | PREFIX lexinfo: 11 | PREFIX geom: 12 | PREFIX wd: 13 | PREFIX dbo: 14 | PREFIX lemon: 15 | PREFIX lgdo: 16 | PREFIX rdf: 17 | PREFIX geo-pos: 18 | PREFIX lgdr: 19 | PREFIX omgeo: 20 | PREFIX foaf: 21 | PREFIX bn-lemon: 22 | PREFIX om: 23 | 24 | SELECT DISTINCT ?class ?classLabel ?ind ?indLabel 25 | WHERE 26 | { ?ind rdfs:label "Aachener Straße"@de . 27 | ?ind wdt:P31 ?class . 28 | ?class (wdt:P279)* wd:Q34442 29 | SERVICE wikibase:label 30 | { bd:serviceParam wikibase:language "de" } 31 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/toolLookingForGeoInformation2.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX ff: 2 | PREFIX bd: 3 | PREFIX owl: 4 | PREFIX wdt: 5 | PREFIX wikibase: 6 | PREFIX skos: 7 | PREFIX rdfs: 8 | PREFIX ogc: 9 | PREFIX dbpedia: 10 | PREFIX lexinfo: 11 | PREFIX geom: 12 | PREFIX wd: 13 | PREFIX dbo: 14 | PREFIX lemon: 15 | PREFIX lgdo: 16 | PREFIX rdf: 17 | PREFIX geo-pos: 18 | PREFIX lgdr: 19 | PREFIX omgeo: 20 | PREFIX foaf: 21 | PREFIX bn-lemon: 22 | PREFIX om: 23 | 24 | SELECT DISTINCT ?labelLabel 25 | WHERE 26 | { wd:Q1413235 rdfs:label ?label 27 | SERVICE wikibase:label 28 | { bd:serviceParam wikibase:language "en" } 29 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/toolLookingForGeoInformationClasses.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX ff: 2 | PREFIX bd: 3 | PREFIX owl: 4 | PREFIX wdt: 5 | PREFIX wikibase: 6 | PREFIX skos: 7 | PREFIX rdfs: 8 | PREFIX ogc: 9 | PREFIX dbpedia: 10 | PREFIX lexinfo: 11 | PREFIX geom: 12 | PREFIX wd: 13 | PREFIX dbo: 14 | PREFIX lemon: 15 | PREFIX lgdo: 16 | PREFIX rdf: 17 | PREFIX geo-pos: 18 | PREFIX lgdr: 19 | PREFIX omgeo: 20 | PREFIX foaf: 21 | PREFIX bn-lemon: 22 | PREFIX om: 23 | 24 | SELECT DISTINCT ?class 25 | WHERE 26 | { rdf:type ?class } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/toolLookingForNNDBIdForAllFootballPlayers.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wdt: 2 | PREFIX wd: 3 | 4 | SELECT * 5 | WHERE 6 | { wd:Q11948 wdt:P1263 ?req_id } 7 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/toolLookingForWikidataIdBasedOnTwitterUsername.preBuildQueryType: -------------------------------------------------------------------------------- 1 | 2 | PREFIX wikibase: 3 | PREFIX wd: 4 | PREFIX wdt: 5 | PREFIX rdfs: 6 | PREFIX p: 7 | PREFIX v: 8 | SELECT ?item ?itemLabel WHERE { 9 | ?item wdt:P2002 'ma_kr' . 10 | SERVICE wikibase:label { 11 | bd:serviceParam wikibase:language "en" . 12 | } 13 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/totalRevenueWithTotalRevenuePointInTimeLabel.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT * WHERE { 2 | wd:Q18355281 p:P2139 ?revenueStatement. 3 | ?revenueStatement ps:P2139 ?statementValue. 4 | ?revenueStatement pq:P585 ?statementQualifier. 5 | SERVICE wikibase:label { bd:serviceParam wikibase:language "en" } 6 | } 7 | ORDER BY DESC (?statementQualifier) LIMIT 1 -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/transfermarktPlayerIDtransfermarktManagerIDthis.preBuildQueryType: -------------------------------------------------------------------------------- 1 | PREFIX wdt: 2 | PREFIX wd: 3 | 4 | SELECT * 5 | WHERE { 6 | {wd:Q6800443 wdt:P2446 ?P2446} 7 | UNION 8 | {wd:Q6800443 wdt:P2447 ?P2447} 9 | } 10 | -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/transitiveSubclassOfThis.preBuildQueryType: -------------------------------------------------------------------------------- 1 | SELECT * 2 | WHERE { 3 | ()* ?c 4 | } -------------------------------------------------------------------------------- /preBuildQueryTypeFiles/wikidataLastModified.preBuildQueryType: -------------------------------------------------------------------------------- 1 | prefix schema: 2 | SELECT * 3 | WHERE { 4 | schema:dateModified ?y 5 | } 6 | -------------------------------------------------------------------------------- /src/main/java/anonymize/Test.java: -------------------------------------------------------------------------------- 1 | package anonymize; 2 | 3 | import general.Main; 4 | import openrdffork.RenderVisitor; 5 | import openrdffork.StandardizingPrefixDeclProcessor; 6 | import openrdffork.StandardizingSPARQLParser; 7 | import org.openrdf.query.MalformedQueryException; 8 | import org.openrdf.query.parser.ParsedQuery; 9 | import org.openrdf.query.parser.sparql.BaseDeclProcessor; 10 | import org.openrdf.query.parser.sparql.StringEscapesProcessor; 11 | import org.openrdf.query.parser.sparql.ast.*; 12 | import query.OpenRDFQueryHandler; 13 | 14 | import java.io.IOException; 15 | import java.nio.file.DirectoryStream; 16 | import java.nio.file.Files; 17 | import java.nio.file.Path; 18 | import java.nio.file.Paths; 19 | 20 | import static java.nio.file.Files.readAllBytes; 21 | 22 | public class Test 23 | { 24 | 25 | public static void main(String[] args) 26 | { 27 | Main.loadStandardPrefixes(); 28 | 29 | Anonymizer.loadWhitelistDatatypes(); 30 | 31 | int worked = 0; 32 | int failed = 0; 33 | int failedToParse = 0; 34 | 35 | try (DirectoryStream directoryStream = 36 | Files.newDirectoryStream(Paths.get("/home/adrian/workspace/java/months/exampleQueries/"))) { 37 | for (Path filePath : directoryStream) { 38 | if (Files.isRegularFile(filePath)) { 39 | String queryString = new String(readAllBytes(filePath)); 40 | 41 | try { 42 | ParsedQuery parsedQuery = new StandardizingSPARQLParser().parseQuery(queryString, OpenRDFQueryHandler.BASE_URI); 43 | } catch (MalformedQueryException e) { 44 | failedToParse++; 45 | continue; 46 | } 47 | 48 | ASTQueryContainer qc; 49 | try { 50 | qc = SyntaxTreeBuilder.parseQuery(queryString); 51 | } catch (TokenMgrError | ParseException e) { 52 | //e.printStackTrace(); 53 | continue; 54 | } 55 | 56 | try { 57 | StandardizingSPARQLParser.debug(qc); 58 | StringEscapesProcessor.process(qc); 59 | BaseDeclProcessor.process(qc, OpenRDFQueryHandler.BASE_URI); 60 | StandardizingPrefixDeclProcessor.process(qc); 61 | StandardizingSPARQLParser.anonymize(qc); 62 | } catch (MalformedQueryException e) { 63 | System.out.println("Failed to debug or anonymize query. " + queryString); 64 | } 65 | String renderedQueryString; 66 | try { 67 | renderedQueryString = qc.jjtAccept(new RenderVisitor(), "").toString(); 68 | //System.out.println(renderedQueryString); 69 | } catch (VisitorException e) { 70 | //e.printStackTrace(); 71 | continue; 72 | } 73 | 74 | try { 75 | ParsedQuery parsedQuery = new StandardizingSPARQLParser().parseQuery(renderedQueryString, OpenRDFQueryHandler.BASE_URI); 76 | worked++; 77 | } catch (MalformedQueryException | ClassCastException e) { 78 | failed++; 79 | System.out.println("-----------------------------------"); 80 | System.out.println(filePath); 81 | System.out.println(queryString); 82 | continue; 83 | } 84 | 85 | } 86 | } 87 | } catch (IOException e) { 88 | e.printStackTrace(); 89 | } 90 | 91 | System.out.println("Worked: " + worked + " Failed: " + failed + " Failed to Parse: " + failedToParse); 92 | 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /src/main/java/anonymize/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * @author adrian 3 | */ 4 | package anonymize; 5 | -------------------------------------------------------------------------------- /src/main/java/general/ParseOneDayWorker.java: -------------------------------------------------------------------------------- 1 | package general; 2 | 3 | 4 | import input.InputHandler; 5 | import input.factories.InputHandlerFactory; 6 | import openrdffork.TupleExprWrapper; 7 | import org.apache.log4j.Logger; 8 | import output.OutputHandler; 9 | import output.factories.OutputHandlerFactory; 10 | import query.factories.QueryHandlerFactory; 11 | 12 | import java.io.FileNotFoundException; 13 | import java.io.IOException; 14 | import java.util.HashMap; 15 | import java.util.Map; 16 | 17 | /** 18 | * @author Julius Gonsior 19 | */ 20 | public class ParseOneDayWorker implements Runnable 21 | { 22 | /** 23 | * Define a static logger variable. 24 | */ 25 | private static final Logger logger = Logger.getLogger(ParseOneDayWorker.class); 26 | /** 27 | * The inputHandler to process from. 28 | */ 29 | private InputHandler inputHandler; 30 | /** 31 | * The output handler to process to. 32 | */ 33 | private OutputHandler outputHandler; 34 | /** 35 | * The day this worker is processing. 36 | */ 37 | private int day; 38 | 39 | /** 40 | * @param inputHandlerFactory The input handler factory to supply the input handler. 41 | * @param inputFile The file to read from. 42 | * @param outputHandlerFactory The output handler factory to supply the output handler. 43 | * @param outputFile The file to write to. 44 | * @param queryHandlerFactory The query handler factory to supply the query handler. 45 | * @param dayToSet The day being processed. 46 | * @param writeQueryTypes If the query types should be written or not. 47 | * @throws IOException If the input or output handler could not be created. 48 | */ 49 | public ParseOneDayWorker(InputHandlerFactory inputHandlerFactory, String inputFile, OutputHandlerFactory outputHandlerFactory, String outputFile, QueryHandlerFactory queryHandlerFactory, int dayToSet, Boolean writeQueryTypes) throws IOException 50 | { 51 | try { 52 | this.inputHandler = inputHandlerFactory.getInputHandler(inputFile); 53 | } catch (IOException e) { 54 | logger.warn("File " + inputFile + " could not be found."); 55 | throw e; 56 | } 57 | try { 58 | this.outputHandler = outputHandlerFactory.getOutputHandler(outputFile, queryHandlerFactory); 59 | } catch (FileNotFoundException e) { 60 | logger.error("File " + outputFile + "could not be created or written to.", e); 61 | throw e; 62 | } 63 | this.day = dayToSet; 64 | } 65 | 66 | 67 | @Override 68 | public void run() 69 | { 70 | logger.info("Start processing " + inputHandler.getInputFile()); 71 | outputHandler.setThreadNumber(day); 72 | try { 73 | inputHandler.parseTo(outputHandler, day); 74 | logger.info("Done processing " + inputHandler.getInputFile() + " to " + outputHandler.getOutputFile() + "."); 75 | } 76 | catch (IOException e) { 77 | logger.error("Error processing " + inputHandler.getInputFile() + " to " + outputHandler.getOutputFile() + ".", e); 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/general/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * @author adrian 3 | */ 4 | package general; 5 | -------------------------------------------------------------------------------- /src/main/java/input/SparkHadoopExperiment.java: -------------------------------------------------------------------------------- 1 | package input; 2 | 3 | import logging.LoggingHandler; 4 | import org.apache.log4j.Level; 5 | import org.apache.spark.SparkConf; 6 | import org.apache.spark.api.java.JavaSparkContext; 7 | 8 | /** 9 | * @author adrian 10 | */ 11 | public final class SparkHadoopExperiment 12 | { 13 | /** 14 | * Since this is a utility class, it should not be instantiated. 15 | */ 16 | private SparkHadoopExperiment() 17 | { 18 | throw new AssertionError("Instantiating utility class SparkHadoopExperiment"); 19 | } 20 | 21 | /** 22 | * @param args Arguments to specify runtime behavior. 23 | */ 24 | public static void main(String[] args) 25 | { 26 | LoggingHandler.initFileLog("logsExperimental/logging.%timestamp.log", Level.INFO); 27 | SparkConf conf = new SparkConf().setAppName("SPARQLQueryAnalyzer_Experimental").setMaster("local"); 28 | JavaSparkContext sc = new JavaSparkContext(conf); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/input/factories/InputHandlerFactory.java: -------------------------------------------------------------------------------- 1 | package input.factories; 2 | 3 | import input.InputHandler; 4 | 5 | import java.io.IOException; 6 | 7 | /** 8 | * @author adrian 9 | */ 10 | public interface InputHandlerFactory 11 | { 12 | /** 13 | * @param inputFile The input file to read from. 14 | * @return An input handler corresponding to this factory based on the parameters. 15 | * @throws IOException If the file specified in inputFile could not be found or read from. 16 | */ 17 | InputHandler getInputHandler(String inputFile) throws IOException; 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/input/factories/InputHandlerTSVFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package input.factories; 5 | 6 | import input.InputHandler; 7 | import input.InputHandlerTSV; 8 | 9 | import java.io.IOException; 10 | 11 | /** 12 | * @author adrian 13 | */ 14 | public class InputHandlerTSVFactory implements InputHandlerFactory 15 | { 16 | 17 | /** 18 | * @param inputFile The input file to read from. 19 | * @return An input handler tsv based on the parameters. 20 | * @throws IOException If the file specified in inputFile could not be found or read from. 21 | */ 22 | @Override 23 | public InputHandler getInputHandler(String inputFile) throws IOException 24 | { 25 | return new InputHandlerTSV(inputFile); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/input/factories/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * @author adrian 3 | */ 4 | package input.factories; 5 | -------------------------------------------------------------------------------- /src/main/java/input/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * @author adrian 3 | */ 4 | package input; 5 | -------------------------------------------------------------------------------- /src/main/java/logging/LoggingHandler.java: -------------------------------------------------------------------------------- 1 | package logging; 2 | 3 | import org.apache.log4j.ConsoleAppender; 4 | import org.apache.log4j.Level; 5 | import org.apache.log4j.Logger; 6 | import org.apache.log4j.PatternLayout; 7 | import org.apache.log4j.varia.LevelRangeFilter; 8 | 9 | /** 10 | * @author adrian 11 | */ 12 | public final class LoggingHandler 13 | { 14 | /** 15 | * Since this is a utility class, it should not be instantiated. 16 | */ 17 | private LoggingHandler() 18 | { 19 | throw new AssertionError("Instantiating utility class LoggingHandler"); 20 | } 21 | 22 | /** 23 | * Method for initiating a file log with default parameters. 24 | */ 25 | public static void initFileLog(String queryParserName, String inputFilePrefix) 26 | { 27 | initFileLog("logs/" + queryParserName + inputFilePrefix.substring(inputFilePrefix.lastIndexOf("/") + 1) + ".%timestamp.log", Level.ALL); 28 | } 29 | 30 | /** 31 | * Defines an appender that writes all log messages of level levelToWrite or higher to the file fileToWrite. 32 | * 33 | * @param fileToWrite File to write the log to. 34 | */ 35 | public static void initFileLog(String fileToWrite, Level levelToWrite) 36 | { 37 | TimestampFileAppender fileAppender = new TimestampFileAppender(); 38 | fileAppender.setName("FileLogger"); 39 | fileAppender.setLayout(new PatternLayout(" [%p] %d [%t] %x %c %M - %m%n")); 40 | fileAppender.setFile(fileToWrite); 41 | fileAppender.setThreshold(levelToWrite); 42 | fileAppender.activateOptions(); 43 | Logger.getRootLogger().addAppender(fileAppender); 44 | } 45 | 46 | /** 47 | * Defines an appender that writes INFO log messages to the console. 48 | */ 49 | public static void initConsoleLog() 50 | { 51 | ConsoleAppender consoleAppender = new ConsoleAppender(); 52 | consoleAppender.setName("ConsoleLogger"); 53 | consoleAppender.setLayout(new PatternLayout(" [%p] %d [%t] %x %c %M - %m%n")); 54 | LevelRangeFilter levelRangeFilter = new LevelRangeFilter(); 55 | levelRangeFilter.setLevelMax(Level.INFO); 56 | levelRangeFilter.setLevelMin(Level.INFO); 57 | consoleAppender.addFilter(levelRangeFilter); 58 | consoleAppender.activateOptions(); 59 | Logger.getRootLogger().addAppender(consoleAppender); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/logging/TimestampFileAppender.java: -------------------------------------------------------------------------------- 1 | package logging; 2 | 3 | import org.apache.log4j.FileAppender; 4 | 5 | import java.text.SimpleDateFormat; 6 | import java.util.Date; 7 | 8 | /** 9 | * @author adrian 10 | */ 11 | 12 | public class TimestampFileAppender extends FileAppender 13 | { 14 | 15 | @Override 16 | public final void setFile(String file) 17 | { 18 | Date d = new Date(); 19 | SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss"); 20 | file.replaceAll("%timestamp", ""); 21 | super.setFile(file.replaceAll("%timestamp", format.format(d))); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/logging/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * @author adrian 3 | */ 4 | package logging; 5 | -------------------------------------------------------------------------------- /src/main/java/openrdffork/IRIShorteningRenderVisitor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package openrdffork; 5 | 6 | import org.openrdf.query.parser.sparql.ast.ASTIRI; 7 | import org.openrdf.query.parser.sparql.ast.VisitorException; 8 | 9 | import query.QueryHandler; 10 | import scala.Tuple2; 11 | 12 | /** 13 | * @author adrian 14 | * 15 | */ 16 | public class IRIShorteningRenderVisitor extends RenderVisitor 17 | { 18 | 19 | @Override 20 | public final Object visit(ASTIRI node, Object data) throws VisitorException 21 | { 22 | String result = super.visit(node, data).toString(); 23 | Tuple2 shorteningResult = QueryHandler.replaceExplicitURI(node.getValue()); 24 | 25 | if (shorteningResult._1) { 26 | result = " " + shorteningResult._2 + " "; 27 | result += super.visitChildren(node, data.toString() + " "); 28 | } 29 | return result; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/openrdffork/TupleExprWrapper.java: -------------------------------------------------------------------------------- 1 | package openrdffork; 2 | 3 | import org.openrdf.query.algebra.TupleExpr; 4 | 5 | /** 6 | * @author adrian 7 | * A wrapper class as a workaround for the broken .hashCode()-function of the TupleExpr-class 8 | */ 9 | public class TupleExprWrapper 10 | { 11 | /** 12 | * The TupleExpr represented by this wrapper. 13 | */ 14 | private TupleExpr tupleExpr; 15 | /** 16 | * The hashCode of the represented TupleExpr for performance optimization. 17 | */ 18 | private int tupleExprHashCode; 19 | 20 | /** 21 | * @param tupleExprToSet {@link #tupleExpr} 22 | */ 23 | public TupleExprWrapper(TupleExpr tupleExprToSet) 24 | { 25 | this.setTupleExpr(tupleExprToSet); 26 | } 27 | 28 | private void computeHashCode() 29 | { 30 | if (this.tupleExpr != null) { 31 | this.tupleExprHashCode = getTupleExpr().toString().hashCode(); 32 | } else { 33 | this.tupleExprHashCode = 0; 34 | } 35 | } 36 | 37 | /** 38 | * @return {@link #tupleExpr} 39 | */ 40 | public TupleExpr getTupleExpr() 41 | { 42 | return this.tupleExpr; 43 | } 44 | 45 | /** 46 | * @param tupleExprToSet {@link #tupleExpr} 47 | */ 48 | public void setTupleExpr(TupleExpr tupleExprToSet) 49 | { 50 | this.tupleExpr = tupleExprToSet; 51 | computeHashCode(); 52 | } 53 | 54 | @Override 55 | public final String toString() 56 | { 57 | return this.tupleExpr.toString(); 58 | } 59 | 60 | @Override 61 | public final int hashCode() 62 | { 63 | return this.tupleExprHashCode; 64 | } 65 | 66 | @Override 67 | public final boolean equals(Object o) 68 | { 69 | if (this == o) return true; 70 | 71 | if (null == o) return false; 72 | 73 | if (getClass() != o.getClass()) return false; 74 | 75 | TupleExprWrapper tupleExprWrapper = (TupleExprWrapper) o; 76 | 77 | if (this.hashCode() != tupleExprWrapper.hashCode()) return false; 78 | 79 | return this.getTupleExpr().equals(tupleExprWrapper.getTupleExpr()); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/openrdffork/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * @author adrian 3 | */ 4 | package openrdffork; 5 | -------------------------------------------------------------------------------- /src/main/java/output/factories/OutputHandlerAnonymizerFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package output.factories; 5 | 6 | import output.OutputHandler; 7 | import output.OutputHandlerAnonymizer; 8 | import query.factories.QueryHandlerFactory; 9 | 10 | import java.io.FileNotFoundException; 11 | import java.io.IOException; 12 | 13 | /** 14 | * @author adrian 15 | */ 16 | public class OutputHandlerAnonymizerFactory implements OutputHandlerFactory 17 | { 18 | /** 19 | * @param outputFile The file the output should be written to. 20 | * @param queryHandlerFactory The factory supplying the query handler to generate the output with. 21 | * @return An output handler anonymizer based on the parameters. 22 | * @throws IOException If the necessary files could not be created. 23 | */ 24 | @Override 25 | public OutputHandler getOutputHandler(String outputFile, QueryHandlerFactory queryHandlerFactory) throws IOException 26 | { 27 | return new OutputHandlerAnonymizer(outputFile, queryHandlerFactory); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/output/factories/OutputHandlerFactory.java: -------------------------------------------------------------------------------- 1 | package output.factories; 2 | 3 | import output.OutputHandler; 4 | import query.factories.QueryHandlerFactory; 5 | 6 | import java.io.FileNotFoundException; 7 | import java.io.IOException; 8 | 9 | /** 10 | * @author adrian 11 | */ 12 | public interface OutputHandlerFactory 13 | { 14 | /** 15 | * @param outputFile The file the output should be written to. 16 | * @param queryHandlerFactory The factory supplying the query handler to generate the output with. 17 | * @return An output handler corresponding to this factory based on the parameters. 18 | * @throws FileNotFoundException If the file could not be created or written to. 19 | * @throws IOException If the necessary files could not be created. 20 | */ 21 | OutputHandler getOutputHandler(String outputFile, QueryHandlerFactory queryHandlerFactory) throws IOException; 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/output/factories/OutputHandlerTSVFactory.java: -------------------------------------------------------------------------------- 1 | package output.factories; 2 | 3 | import output.OutputHandler; 4 | import output.OutputHandlerTSV; 5 | import query.factories.QueryHandlerFactory; 6 | 7 | import java.io.FileNotFoundException; 8 | import java.io.IOException; 9 | 10 | /** 11 | * @author adrian 12 | */ 13 | public class OutputHandlerTSVFactory implements OutputHandlerFactory 14 | { 15 | 16 | /** 17 | * @param outputFile The file the output should be written to. 18 | * @param queryHandlerFactory The factory supplying the query handler to generate the output with. 19 | * @return An output handler tsv based on the parameters. 20 | * @throws IOException If the necessary files could not be created. 21 | */ 22 | @Override 23 | public OutputHandler getOutputHandler(String outputFile, QueryHandlerFactory queryHandlerFactory) throws IOException 24 | { 25 | return new OutputHandlerTSV(outputFile, queryHandlerFactory); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/output/factories/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * @author adrian 3 | */ 4 | package output.factories; 5 | -------------------------------------------------------------------------------- /src/main/java/output/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * @author adrian 3 | */ 4 | package output; 5 | -------------------------------------------------------------------------------- /src/main/java/query/factories/OpenRDFQueryHandlerFactory.java: -------------------------------------------------------------------------------- 1 | package query.factories; 2 | 3 | import query.OpenRDFQueryHandler; 4 | import query.QueryHandler; 5 | import query.QueryHandler.Validity; 6 | 7 | /** 8 | * @author adrian 9 | */ 10 | public class OpenRDFQueryHandlerFactory implements QueryHandlerFactory 11 | { 12 | /** 13 | * @param validity The validity as determined by the decoding process. 14 | * @param lineToSet The line this query came from. 15 | * @param dayToSet The day this query came from. 16 | * @param queryStringToSet The query as a string. 17 | * @param userAgentToSet The user agent that send this query. 18 | * @param currentFileToSet The file this query came from. 19 | * @param threadNumberToSet The number of the thread (Needs to be unique per thread). 20 | * @return An OpenRDFQueryHandler based on the parameters. 21 | */ 22 | @Override 23 | public QueryHandler getQueryHandler(Validity validity, Long lineToSet, Integer dayToSet, String queryStringToSet, String userAgentToSet, String currentFileToSet, int threadNumberToSet) 24 | { 25 | return new OpenRDFQueryHandler(validity, lineToSet, dayToSet, queryStringToSet, userAgentToSet, currentFileToSet, threadNumberToSet); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/query/factories/QueryHandlerFactory.java: -------------------------------------------------------------------------------- 1 | package query.factories; 2 | 3 | import query.QueryHandler; 4 | import query.QueryHandler.Validity; 5 | 6 | /** 7 | * @author adrian 8 | */ 9 | public interface QueryHandlerFactory 10 | { 11 | /** 12 | * @param validity The validity as determined by the decoding process. 13 | * @param lineToSet The line this query came from. 14 | * @param dayToSet The day this query came from. 15 | * @param queryStringToSet The query as a string. 16 | * @param userAgentToSet The user agent that send this query. 17 | * @param currentFileToSet The file this query came from. 18 | * @param threadNumberToSet The number of the thread (Needs to be unique per thread). 19 | * @return A query handler corresponding to this factory based on the parameters. 20 | */ 21 | QueryHandler getQueryHandler(Validity validity, Long lineToSet, Integer dayToSet, String queryStringToSet, String userAgentToSet, String currentFileToSet, int threadNumberToSet); 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/query/factories/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * @author adrian 3 | */ 4 | package query.factories; 5 | -------------------------------------------------------------------------------- /src/main/java/query/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * @author Adrian-Bielefeldt 3 | */ 4 | package query; 5 | -------------------------------------------------------------------------------- /src/main/java/query/statistics/TupleExprSparqlStatisticsCollector.java: -------------------------------------------------------------------------------- 1 | package query.statistics; 2 | 3 | import org.openrdf.query.algebra.ArbitraryLengthPath; 4 | import org.openrdf.query.algebra.QueryModelNode; 5 | import org.openrdf.query.algebra.Service; 6 | import org.openrdf.query.algebra.StatementPattern; 7 | import org.openrdf.query.algebra.helpers.QueryModelVisitorBase; 8 | 9 | import java.util.HashMap; 10 | import java.util.LinkedList; 11 | 12 | 13 | /** 14 | * @author Julius Gonsior 15 | */ 16 | public class TupleExprSparqlStatisticsCollector extends QueryModelVisitorBase 17 | { 18 | private HashMap statistics = new HashMap<>(); 19 | private String primaryLanguage = "--DEFAULT--"; 20 | 21 | public HashMap getStatistics() 22 | { 23 | return statistics; 24 | } 25 | 26 | public String getPrimaryLanguage() 27 | { 28 | return primaryLanguage; 29 | } 30 | 31 | /** 32 | * Increment the statistic by one per keyword, but only once per Query. 33 | */ 34 | public void add(String keyword) 35 | { 36 | statistics.put(keyword, 1); 37 | } 38 | 39 | /** 40 | * default method which get's called by all meet statements 41 | * 42 | * @param node 43 | * @throws Exception 44 | */ 45 | @Override 46 | public void meetNode(QueryModelNode node) throws Exception 47 | { 48 | LinkedList neededNodes = new LinkedList<>(); 49 | //Distinct, Filter, And, , Join, Reduced, Value, 50 | neededNodes.add("Distinct"); 51 | neededNodes.add("Filter"); 52 | neededNodes.add("Join"); 53 | neededNodes.add("Reduced"); 54 | neededNodes.add("Graph"); 55 | String className = node.getClass().getSimpleName(); 56 | if (neededNodes.contains(className)) { 57 | this.add(className); 58 | } 59 | super.meetNode(node); 60 | } 61 | 62 | public void meet(StatementPattern node) throws Exception 63 | { 64 | if (node.getParentNode() instanceof Service && node.getSubjectVar().getName().equals("-const-http://www.bigdata.com/rdf#serviceParam-uri") && node.getPredicateVar().getName().equals("-const-http://wikiba.se/ontology#language-uri")) { 65 | this.add("LangService"); 66 | 67 | // save first asked language 68 | String value = node.getObjectVar().getValue().stringValue(); 69 | int index = value.indexOf(","); 70 | if (index != -1) { 71 | this.primaryLanguage = value.substring(0, index); 72 | } else { 73 | this.primaryLanguage = value; 74 | } 75 | } 76 | super.meetNode(node); 77 | } 78 | 79 | public void meet(ArbitraryLengthPath node) throws Exception 80 | { 81 | if (node.getMinLength() == 0) { 82 | this.add("*"); 83 | } else { 84 | this.add("+"); 85 | } 86 | super.meetNode(node); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/query/statistics/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * @author adrian 3 | */ 4 | package query.statistics; 5 | -------------------------------------------------------------------------------- /src/main/java/utility/NoURIException.java: -------------------------------------------------------------------------------- 1 | package utility; 2 | 3 | /** 4 | * @author adrian 5 | */ 6 | public class NoURIException extends Exception 7 | { 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/utility/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * @author adrian 3 | */ 4 | package utility; 5 | -------------------------------------------------------------------------------- /tools/Anonymize.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import config 3 | import os 4 | import subprocess 5 | import sys 6 | 7 | from utility import utility 8 | 9 | parser = argparse.ArgumentParser("This script creates an anonymous dataset from the rawLogData.") 10 | parser.add_argument("--ignoreLock", "-i", help="Ignore locked file and " 11 | + "execute anyways", action="store_true") 12 | parser.add_argument("--threads", "-t", default=10, type=int, help="The number " 13 | + "of threads to run the java program with (default 7).") 14 | parser.add_argument("--logging", "-l", help="Enables file logging.", 15 | action="store_true") 16 | parser.add_argument("--monthsFolder", "-m", default=config.monthsFolder, 17 | type=str, 18 | help="The folder in which the months directory are " 19 | + "residing.") 20 | parser.add_argument("--unanonymizedStringLength", "-u", default=10, type=int, 21 | help="Strings of this length or lower should not be anonymized. Default is ten.") 22 | parser.add_argument("months", type=str, help="The months to be processed") 23 | 24 | if (len(sys.argv[1:]) == 0): 25 | parser.print_help() 26 | parser.exit() 27 | 28 | args = parser.parse_args() 29 | 30 | for monthName in args.months.split(","): 31 | 32 | mavenCall = ['mvn', 'exec:java@Anonymizer'] 33 | 34 | month = utility.addMissingSlash(os.path.abspath(utility.addMissingSlash(args.monthsFolder) + utility.addMissingSlash(monthName))) 35 | mavenArguments = '-Dexec.args=-w ' + month + ' -n ' + str(args.threads) + ' -u ' + str(args.unanonymizedStringLength) 36 | if args.logging: 37 | mavenArguments += " -l" 38 | mavenCall.append(mavenArguments) 39 | 40 | owd = os.getcwd() 41 | os.chdir("..") 42 | 43 | print "Starting data processing using Anonymizer for " + monthName + "." 44 | 45 | if subprocess.call(['mvn', 'clean', 'package']) != 0: 46 | print "ERROR: Could not package the java application." 47 | sys.exit(1) 48 | 49 | if subprocess.call(mavenCall) != 0: 50 | print("ERROR: Could not execute the java application. Check the logs " 51 | + "for details or rerun this script with -l to generate logs.") 52 | sys.exit(1) 53 | 54 | os.chdir(owd) 55 | -------------------------------------------------------------------------------- /tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikidata/QueryAnalysis/2ae7b6226a3ce6a7ebccd3deff0ddd9460d4aa57/tools/__init__.py -------------------------------------------------------------------------------- /tools/config.py: -------------------------------------------------------------------------------- 1 | monthsFolder = "/home/mkroetzsch/querydata/months" 2 | queryReferenceDirectory = "/home/mkroetzsch/querydata/queryTypeReferenceFolder" 3 | fdupesExecutable = "/home/mkroetzsch/querydata/utility/fdupes" 4 | dbLocation = "/home/mkroetzsch/querydata/months/uniqueQueryMap.db" 5 | queryTypeMapDbLocation = "/home/mkroetzsch/querydata/months/queryTypeMap.db" 6 | 7 | processedFolderName = "processedLogData/" 8 | sourceFolderName = "rawLogData/" 9 | 10 | processedFilePrefix = "QueryProcessedOpenRDF" 11 | sourceFilePrefix = "QueryCnt" 12 | 13 | processedPrefix = processedFolderName + processedFilePrefix 14 | sourcePrefix = sourceFolderName + sourceFilePrefix 15 | anonymousPrefix = "anonymousRawData/AnonymousQueryCnt" 16 | -------------------------------------------------------------------------------- /tools/countAnything.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | import argparse 5 | from postprocess import processdata 6 | import config 7 | import os 8 | import sys 9 | from utility import utility 10 | import operator 11 | 12 | os.nice(19) 13 | 14 | parser = argparse.ArgumentParser( 15 | description="Tool to sum up the uses of RDF properties in queries") 16 | parser.add_argument("--monthsFolder", "-m", default=config.monthsFolder, 17 | type=str, help="The folder in which the months directory are residing.") 18 | parser.add_argument("--ignoreLock", "-i", help="Ignore locked file and " 19 | + "execute anyways", action="store_true") 20 | parser.add_argument('--onlyValid', "-o", action='store_true', help="If set " 21 | + "only valid lines are being looked at") 22 | parser.add_argument("month", type=str, help="The month from which lines " 23 | + "should be displayed.") 24 | parser.add_argument("parameter", type=str, help="The parameter by which to group.") 25 | 26 | if (len(sys.argv[1:]) == 0): 27 | parser.print_help() 28 | parser.exit() 29 | 30 | args = parser.parse_args() 31 | 32 | if os.path.isfile(utility.addMissingSlash(args.monthsFolder) + utility.addMissingSlash(args.month) + "locked") and not args.ignoreLock: 33 | print "ERROR: The month " + args.month + " is being edited at the moment. Use -i if you want to force the execution of this script." 34 | sys.exit() 35 | 36 | 37 | class CountRdfPropertiesHandler: 38 | queryCount = 0 39 | propQueryCounts = {} 40 | 41 | def handle(self,sparqlQuery,processed): 42 | self.queryCount += 1 43 | 44 | if args.onlyValid: 45 | if processed['#Valid'] is not 'VALID': 46 | return 47 | 48 | props = processed[args.parameter].split(",") 49 | 50 | for prop in props: 51 | if prop in self.propQueryCounts: 52 | c = self.propQueryCounts[prop] + 1 53 | else: 54 | c = 1 55 | self.propQueryCounts[prop] = c 56 | 57 | def printResults(self): 58 | print "Queries: %d" % (self.queryCount) 59 | print "\n\n%s\tcount" % (args.parameter) 60 | for p, c in sorted(self.propQueryCounts.iteritems(), key=operator.itemgetter(1), reverse=True): 61 | print "%s\t%d" % (p,c) 62 | 63 | handler = CountRdfPropertiesHandler() 64 | processdata.processMonth(handler, args.month, args.monthsFolder) 65 | 66 | handler.printResults() 67 | -------------------------------------------------------------------------------- /tools/countRdfProperties.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | import argparse 5 | from postprocess import processdata 6 | import config 7 | import os 8 | import sys 9 | from utility import utility 10 | import operator 11 | 12 | os.nice(19) 13 | 14 | parser = argparse.ArgumentParser( 15 | description="Tool to sum up the uses of RDF properties in queries") 16 | parser.add_argument("--monthsFolder", "-m", default=config.monthsFolder, 17 | type=str, help="The folder in which the months directory are residing.") 18 | parser.add_argument("--ignoreLock", "-i", help="Ignore locked file and " 19 | + "execute anyways", action="store_true") 20 | parser.add_argument('--onlyValid', "-o", action='store_true', help="If set " 21 | + "only valid lines are being looked at") 22 | parser.add_argument("month", type=str, help="The month from which lines " 23 | + "should be displayed.") 24 | 25 | if (len(sys.argv[1:]) == 0): 26 | parser.print_help() 27 | parser.exit() 28 | 29 | args = parser.parse_args() 30 | 31 | if os.path.isfile(utility.addMissingSlash(args.monthsFolder) + utility.addMissingSlash(args.month) + "locked") and not args.ignoreLock: 32 | print "ERROR: The month " + args.month + " is being edited at the moment. Use -i if you want to force the execution of this script." 33 | sys.exit() 34 | 35 | 36 | class CountRdfPropertiesHandler: 37 | queryCount = 0 38 | propQueryCounts = {} 39 | 40 | def handle(self,sparqlQuery,processed): 41 | self.queryCount += 1 42 | 43 | if args.onlyValid: 44 | if processed['#Valid'] is not 'VALID': 45 | return 46 | 47 | props = processed["#Predicates"].split(",") 48 | 49 | for prop in props: 50 | if prop in self.propQueryCounts: 51 | c = self.propQueryCounts[prop] + 1 52 | else: 53 | c = 1 54 | self.propQueryCounts[prop] = c 55 | 56 | def printResults(self): 57 | print "Queries: %d" % (self.queryCount) 58 | print "\n\nproperty\tcount" 59 | for p, c in sorted(self.propQueryCounts.iteritems(), key=operator.itemgetter(1), reverse=True): 60 | print "%s\t%d" % (p,c) 61 | 62 | handler = CountRdfPropertiesHandler() 63 | processdata.processMonth(handler, args.month, args.monthsFolder) 64 | 65 | handler.printResults() 66 | -------------------------------------------------------------------------------- /tools/countTools.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import pprint 4 | import sys 5 | from collections import defaultdict 6 | from postprocess import processdata 7 | from utility import utility 8 | import config 9 | 10 | parser = argparse.ArgumentParser( 11 | description="Counts the used tools/bots in the given folder") 12 | parser.add_argument("--monthsFolder", "-m", default=config.monthsFolder, 13 | type=str, help="the folder in which the months" 14 | + " directory are residing") 15 | parser.add_argument("--ignoreLock", "-i", 16 | help="Ignore locked file and execute anyways", 17 | action="store_true") 18 | parser.add_argument("month", type=str, 19 | help="the month which we're interested in") 20 | 21 | if (len(sys.argv[1:]) == 0): 22 | parser.print_help() 23 | parser.exit() 24 | 25 | args = parser.parse_args() 26 | 27 | if os.path.isfile(utility.addMissingSlash(args.monthsFolder) + 28 | utility.addMissingSlash(args.month) + "locked") \ 29 | and not args.ignoreLock: 30 | print "ERROR: The month " + args.month + " is being edited at the moment." 31 | + " Use -i if you want to force the execution of this script." 32 | sys.exit() 33 | 34 | 35 | class CountToolsHandler: 36 | toolCounter = defaultdict(int) 37 | 38 | def handle(self, sparqlQuery, processed): 39 | if (processed['#Valid'] == 'VALID' or processed['#Valid'] == '1'): 40 | self.toolCounter[processed['#ToolName']] += 1 41 | 42 | def __str__(self): 43 | return pprint.pformat(sorted(self.toolCounter.iteritems(), 44 | key=lambda x: x[1], reverse=True)) 45 | 46 | 47 | handler = CountToolsHandler() 48 | 49 | processdata.processMonth(handler, args.month, args.monthsFolder) 50 | 51 | print handler 52 | -------------------------------------------------------------------------------- /tools/countValid.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import sys 4 | from collections import defaultdict 5 | from postprocess import processdata 6 | from utility import utility 7 | import pprint 8 | import config 9 | 10 | parser = argparse.ArgumentParser(description="Counts the valid queries") 11 | parser.add_argument("--monthsFolder", "-m", default=config.monthsFolder, 12 | type=str, help="the folder in which the months directory" 13 | + " are residing") 14 | parser.add_argument("--ignoreLock", "-i", 15 | help="Ignore locked file and execute anyways", 16 | action="store_true") 17 | parser.add_argument("month", type=str, 18 | help="the month which we're interested in") 19 | 20 | if (len(sys.argv[1:]) == 0): 21 | parser.print_help() 22 | parser.exit() 23 | 24 | args = parser.parse_args() 25 | 26 | if os.path.isfile(utility.addMissingSlash(args.monthsFolder) 27 | + utility.addMissingSlash(args.month) + "locked") \ 28 | and not args.ignoreLock: 29 | print("ERROR: The month " + args.month + " is being edited at the moment." 30 | + " Use -i if you want to force the execution of this script.") 31 | sys.exit() 32 | 33 | 34 | class CountValidityHandler: 35 | validCounter = defaultdict(int) 36 | 37 | def handle(self, sparqlQuery, processed): 38 | self.validCounter[processed['#Valid']] += 1 39 | 40 | def __str__(self): 41 | return pprint.pformat(self.validCounter) 42 | 43 | 44 | handler = CountValidityHandler() 45 | 46 | processdata.processMonth(handler, args.month, args.monthsFolder) 47 | 48 | print(handler) 49 | -------------------------------------------------------------------------------- /tools/createTestData.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import sys 4 | from postprocess import processdata 5 | from utility import utility 6 | import config 7 | import glob 8 | import gzip 9 | from itertools import izip 10 | from random import random 11 | 12 | parser = argparse.ArgumentParser( 13 | description="Creates a smaller testdata set for developing of the" 14 | + " current querys in this folder") 15 | parser.add_argument("--monthsFolder", "-m", default=config.monthsFolder, 16 | type=str, help="the folder in which the months" 17 | + " directory are residing") 18 | parser.add_argument("--ignoreLock", "-i", 19 | help="Ignore locked file and execute anyways", 20 | action="store_true") 21 | parser.add_argument("month", type=str, 22 | help="the month which we're interested in") 23 | parser.add_argument("lines", type=int, 24 | help="number of lines the testfiles should have") 25 | 26 | if (len(sys.argv[1:]) == 0): 27 | parser.print_help() 28 | parser.exit() 29 | 30 | args = parser.parse_args() 31 | monthsFolder = args.monthsFolder 32 | month = args.month 33 | 34 | 35 | if os.path.isfile(utility.addMissingSlash(monthsFolder) + 36 | utility.addMissingSlash(month) + "locked") \ 37 | and not args.ignoreLock: 38 | print "ERROR: The month " + args.month + " is being edited at the moment." 39 | + " Use -i if you want to force the execution of this script." 40 | sys.exit() 41 | 42 | # create new folder for the test data 43 | os.makedirs("testData/processedLogData") 44 | os.makedirs("testData/rawLogData") 45 | 46 | for filename in glob.glob(monthsFolder + "/" + month 47 | + "/processedLogData/" + processdata.processedPrefix 48 | + "*" + processdata.processedSuffix): 49 | day = int(os.path.basename(filename)[len(processdata.processedPrefix):] 50 | [:-len(processdata.processedSuffix)]) 51 | processedFileName = utility.addMissingSlash(monthsFolder) + month \ 52 | + "/processedLogData/" + processdata.processedPrefix \ 53 | + "%02d" % day + processdata.processedSuffix 54 | print "Working on: " + processedFileName 55 | 56 | with gzip.open(processedFileName) as p, \ 57 | gzip.open("testData/processedLogData/" + 58 | processdata.processedPrefix + "%02d" % day 59 | + ".tsv.gz", "wb") as pc,\ 60 | gzip.open("testData/rawLogData/" + 61 | processdata.sourcePrefix + 62 | "%02d" % day + ".tsv.gz", "wb") as sc: 63 | 64 | chancesSelected = float(args.lines) / float(sum(1 for line in p)) 65 | 66 | print "Done counting lines, chance for selection is " \ 67 | + str(chancesSelected) 68 | 69 | headerRow = True 70 | 71 | for processed, source in izip(gzip.open(processedFileName), 72 | gzip.open(monthsFolder + "/" + month + 73 | "/rawLogData/" + 74 | processdata.sourcePrefix + 75 | "%02d" % day + ".tsv.gz")): 76 | if(random() >= chancesSelected and not headerRow): 77 | continue 78 | elif headerRow: 79 | headerRow = False 80 | 81 | pc.write(processed) 82 | sc.write(source) 83 | -------------------------------------------------------------------------------- /tools/exampleDatasetGenerator.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import argparse 4 | import glob 5 | import gzip 6 | import random 7 | import sys 8 | import urllib 9 | 10 | from utility import utility 11 | 12 | parser = argparse.ArgumentParser(description = "This script generates an example raw log file based on a folder with example queries") 13 | parser.add_argument("--exampleQueryFolder", "-e", default="", 14 | type=str, help="The folder in which the example queries are residing.") 15 | parser.add_argument("--outputDirectory", "-o", default="", 16 | type=str, help="The directory in which the QueryCnt01.tsv.gz-file should be created.") 17 | 18 | if (len(sys.argv[1:]) == 0): 19 | parser.print_help() 20 | parser.exit() 21 | 22 | args = parser.parse_args() 23 | 24 | uri_path = {"/sparql", "/bigdata/namespace/wdq/sparql"} 25 | user_agent = {"Mozilla/5.0 (Android 4.4; Mobile; rv:41.0) Gecko/41.0 Firefox/41.0", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:53.0) Gecko/20100101 Firefox/53.0"} 26 | timestamp = {"2017-07-01 01:31:14", "2017-07-01 01:32:54", "2017-07-01 01:34:10"} 27 | agent_type = {"spider", "user"} 28 | http_status = "200" 29 | 30 | with gzip.open(utility.addMissingSlash(args.outputDirectory) + "QueryCnt01.tsv.gz", "w") as target: 31 | print("uri_query\turi_path\tuser_agent\tts\tagent_type\thour\thttp_status", file = target) 32 | 33 | exampleQueryFolder = utility.addMissingSlash(args.exampleQueryFolder) 34 | 35 | for filename in glob.glob(exampleQueryFolder + "*.exampleQuery"): 36 | with open(filename) as exampleFile: 37 | line = "?query=" + urllib.quote_plus(exampleFile.read()) + "\t" 38 | line += random.sample(uri_path, 1)[0] + "\t" 39 | line += random.sample(user_agent, 1)[0] + "\t" 40 | line += random.sample(timestamp, 1)[0] + "\t" 41 | line += random.sample(agent_type, 1)[0] + "\t" 42 | line += str(random.randint(0,23)) + "\t" 43 | line += http_status 44 | print(line, file = target) 45 | -------------------------------------------------------------------------------- /tools/extractMonth.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ ! $# -eq 3 ] 4 | then 5 | echo "Arguments: " 6 | exit 7 | fi 8 | 9 | found=false 10 | month=0 11 | declare -A months=( ["Jan"]="1" ["Feb"]="2" ["Mar"]="3" ["Apr"]="4" ["May"]="5" ["Jun"]="6" ["Jul"]="7" ["Aug"]="8" ["Sep"]="9" ["Oct"]="10" ["Nov"]="11" ["Dec"]="12" ) 12 | for m in "${!months[@]}"; 13 | do 14 | if [ $2 = $m ] 15 | then 16 | month=${months[$m]}; 17 | found=true 18 | fi 19 | done 20 | 21 | if [ $found = false ] 22 | then 23 | echo "Month not in list. Use one of the following:" 24 | echo "Jan, Feb, Mar, Apr, May, Jun, Jul, Aug, Sep, Oct, Nov, Dec" 25 | exit 26 | fi 27 | 28 | filePrefix="queryCnt" 29 | ext=".tsv" 30 | 31 | for i in $(seq -f "%02g" 1 $3) 32 | do 33 | mkdir temp 34 | hive -e "insert overwrite local directory 'temp' row format delimited fields terminated by '\t' select uri_query, uri_path, user_agent, ts, agent_type, hour, http_status from wmf.wdqs_extract where uri_query<>\"\" and year='$1' and month='$month' and day='$i'" 35 | echo -e "uri_query\turi_path\tuser_agent\tts\tagent_type\thour\thttp_status" > $filePrefix$i$ext 36 | cat temp/* >> $filePrefix$i$ext 37 | rm -r temp 38 | done 39 | -------------------------------------------------------------------------------- /tools/generalStat.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import sys 4 | from collections import defaultdict 5 | from pprint import pprint 6 | 7 | import config 8 | from postprocess import processdata 9 | from utility import utility 10 | import itertools 11 | 12 | parser = argparse.ArgumentParser( 13 | description="Prints out general statistics about FIRST/COPY") 14 | parser.add_argument( 15 | "--monthsFolder", 16 | "-m", 17 | default=config.monthsFolder, 18 | type=str, 19 | help="the folder in which the months directory " + "are residing") 20 | parser.add_argument( 21 | "--ignoreLock", 22 | "-i", 23 | help="Ignore locked file and execute" + " anyways", 24 | action="store_true") 25 | parser.add_argument( 26 | "--position", 27 | "-p", 28 | default="default position", 29 | type=str, 30 | help="The position to be displayed before the data.") 31 | parser.add_argument( 32 | "month", type=str, help="the month which we're interested in") 33 | 34 | if (len(sys.argv[1:]) == 0): 35 | parser.print_help() 36 | parser.exit() 37 | 38 | args = parser.parse_args() 39 | 40 | if os.path.isfile(utility.addMissingSlash(args.monthsFolder) 41 | + utility.addMissingSlash(args.month) + "locked") \ 42 | and not args.ignoreLock: 43 | print("ERROR: The month " + str(args.month) + 44 | " is being edited at the moment." + 45 | " Use -i if you want to force the execution of this script.") 46 | sys.exit() 47 | 48 | 49 | class GeneralStatisticsHandler: 50 | statistic = defaultdict(int) 51 | totalCount = 0 52 | 53 | def handle(self, sparqlQuery, processed): 54 | if (processed['#Valid'] == 'VALID'): 55 | self.totalCount += 1 56 | self.statistic[processed['#First']] += 1 57 | self.statistic[processed['#QueryComplexity']] += 1 58 | if processed['#ExampleQueryStringComparison'] != "NONE": 59 | self.statistic['EXAMPLE_STRING'] += 1 60 | if processed['#ExampleQueryParsedComparison'] != "NONE": 61 | self.statistic['EXAMPLE_PARSED'] += 1 62 | 63 | def printStat(self): 64 | #pprint(self.statistic) 65 | print( 66 | "Month\tFirst\tCopy\tSIMPLE\tCOMPLEX\tEXAMPLE_STRING\tEXAMPLE_PARSED" 67 | ) 68 | print( 69 | args.month + "\t" + str(self.statistic["FIRST"]) + "\t" + 70 | str(self.statistic["COPY"]) + "\t" + str(self.statistic["SIMPLE"]) 71 | + "\t" + str(self.statistic["COMPLEX"]) + "\t" + 72 | str(self.statistic["EXAMPLE_STRING"]) + "\t" + 73 | str(self.statistic["EXAMPLE_PARSED"])) 74 | 75 | 76 | handler = GeneralStatisticsHandler() 77 | 78 | processdata.processMonth( 79 | handler, args.month, args.monthsFolder, notifications=False) 80 | 81 | print args.position 82 | print "" 83 | 84 | handler.printStat() 85 | -------------------------------------------------------------------------------- /tools/geoHeatMap.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | from pprint import pprint 4 | import sys 5 | import os.path 6 | import cartopy.crs as ccrs 7 | import matplotlib.pyplot as plt 8 | from postprocess import processdata 9 | from utility import utility 10 | import config 11 | 12 | parser = argparse.ArgumentParser( 13 | description="Generaets a heatmap based on geo coordinates") 14 | parser.add_argument("--monthsFolder", "-m", default=config.monthsFolder, 15 | type=str, help="the folder in which the months directory " 16 | + "are residing") 17 | parser.add_argument("--ignoreLock", "-i", help="Ignore locked file and execute" 18 | + " anyways", action="store_true") 19 | parser.add_argument("month", type=str, 20 | help="the month which we're interested in") 21 | 22 | if (len(sys.argv[1:]) == 0): 23 | parser.print_help() 24 | parser.exit() 25 | 26 | args = parser.parse_args() 27 | 28 | if os.path.isfile(utility.addMissingSlash(args.monthsFolder) 29 | + utility.addMissingSlash(args.month) + "locked") \ 30 | and not args.ignoreLock: 31 | print "ERROR: The month " + args.month + " is being edited at the moment." 32 | + " Use -i if you want to force the execution of this script." 33 | sys.exit() 34 | 35 | 36 | class GeoCoordinateCollectorHandler: 37 | coordinates = set() 38 | 39 | def handle(self, sparqlQuery, processed): 40 | if (processed['#Valid'] == 'VALID' or processed['#Valid'] == '1'): 41 | if(processed['#Coordinates'] is not ''): 42 | for coordinate in processed['#Coordinates'].split(","): 43 | self.coordinates.add(coordinate) 44 | 45 | def saveSetToJson(self): 46 | pprint(self.coordinates) 47 | with open('geoCoordinates.tsv', 'w') as geoCoordinatesFile: 48 | for coordinate in self.coordinates: 49 | geoCoordinatesFile.write(coordinate.replace(" ", "\t") + "\n") 50 | 51 | 52 | if not os.path.isfile('geoCoordinates.tsv'): 53 | # first get all geo coordinates and save them to a file 54 | handler = GeoCoordinateCollectorHandler() 55 | processdata.processMonth(handler, args.month, args.monthsFolder) 56 | handler.saveSetToJson() 57 | 58 | else: 59 | # parse geoCoordinates.tsv and create choropleth map 60 | 61 | ax = plt.axes(projection=ccrs.PlateCarree()) 62 | ax.coastlines() 63 | ax.stock_img() 64 | 65 | with open('geoCoordinates.tsv', 'r') as file: 66 | for line in file: 67 | lat, lon = line.strip('\n').split(" ") 68 | plt.plot(float(lat), float(lon), color='red', 69 | alpha=.3, marker='.', transform=ccrs.PlateCarree()) 70 | # pprint(line.split(" ")) 71 | 72 | plt.show() 73 | -------------------------------------------------------------------------------- /tools/getDriveStatistics.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # must be executed from inside of the tools folder 3 | WORKING_DIR=$1 4 | MONTH=$2 5 | 6 | # extract datasets 7 | python2 extractUserData.py -m $1 $2 8 | python2 extractUniqueDataset.py -m $1/$2 userData 9 | python2 extractQueryTypeDataset.py -m $1/$2 userData 10 | 11 | # calculate stats 12 | mkdir -p logs/sp 13 | mkdir -p logs/op 14 | 15 | function analyzeMonth() { 16 | python2 getSparqlStatistic.py -m $1 $2 > logs/sp/sparqlStatistic$3$2.txt 17 | python2 operatorUsageStatistic.py -m $1 $2 > logs/op/operatorUsageStatistic$3$2.txt 18 | python2 generalStat.py -m $1 $2 > logs/generalStat$3$2.txt 19 | echo $3_$2 >> logs/generalStat$3$2.txt 20 | echo $3_$2 >> logs/sp/sparqlStatistic$3$2.txt 21 | echo $3_$2 >> logs/op/operatorUsageStatistic$3$2.txt 22 | } 23 | 24 | analyzeMonth $1 $2 $2 25 | analyzeMonth $1/$2 userData $2 26 | analyzeMonth $1/$2/userData queryTypeDataset $2 27 | analyzeMonth $1/$2/userData uniqueQueryDataset $2 28 | -------------------------------------------------------------------------------- /tools/joinMonth.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import gzip 3 | import os 4 | import sys 5 | 6 | import config 7 | from utility import utility 8 | 9 | def joinMonth(month, monthsFolder = config.monthsFolder, ignoreLock = False, outputPath = None, outputFilename = None): 10 | if os.path.isfile(utility.addMissingSlash(monthsFolder) + utility.addMissingSlash(month) + "locked") and not ignoreLock: 11 | print "ERROR: The month " + month + " is being edited at the moment. Use -i or ignoreLock = True if you want to force the execution of this script." 12 | sys.exit() 13 | 14 | anonymizedFolder = "anonymousRawData/" 15 | anonymizedPrefix = anonymizedFolder + "AnonymousQueryCnt" 16 | 17 | pathBase = utility.addMissingSlash(monthsFolder) \ 18 | + utility.addMissingSlash(month) 19 | 20 | outputFile = month.strip("/").replace("/", "_") + "_Joined.tsv.gz" 21 | 22 | if outputFilename is not None: 23 | outputFile = outputFilename 24 | 25 | targetFile = pathBase + anonymizedFolder 26 | if outputPath is not None: 27 | targetFile = outputPath 28 | if not os.path.exists(targetFile): 29 | os.makedirs(targetFile) 30 | targetFile += outputFile 31 | 32 | with gzip.open(targetFile, "w") as target: 33 | headerSet = False 34 | for i in xrange(1, 32): 35 | print "Working on %02d" % i 36 | sourceFile = pathBase + anonymizedPrefix + "%02d" % i + ".tsv.gz" 37 | if not (os.path.exists(sourceFile)): 38 | continue 39 | with gzip.open(sourceFile) as source: 40 | if headerSet: 41 | next(source) 42 | else: 43 | headerSet = True 44 | for line in source: 45 | target.write(line) 46 | 47 | if __name__ == '__main__': 48 | parser = argparse.ArgumentParser( 49 | description="This script joins all the anonymized files into one while keeping only the first header.") 50 | parser.add_argument("--monthsFolder", "-m", default=config.monthsFolder, 51 | type=str, help="The folder in which the months directory " 52 | + "are residing.") 53 | parser.add_argument("--ignoreLock", "-i", help="Ignore locked file and execute" 54 | + " anyways", action="store_true") 55 | parser.add_argument("--outputPath", "-p", type=str, help="The path where the " 56 | + "output file should be generated.") 57 | parser.add_argument("--outputFilename", "-o", type=str, help="The name of the output file to be generated.") 58 | parser.add_argument("months", type=str, 59 | help="the months which we're interested in") 60 | 61 | if (len(sys.argv[1:]) == 0): 62 | parser.print_help() 63 | parser.exit() 64 | 65 | args = parser.parse_args() 66 | 67 | for monthName in args.months.split(","): 68 | joinMonth(monthName, monthsFolder = args.monthsFolder, ignoreLock = args.ignoreLock, outputPath = args.outputPath, outputFilename = args.outputFilename) 69 | -------------------------------------------------------------------------------- /tools/mergeOneColumnTsv.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | import argparse 5 | import sys 6 | 7 | parser = argparse.ArgumentParser( 8 | description="Tool to combine TSV files that have one key column and one value column into one TSV") 9 | parser.add_argument("files", type=str, nargs='+', help="List of files to merge") 10 | 11 | args = parser.parse_args() 12 | 13 | def getEmptyRecord(): 14 | record = {} 15 | for filename in args.files: 16 | record[filename] = '0' 17 | return record 18 | 19 | results = {} 20 | for filename in args.files: 21 | with open(filename, 'rU') as f: 22 | for line in f: 23 | pieces = line.strip().split("\t") 24 | if len(pieces) == 2: 25 | if pieces[0] in results: 26 | record = results[pieces[0]] 27 | else: 28 | record = getEmptyRecord() 29 | record[filename] = pieces[1] 30 | results[pieces[0]] = record 31 | 32 | sys.stdout.write("key\t") 33 | for filename in args.files: 34 | sys.stdout.write("%s\t" % (filename)) 35 | sys.stdout.write("\n") 36 | 37 | 38 | for record in results: 39 | sys.stdout.write("%s\t" % (record)) 40 | for filename in args.files: 41 | sys.stdout.write("%s\t" % (results[record][filename])) 42 | sys.stdout.write("\n") 43 | 44 | -------------------------------------------------------------------------------- /tools/oldScripts/fieldRanking.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wikidata/QueryAnalysis/2ae7b6226a3ce6a7ebccd3deff0ddd9460d4aa57/tools/oldScripts/fieldRanking.py -------------------------------------------------------------------------------- /tools/oldScripts/getProperties.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os.path 3 | 4 | import requests 5 | import urllib.request 6 | 7 | with urllib.request.urlopen("https://tools.wmflabs.org/hay/propbrowse/props." 8 | + "json") as url: 9 | allProperties = json.loads(url.read().decode()) 10 | 11 | categories = requests.get('https://query.wikidata.org/bigdata/namespace' 12 | + '/wdq/sparql', params={ 13 | 'query': '#TOOL: jgonsior \n' 14 | 'SELECT ?item ' 15 | '?itemLabel WHERE {' 16 | '?item wdt:P279 wd:Q18616576 . ' 17 | 'SERVICE wikibase:label ' 18 | '{bd:serviceParam wikibase:language "en"' 19 | '}}', 20 | 'format': 'json' 21 | }).json() 22 | print(len(allProperties)) 23 | 24 | for category in categories['results']['bindings']: 25 | for property in allProperties: 26 | answer = requests.get( 27 | 'https://query.wikidata.org/bigdata/namespace' 28 | '/wdq/sparql', 29 | params={ 30 | 'query': '#TOOL: jgonsior \n' 31 | 'SELECT ?x ' 32 | '{?x wdt:P31 wd:' + 33 | os.path.basename(category['item']['value']) + '}', 34 | 'format': 'json'}).json() 35 | # pprint(answer) 36 | -------------------------------------------------------------------------------- /tools/postprocess/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ['submodule1'] 2 | -------------------------------------------------------------------------------- /tools/rankDataTypes.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import re 4 | import sys 5 | 6 | from collections import defaultdict 7 | 8 | import config 9 | 10 | from postprocess import processdata 11 | from utility import utility 12 | 13 | parser = argparse.ArgumentParser( 14 | description="Tool to rank the used data types") 15 | parser.add_argument("--monthsFolder", "-m", default=config.monthsFolder, 16 | type=str, help="The folder in which the months directory" 17 | + " are residing.") 18 | parser.add_argument("--ignoreLock", "-i", help="Ignore locked file and " 19 | + "execute anyways", action="store_true") 20 | parser.add_argument("month", type=str, help="The month from which lines " 21 | + "should be displayed.") 22 | parser.add_argument("--anonymous", "-a", action="store_true", help="Check to switch to ranking the anonymous data.") 23 | 24 | if (len(sys.argv[1:]) == 0): 25 | parser.print_help() 26 | parser.exit() 27 | 28 | args = parser.parse_args() 29 | 30 | if os.path.isfile(utility.addMissingSlash(args.monthsFolder) 31 | + utility.addMissingSlash(args.month) + "locked") \ 32 | and not args.ignoreLock: 33 | print "ERROR: The month " + args.month + " is being edited at the " 34 | + "moment. Use -i if you want to force the execution of this script." 35 | sys.exit() 36 | 37 | ranking = defaultdict(int) 38 | 39 | class rankDataTypesHandler: 40 | 41 | def handle(self, sparqlQuery, processed): 42 | for entry in re.findall(r'\^\^(.*?)( |\)|\\n)', str(sparqlQuery)): 43 | ranking[entry[0]] += 1 44 | 45 | handler = rankDataTypesHandler() 46 | 47 | if args.anonymous: 48 | processdata.processMonth(handler, args.month, args.monthsFolder, anonymous = True) 49 | else: 50 | processdata.processMonth(handler, args.month, args.monthsFolder) 51 | 52 | print "count\tdataType" 53 | for k, v in sorted(ranking.iteritems(), key=lambda (k, v): (v, k), reverse=True): 54 | print str(v) + "\t" + k 55 | -------------------------------------------------------------------------------- /tools/sortByTime.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import csv 3 | import gzip 4 | import os 5 | import shutil 6 | import sys 7 | 8 | import pandas 9 | 10 | import config 11 | 12 | from utility import utility 13 | 14 | parser = argparse.ArgumentParser(description="Sorts the raw log files by timestamp.") 15 | parser.add_argument("--monthsFolder", "-m", 16 | default=config.monthsFolder, type=str, 17 | help="The folder in which the months directories are" 18 | + " residing.") 19 | parser.add_argument("--ignoreLock", "-i", help="Ignore locked file and execute" 20 | + " anyways", action="store_true") 21 | parser.add_argument("--anonymous", "-a", action="store_true", help="Check to switch to sort the anonymous data.") 22 | parser.add_argument("month", type=str, 23 | help="The month whose raw log files should be sorted.") 24 | 25 | 26 | if (len(sys.argv[1:]) == 0): 27 | parser.print_help() 28 | parser.exit() 29 | 30 | args = parser.parse_args() 31 | 32 | if args.anonymous: 33 | sourcePrefix = config.anonymousPrefix 34 | key = "timestamp" 35 | else: 36 | sourcePrefix = config.sourcePrefix 37 | key = "ts" 38 | 39 | if os.path.isfile(utility.addMissingSlash(args.monthsFolder) + utility.addMissingSlash(args.month) + "locked") and not args.ignoreLock: 40 | print "ERROR: The month " + args.month + " is being edited at the moment. Use -i if you want to force the execution of this script." 41 | sys.exit() 42 | 43 | os.chdir(utility.addMissingSlash(args.monthsFolder) + utility.addMissingSlash(args.month)) 44 | 45 | for i in xrange(1, 32): 46 | filename = sourcePrefix + "%02d" % i 47 | filename_gzip = filename + ".tsv.gz" 48 | filename_tsv = filename + ".tsv" 49 | if not os.path.exists(filename_gzip): 50 | continue 51 | print "Working on %02d" % i 52 | with gzip.open(filename_gzip, 'rb') as input_file, open(filename_tsv, 'wb') as output_file: 53 | shutil.copyfileobj(input_file, output_file) 54 | os.remove(filename_gzip) 55 | df = pandas.read_csv(filename_tsv, sep="\t", header=0, index_col=0) 56 | df = df.sort_values(by=[key]) 57 | df.to_csv(filename_tsv, sep="\t") 58 | with open (filename_tsv, 'rb') as input_file, gzip.open(filename_gzip, 'wb') as output_file: 59 | shutil.copyfileobj(input_file, output_file) 60 | os.remove(filename_tsv) 61 | -------------------------------------------------------------------------------- /tools/utility/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ['submodule1'] 2 | -------------------------------------------------------------------------------- /tools/utility/utility.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import re 4 | 5 | # This list contains all fields that should not be split because they could contain commas 6 | notToSplit = ["user_agent", "ToolName"] 7 | 8 | def listToString(list): 9 | returnString = "" 10 | for entry in list: 11 | returnString += entry + "," 12 | return returnString[:-1] 13 | 14 | def addMissingSlash(directoryString): 15 | if not directoryString.endswith("/"): 16 | return directoryString + "/" 17 | return directoryString 18 | 19 | def argMetric(metric): 20 | if metric.startswith("#"): 21 | return metric[1:] 22 | else: 23 | return metric 24 | 25 | def fetchEntries(processed, metric, nosplitting = False): 26 | metric = argMetric(metric) 27 | if metric == "monthly_hour": 28 | try: 29 | hour = int(processed["hour"]) 30 | except ValueError: 31 | print processed["hour"] + " could not be parsed as integer" 32 | return [] 33 | if hour not in xrange(0,24): 34 | print str(hour) + " is not in 0-23" 35 | return [] 36 | try: 37 | day = int(processed["day"]) 38 | except: 39 | print processed["day"] + " could not be parsed as integer" 40 | return [] 41 | 42 | return [hour + 24 * (day - 1)] 43 | else: 44 | data = processed[metric] 45 | if metric in notToSplit: 46 | return [data] 47 | else: 48 | field_array = splitEntry(data) 49 | if nosplitting: 50 | field_array = sorted(field_array) 51 | return [listToString(field_array)] 52 | else: 53 | return field_array 54 | 55 | def splitEntry(entry): 56 | field_array = entry.split(",") 57 | field_array = map(lambda it: it.strip(), field_array) 58 | field_array = [x for x in field_array if x] 59 | return field_array 60 | 61 | class filter: 62 | 63 | parameters = dict() 64 | 65 | def setup(self, filterParameter): 66 | self.parameters["Valid"] = re.compile("^VALID$") 67 | 68 | filters = filterParameter.split(",") 69 | 70 | if filters == ['']: 71 | return 72 | 73 | for element in filters: 74 | arguments = element.split("=") 75 | self.parameters[arguments[0]] = re.compile(arguments[1]) 76 | 77 | def checkLine(self, processed): 78 | for key, value in self.parameters.iteritems(): 79 | match = re.match(value, str(processed[key])) 80 | if match == None: 81 | return False 82 | return True 83 | -------------------------------------------------------------------------------- /tools/visualisePropertyTree/bower.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "visualisePropertyTree", 3 | "homepage": "https://github.com/Wikidata/QueryAnalysis", 4 | "authors": [ 5 | "Julius Gonsior " 6 | ], 7 | "description": "", 8 | "main": "", 9 | "license": "Apache-2.0", 10 | "private": true, 11 | "ignore": [ 12 | "**/.*", 13 | "node_modules", 14 | "bower_components", 15 | "test", 16 | "tests" 17 | ], 18 | "dependencies": { 19 | "jquery-treetable": "^3.2.0", 20 | "bootstrap": "^3.3.7" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /tools/visualisePropertyTree/requirements.txt: -------------------------------------------------------------------------------- 1 | certifi==2017.4.17 2 | chardet==3.0.4 3 | idna==2.5 4 | isodate==0.5.4 5 | numpy==1.13.0 6 | pandas==0.20.2 7 | pyparsing==2.2.0 8 | python-dateutil==2.6.0 9 | pytz==2017.2 10 | rdflib==4.2.2 11 | requests==2.18.1 12 | six==1.10.0 13 | SPARQLWrapper==1.8.0 14 | urllib3==1.21.1 15 | -------------------------------------------------------------------------------- /tools/visualisePropertyTree/script.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | $('#propertyTreeTable').treetable({ 3 | expandable: true, 4 | clickableNodeNames: true 5 | }); 6 | 7 | $('#propertyTreeTable').treetable("expandNode", "/http://www.wikidata.org/entity/Q18616576"); -------------------------------------------------------------------------------- /tools/visualisePropertyTree/style.css: -------------------------------------------------------------------------------- 1 | 2 | table.treetable { 3 | table-layout: fixed !important; 4 | width: 47em; 5 | margin-left: auto; 6 | margin-right: auto; 7 | } 8 | 9 | th { 10 | font-weight: bold !important; 11 | vertical-align: middle !important; 12 | text-align: center !important; 13 | font-size: 1.1em; 14 | border: 0 !important; 15 | background: #f9f9f9 !important; 16 | } 17 | 18 | th.property { 19 | text-align: left !important; 20 | padding-left: 2em !important; 21 | } 22 | 23 | th.property, th.qid { 24 | font-size: 1.6em; 25 | } 26 | 27 | tr.branch td.property { 28 | cursor:pointer !important; 29 | } 30 | 31 | .property { 32 | width: 30em; 33 | } 34 | 35 | .qid { 36 | text-align: center; 37 | width: 7em; 38 | } 39 | 40 | .percentage { 41 | text-align: right !important; 42 | width: 10em; 43 | } -------------------------------------------------------------------------------- /userAgentClassification/toolNameForUserCategory.tsv: -------------------------------------------------------------------------------- 1 | tool names to be included in the user source category 2 | WikiShootMe 3 | SQID, http://tools.wmflabs.org/sqid/ 4 | Histropedia-WQT v0.2.5 5 | -------------------------------------------------------------------------------- /userAgentClassification/userAgentRegex.dat: -------------------------------------------------------------------------------- 1 | ^.*Mozilla/[0-9]+\.[0-9]+.*$ 2 | ^.*Firefox/[0-9]+\.[0-9]+.*$ 3 | ^.*Seamonkey/[0-9]+\.[0-9]+.*$ 4 | ^.*Chrome/[0-9]+\.[0-9]+.*$ 5 | ^.*Chromium/[0-9]+\.[0-9]+.*$ 6 | ^.*Safari/[0-9]+\.[0-9]+.*$ 7 | ^.*OPR/[0-9]+\.[0-9]+.*$ 8 | ^.*Opera/[0-9]+\.[0-9]+.*$ 9 | ^.*Firefox/[0-9]+\.[0-9]+.*$ 10 | ^.*UCWEB/[0-9]+\.[0-9]+.*$ 11 | ^.*; MSIE 0-9]+\.[0-9]+;.*$ 12 | --------------------------------------------------------------------------------