├── .github └── workflows │ └── publish-docker.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── LSQVocab_v2.ttl ├── LSQ_Vocab.ttl ├── Makefile ├── README.md ├── docker-from-uberjar ├── Dockerfile ├── README.md └── copy-uberjar.sh ├── docs ├── CNAME ├── _config.yml ├── index.md ├── legacy.md ├── v1 │ ├── images │ │ ├── background.png │ │ ├── bg-ramp.jpg │ │ ├── bg_hr.png │ │ ├── blacktocat.png │ │ ├── blockquote-gfx-2x.png │ │ ├── blockquote-gfx.png │ │ ├── body-background.png │ │ ├── body-bg.jpg │ │ ├── bullet.png │ │ ├── chevron-2x.png │ │ ├── chevron.png │ │ ├── download-button.png │ │ ├── download-fallback-bg.png │ │ ├── download-sprite.png │ │ ├── footer-ramp.jpg │ │ ├── fork-sprite.png │ │ ├── github-button.png │ │ ├── header-bg.jpg │ │ ├── highlight-bg.jpg │ │ ├── hr-2x.jpg │ │ ├── hr.gif │ │ ├── hr.jpg │ │ ├── icon_download.png │ │ ├── octocat-2x.png │ │ ├── octocat-logo.png │ │ ├── octocat.png │ │ ├── ribbon-tail-sprite-2x.png │ │ ├── ribbon-tail-sprite.png │ │ ├── shield-fallback.png │ │ ├── shield.png │ │ ├── sidebar-bg.jpg │ │ ├── site-2.png │ │ ├── small-ribbon-tail-sprite-2x.png │ │ ├── small-ribbon-tail-sprite.png │ │ └── sprite_download.png │ ├── index.html │ ├── lsq-images │ │ └── lsq-vocab.png │ └── stylesheets │ │ ├── core.css │ │ ├── github-light.css │ │ ├── mobile.css │ │ ├── non-screen.css │ │ ├── normalize.css │ │ ├── print.css │ │ ├── pygment_trac.css │ │ ├── screen.css │ │ ├── styles.css │ │ └── stylesheet.css └── v2 │ ├── Ad-hoc-Analytical-Queries.md │ ├── Erroneous-Queries.md │ ├── concepts │ ├── concepts.md │ ├── data-model.md │ ├── named-graph-streams.md │ └── skolemization.md │ ├── development │ ├── development.md │ └── modules.md │ ├── images │ ├── lsq2-datamodel.png │ └── lsq2-datamodel.svg │ ├── news │ ├── 2020-08-13.md │ └── news.md │ ├── resources │ ├── downloads │ │ └── index.md │ ├── index.md │ └── sparql-endpoint.md │ ├── setup.md │ └── usage │ ├── benchmarking.md │ ├── probing-logs.md │ ├── processing-lsq-data.md │ ├── rdfization.md │ ├── rx-vs-spark.md │ ├── usage.md │ └── useful-queries.md ├── lsq-cli ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── aksw │ │ │ └── simba │ │ │ └── lsq │ │ │ └── cli │ │ │ ├── cmd │ │ │ ├── base │ │ │ │ ├── CmdLsqAnalyzeBase.java │ │ │ │ ├── CmdLsqMain.java │ │ │ │ ├── CmdLsqRdfizeBase.java │ │ │ │ └── CmdOutputSpecBase.java │ │ │ ├── rx │ │ │ │ └── api │ │ │ │ │ ├── CmdLsqRxAnalyze.java │ │ │ │ │ ├── CmdLsqRxBenchmarkCreate.java │ │ │ │ │ ├── CmdLsqRxBenchmarkParent.java │ │ │ │ │ ├── CmdLsqRxBenchmarkPrepare.java │ │ │ │ │ ├── CmdLsqRxBenchmarkRun.java │ │ │ │ │ ├── CmdLsqRxParent.java │ │ │ │ │ ├── CmdLsqRxProbe.java │ │ │ │ │ ├── CmdLsqRxRdfize.java │ │ │ │ │ ├── CmdRdfIoBase.java │ │ │ │ │ └── CmdRdfIoBaseTmp.java │ │ │ └── spark │ │ │ │ ├── api │ │ │ │ ├── CmdLsqSparkAnalyze.java │ │ │ │ ├── CmdLsqSparkParent.java │ │ │ │ ├── CmdLsqSparkProbe.java │ │ │ │ └── CmdLsqSparkRdfize.java │ │ │ │ └── impl │ │ │ │ └── LsqCmdSparkImpls.java │ │ │ ├── main │ │ │ └── MainCliLsq.java │ │ │ └── util │ │ │ └── spark │ │ │ └── LsqCliSparkUtils.java │ └── resources │ │ └── log4j2.properties │ └── test │ └── resources │ ├── lsq-example-config.ttl │ └── lsq-example-run.ttl ├── lsq-core ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── aksw │ │ │ └── simba │ │ │ └── lsq │ │ │ ├── core │ │ │ ├── LsqRdfizeSpec.java │ │ │ ├── LsqRdfizer.java │ │ │ ├── ResourceParser.java │ │ │ ├── SinkIO.java │ │ │ ├── io │ │ │ │ └── input │ │ │ │ │ └── registry │ │ │ │ │ └── LsqInputFormatRegistry.java │ │ │ ├── rx │ │ │ │ └── io │ │ │ │ │ └── input │ │ │ │ │ ├── LsqLogRecordRdfizer.java │ │ │ │ │ ├── LsqLogRecordRdfizerQueryOnly.java │ │ │ │ │ ├── LsqProbeUtils.java │ │ │ │ │ └── LsqRxIo.java │ │ │ └── util │ │ │ │ ├── SkolemizeBackport.java │ │ │ │ ├── SkolemizeExtra.java │ │ │ │ └── SpinUtils.java │ │ │ └── util │ │ │ ├── ConceptModelUtils.java │ │ │ ├── ElementVisitorFeatureExtractor.java │ │ │ ├── NestedResource.java │ │ │ └── PathVisitorFeatureExtractor.java │ ├── resources-filtered │ │ └── lsq-core.properties │ └── resources │ │ ├── lsq-invert-rdfized-log.sparql │ │ ├── lsq-prefixes.json │ │ ├── lsq-prefixes.ttl │ │ ├── lsq-slimify.sparql │ │ ├── lsq.shacl.ttl │ │ └── rdfa11.json │ └── test │ ├── java │ └── org │ │ └── aksw │ │ └── simba │ │ └── lsq │ │ ├── TestConnectionLoss.java │ │ ├── TestLsqFeatureExtraction.java │ │ ├── TestLsqHashId.java │ │ ├── TestLsqResourceViews.java │ │ ├── TestLsqTimeout.java │ │ ├── TestLsqValidationWithShacl.java │ │ ├── TestLsqWebServerAccessLogParser.java │ │ └── TestTripleRemoval.java │ └── resources │ ├── 2019-02-14-single-resource.lsq.ttl │ ├── broken-triple-removal-01.ttl │ ├── dbpedia.log-20100502 │ ├── log4j.properties │ ├── logs │ ├── bio2rdf.unknown.log │ ├── combined.swdf.log │ ├── combined.swdf.single-query.log │ ├── combined.swdf.single-triple-pattern.log │ ├── commonVirtuoso.dbpedia.20151025-1.log │ ├── commonVirtuoso.dbpedia.20160411-1000.log │ ├── sparql2.debug.log │ ├── sparql2.debug.ttl │ ├── sparql2.parseError.log │ ├── virtuoso.dbpedia.duplicates.log │ ├── virtuoso.dbpedia.log │ ├── virtuoso.dbpedia351.log │ └── wikidata.wikidata.20170612-10.log │ ├── lsq-expected-output-1.ttl │ ├── lsq-tests-sparql-cache │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-00b105817b1c47b78f23032aeffa91d4.dat │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-00b105817b1c47b78f23032aeffa91d4.sparql │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-112748535ddab11bda9a3ec76e01eebc.dat │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-112748535ddab11bda9a3ec76e01eebc.sparql │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-20f6ffc0970367f255dfbcced95cf0ed.dat │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-20f6ffc0970367f255dfbcced95cf0ed.sparql │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-33f876148694a00482188bf8c9ccd144.dat │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-33f876148694a00482188bf8c9ccd144.sparql │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-34d6fd2cf4bb93cc0d2a774af2823375.dat │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-34d6fd2cf4bb93cc0d2a774af2823375.sparql │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-3a4ad858f543f82daa50f02eb438c218.dat │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-3a4ad858f543f82daa50f02eb438c218.sparql │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-411d9d59eef750e34db25b9509148c67.dat │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-411d9d59eef750e34db25b9509148c67.sparql │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-49179a37bf1e93da3d5ada5aa8db0a79.dat │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-49179a37bf1e93da3d5ada5aa8db0a79.sparql │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-63abcdf4f59fb131f75f24175e60fa7b.dat │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-63abcdf4f59fb131f75f24175e60fa7b.sparql │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-71c172100429809f406cf0907383e679.dat │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-71c172100429809f406cf0907383e679.sparql │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-7c028cda1ee6d80a4a10a32b9ca03f19.dat │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-7c028cda1ee6d80a4a10a32b9ca03f19.sparql │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-af4137d91c3abc348243c2946207b7a2.dat │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-af4137d91c3abc348243c2946207b7a2.sparql │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-b5d6d24f3c3b57170beb4cf6a58e728c.dat │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-b5d6d24f3c3b57170beb4cf6a58e728c.sparql │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-ecc0934f84813da15414fa919fc59ec0.dat │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-ecc0934f84813da15414fa919fc59ec0.sparql │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-fcb889011c1f52dd453f34fb8c9fbbcf.dat │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-fcb889011c1f52dd453f34fb8c9fbbcf.sparql │ ├── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-fdcd3a7f5482a3b90bc3ab1a4afc1e95.dat │ └── http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-fdcd3a7f5482a3b90bc3ab1a4afc1e95.sparql │ ├── lsq-tests │ ├── 01 │ │ ├── data.ttl │ │ └── query.sparql.log │ └── triple-pattern-selectivity │ │ ├── tpsel01-old.ttl │ │ ├── tpsel01.sparql │ │ └── tpsel01.ttl │ ├── queries │ └── queries.txt │ ├── rdf-envs │ ├── example-common-envs.ttl │ ├── swdf-desc.ttl │ ├── trash.ttl │ └── virt.log │ ├── rehash │ ├── data.trig │ └── data2.trig │ ├── test-data.ttl │ ├── unsorted │ ├── bugsearch.log │ └── unknown.bio2rdf2.post.log │ └── watdiv-50-queries.txt ├── lsq-debian-cli ├── CHANGES.txt └── pom.xml ├── lsq-enrichers ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── aksw │ │ └── simba │ │ └── lsq │ │ └── enricher │ │ ├── benchmark │ │ ├── core │ │ │ ├── LsqBenchmarkProcessor.java │ │ │ └── LsqExec.java │ │ └── opcache │ │ │ ├── EvaluatorDispatchWithCaching.java │ │ │ ├── OpExtKey.java │ │ │ ├── OpExtKeyAndTableSupplier.java │ │ │ ├── OpExtTableToMultiset.java │ │ │ ├── TableMgr.java │ │ │ └── TableMultiset.java │ │ └── core │ │ ├── LsqEnricher.java │ │ ├── LsqEnricherFactory.java │ │ ├── LsqEnricherRegistry.java │ │ ├── LsqEnricherShell.java │ │ ├── LsqEnrichments.java │ │ ├── QueryStatistics2.java │ │ └── SpinAccessUtils.java │ └── test │ └── java │ └── org │ └── aksw │ └── simba │ └── lsq │ └── TestLsqModel.java ├── lsq-legacy ├── README.md ├── example-output.ttl ├── examples │ ├── lsq-example-swdf-output.nt │ ├── lsq-example-swdf-todo-add-dataset │ │ ├── lsq-example-swdf.nt │ │ ├── lsq-example-swdf.sparql │ │ └── run-lsq-example-swdf.sh │ └── lsqify-wikidata.sh ├── fixes │ ├── fix-datetime.sh │ └── fix-datetime.ttl ├── lsq-eval │ ├── endpoints.ttl │ ├── executions-per-agent.rq │ ├── hosts-per-endpoint.rq │ ├── query-count-per-host-count.rq │ ├── query-count-per-host.rq │ └── saleem-environment.ttl ├── pom.xml ├── run-example-from-deb.sh ├── run-example-from-jar.sh ├── run-example.sh ├── src │ └── main │ │ └── java │ │ └── org │ │ └── aksw │ │ └── simba │ │ └── lsq │ │ ├── TestLsqQueryExecutionTime.java │ │ ├── TestLsqQueryJoinType.java │ │ ├── TestLsqSelectivity.java │ │ ├── cli │ │ ├── cmd │ │ │ └── experimental │ │ │ │ ├── CmdLsqInvert.java │ │ │ │ ├── CmdLsqRehash.java │ │ │ │ └── CmdLsqSparkRehash.java │ │ └── trash │ │ │ ├── CmdLsqRehashSparkImpl.java │ │ │ ├── LsqCliConfig.java │ │ │ ├── LsqCliParser.java │ │ │ ├── LsqConfigImpl.java │ │ │ ├── LsqProcessor.java │ │ │ ├── MainCliLsqQueryAnalyzer.java │ │ │ ├── MainDeleteme.java │ │ │ └── MainLsqConsumeExample.java │ │ ├── trash │ │ ├── LSQARQ2SPIN.java │ │ └── upgrade │ │ │ └── v0_0_1_to_1_0_0 │ │ │ └── MainLsqUpgrade.java │ │ ├── unrelated │ │ └── filehasher │ │ │ └── PathHasherImpl.java │ │ └── util │ │ └── SpinUtilsOld.java └── test-lsq-cli.sh ├── lsq-model ├── pom.xml └── src │ └── main │ ├── java │ └── org │ │ └── aksw │ │ └── simba │ │ └── lsq │ │ ├── jena │ │ └── plugin │ │ │ └── JenaPluginLsq.java │ │ ├── model │ │ ├── ElementExec.java │ │ ├── ExperimentConfig.java │ │ ├── ExperimentExec.java │ │ ├── ExperimentRun.java │ │ ├── Host.java │ │ ├── JoinVertex.java │ │ ├── LocalExecution.java │ │ ├── LsqQuery.java │ │ ├── LsqStructuralFeatures.java │ │ ├── QualifiedFeature.java │ │ ├── QueryExec.java │ │ ├── RemoteExecution.java │ │ ├── RemoteExecutionHeaders.java │ │ └── util │ │ │ └── SpinCoreUtils.java │ │ └── spinx │ │ └── model │ │ ├── Bgp.java │ │ ├── BgpExec.java │ │ ├── BgpInfo.java │ │ ├── BgpNode.java │ │ ├── BgpNodeExec.java │ │ ├── DirectedHyperEdge.java │ │ ├── JoinVertex.java │ │ ├── Labeled.java │ │ ├── LsqElement.java │ │ ├── LsqTriplePattern.java │ │ ├── SpinQueryEx.java │ │ ├── SpinVarOrLiteral.java │ │ ├── TpExec.java │ │ ├── TpInBgp.java │ │ ├── TpInBgpExec.java │ │ └── TpInSubBgpExec.java │ └── resources │ └── META-INF │ └── services │ └── org.apache.jena.sys.JenaSubsystemLifecycle ├── lsq-parser ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── aksw │ │ │ └── simba │ │ │ └── lsq │ │ │ └── parser │ │ │ ├── ConverterChain.java │ │ │ ├── FieldSpec.java │ │ │ ├── FixMapper.java │ │ │ ├── Mapper.java │ │ │ ├── MapperConverter.java │ │ │ ├── MapperDelegate.java │ │ │ ├── NestedPropertyMapper.java │ │ │ ├── PatternMatcher.java │ │ │ ├── PatternMatcherImpl.java │ │ │ ├── PropertyMapper.java │ │ │ ├── RDFDatatypeDateFormat.java │ │ │ ├── RDFDatatypeRestricted.java │ │ │ ├── StringMapper.java │ │ │ ├── WebLogParser.java │ │ │ └── csv │ │ │ └── CsvParser.java │ └── resources │ │ └── default-log-formats.ttl │ └── test │ └── java │ └── org │ └── aksw │ └── simba │ └── lsq │ └── parser │ └── TestWebLogParser.java ├── lsq-pkg-parent ├── lsq-pkg-deb-cli │ ├── .gitignore │ ├── pom.xml │ └── src │ │ └── deb │ │ ├── control │ │ ├── control │ │ └── copyright │ │ └── resources │ │ └── usr │ │ └── bin │ │ └── lsq ├── lsq-pkg-docker-cli │ └── pom.xml ├── lsq-pkg-dummy │ └── pom.xml ├── lsq-pkg-rpm-cli │ ├── pom.xml │ └── src │ │ └── rpm │ │ └── resources │ │ └── usr │ │ └── bin │ │ └── lsq ├── lsq-pkg-uberjar-cli │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── LsqUberJarDummy.java └── pom.xml ├── lsq-spark-core ├── pom.xml └── src │ └── main │ └── java │ ├── net │ └── sansa_stack │ │ └── rdf │ │ └── spark │ │ └── io │ │ ├── LsqRegistrySparkAdapter.java │ │ ├── LsqSparkIo.java │ │ ├── LsqSparkUtils.java │ │ ├── SourceOfRddOfResources.java │ │ └── csv │ │ └── CsvParserSpark.java │ └── org │ └── aksw │ └── simba │ └── lsq │ ├── rdf │ └── conversion │ │ ├── DatasetGraphOpsStream.java │ │ ├── GraphOpsStream.java │ │ └── StreamTransformer.java │ └── spark │ └── cmd │ └── impl │ ├── CmdRdfIo.java │ └── LsqTarqlTest.java ├── lsq-vocab-jena ├── pom.xml └── src │ └── main │ └── java │ └── org │ └── aksw │ └── simba │ └── lsq │ └── vocab │ ├── LSQ.java │ └── PROV.java ├── pom.xml └── reinstall-deb.sh /.github/workflows/publish-docker.yml: -------------------------------------------------------------------------------- 1 | name: Publish Docker image 2 | # To AKSW GitHub Container Registry 3 | # https://github.com/orgs/AKSW/packages 4 | on: 5 | workflow_dispatch: 6 | push: 7 | # Publish `develop` as Docker `latest` image. 8 | branches: 9 | - develop 10 | # Publish `v1.2.3` tags as releases. 11 | tags: 12 | - v* 13 | release: 14 | types: 15 | - created 16 | 17 | env: 18 | IMAGE_NAME: lsq 19 | 20 | jobs: 21 | build-publish: 22 | runs-on: ubuntu-latest 23 | 24 | steps: 25 | - uses: actions/checkout@v2 26 | 27 | - name: Build image 28 | run: docker build . --file Dockerfile --tag $IMAGE_NAME 29 | 30 | - name: Log into GitHub Container Registry 31 | run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin 32 | 33 | - name: Push image to GitHub Container Registry 34 | run: | 35 | IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME 36 | 37 | # Change all uppercase to lowercase 38 | IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') 39 | 40 | # Strip git ref prefix from version 41 | VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') 42 | 43 | # Strip "v" prefix from tag name 44 | [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') 45 | 46 | # Use Docker `latest` tag convention 47 | [ "$VERSION" == "develop" ] && VERSION=latest 48 | 49 | echo IMAGE_ID=$IMAGE_ID 50 | echo VERSION=$VERSION 51 | 52 | docker tag $IMAGE_NAME $IMAGE_ID:$VERSION 53 | docker push $IMAGE_ID:$VERSION 54 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .buildpath 2 | .classpath 3 | .project 4 | .settings 5 | target 6 | 7 | *.jar 8 | 9 | pom.xml.versionBackup 10 | pom.xml.versionsBackup 11 | 12 | deptree.txt 13 | LSQ-SWDF-test.ttl 14 | 15 | hs_err* 16 | *.releaseBackup 17 | outfile.ttl 18 | out.ttl 19 | 20 | # some file names for testing 21 | a.nt 22 | a.ttl 23 | b.nt 24 | b.ttl 25 | c.nt 26 | c.ttl 27 | d.nt 28 | d.ttl 29 | outfile.ttl 30 | deptree.txt 31 | 32 | trash 33 | *.swp 34 | 35 | tmp 36 | 37 | derby.log 38 | 39 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # IMPORTANT: This dockerfile is merely a fallback! 2 | # The recommended way to build a docker image from this java project is using the jib maven plugin 3 | # on the module lsq-pkg-docker-cli: 4 | # mvn clean install 5 | # mvn -pl :lsq-pkg-docker-cli jib:dockerBuild 6 | 7 | # Building this Dockerfile requires buildkit: 8 | # Ensure { "features": { "buildkit": true } } exists in /etc/docker/daemon.json 9 | # (or wherever your deamon.json resides) 10 | 11 | ARG home="/lsq" 12 | 13 | FROM maven:3-jdk-11 as build 14 | ARG home 15 | ENV HOME "$home" 16 | RUN mkdir -p "$HOME" 17 | WORKDIR "$HOME" 18 | ADD . "$HOME" 19 | RUN --mount=type=cache,target=/root/.m2 mvn -Pdist,standalone clean install 20 | 21 | # Final running image 22 | FROM openjdk:11-jre-slim 23 | ARG home 24 | ENV HOME "$home" 25 | # Import the lsq-cli jar from the build step 26 | COPY --from=build "$HOME/lsq-pkg-parent/lsq-pkg-uberjar-cli/target/"lsq-pkg-uberjar-cli-*-jar-with-dependencies.jar "$HOME/lsq-cli.jar" 27 | 28 | # Using /data as working directory that will be shared with host for input/output files 29 | WORKDIR /data 30 | VOLUME [ "/data" ] 31 | 32 | ENTRYPOINT ["java","-jar","$HOME/lsq-cli.jar"] 33 | CMD ["-h"] 34 | 35 | # Usage: 36 | # docker run -it -v $(pwd):/data ghcr.io/aksw/lsq rx rdfize --endpoint=http://dbpedia.org/sparql virtuoso.dbpedia.log 37 | 38 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Linked SPARQL Queries (LSQ) Framework 2 | A framework for RDFizing query logs and benchmarking queries and graph patterns. 3 | 4 | ## What's New in LSQ V2 5 | 6 | ### 2020-08-06 LSQ2 Pre-release 7 | LSQ2 introduces significant improvements over the prior version in every aspect: Ease-of-use, flexibility, modularity, consintency in the data model and generated IDs. 8 | 9 | * Pretty CLI (thanks to on [picocli](https://github.com/remkop/picocli)) 10 | * Easier yet more flexible to use: RDFization, static analysis and benchmarking now decoupled 11 | * Named graph stream approach: Information for each query is grouped in its own named graph which allows easily selecting subsets with complete information for detailed analysis. 12 | 13 | 14 | ## Documentation 15 | 16 | ### Detailed Documentation 17 | For detailed documentation about setup, use and concepts of the LSQ command line tool please refer to our [LSQ Website](http://lsq.aksw.org/). 18 | 19 | ### Quick Reference 20 | 21 | #### Setup 22 | 23 | This is a typical maven project and can is thus built with `mvn clean install`. 24 | 25 | **For Ubuntu/Debian users:** The build process creates a `.deb` package that can be conviently installed **after build** with 26 | 27 | `./reinstall-deb.sh` (requires root access). 28 | 29 | 30 | #### Quick Usage 31 | A quick reference for the typical process is as follows: 32 | 33 | ``` 34 | lsq rx probe file.log 35 | lsq rx rdfize -e http://server.from/which/the/log/is/from file.log > file.log.trig 36 | lsq rx benchmark create -d myDatasetLabel -e http://localhost:8890/sparql -o > benchmark.conf.ttl 37 | lsq rx benchmark prepare -c benchmark.conf.ttl -o > benchmark.run.ttl 38 | lsq rx benchmark run -c benchmark.run.ttl *.log.trig 39 | ``` 40 | 41 | The `-o` option causes the settings to be written to the console. Omit `-o` to have LSQ auto-generate files. 42 | 43 | 44 | #### Run with Docker 45 | 46 | Run example running LSQ to RDFize SPARQL logs, input and output files in the current working directory (replace `$(pwd)` by `${PWD}` for Windows PowerShell): 47 | 48 | ```bash 49 | docker run -it -v $(pwd):/data ghcr.io/aksw/lsq rx rdfize --endpoint=http://dbpedia.org/sparql virtuoso.dbpedia.log 50 | ``` 51 | 52 | Build the Docker image from the source code: 53 | 54 | ```bash 55 | docker build -t ghcr.io/aksw/lsq . 56 | ``` 57 | 58 | 59 | ## License 60 | The source code of this repo is published under the [Apache License Version 2.0](https://github.com/AKSW/jena-sparql-api/blob/master/LICENSE). 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /docker-from-uberjar/Dockerfile: -------------------------------------------------------------------------------- 1 | # IMPORTANT: This dockerfile is merely a fallback! 2 | # The recommended way to build a docker image from this java project is using the jib maven plugin 3 | # on the module lsq-pkg-docker-cli: 4 | # mvn clean install 5 | # mvn -pl :lsq-pkg-docker-cli jib:dockerBuild 6 | 7 | # Building this Dockerfile requires buildkit: 8 | # Ensure { "features": { "buildkit": true } } exists in /etc/docker/daemon.json 9 | # (or wherever your deamon.json resides) 10 | 11 | ARG home="/lsq" 12 | 13 | # Final running image 14 | FROM openjdk:11-jre-slim 15 | ARG home 16 | ENV HOME "$home" 17 | # Import the lsq-cli jar from the build step 18 | ADD lsq-pkg-uberjar-cli-*-jar-with-dependencies.jar "$HOME/lsq-cli.jar" 19 | 20 | # Using /data as working directory that will be shared with host for input/output files 21 | WORKDIR /data 22 | VOLUME [ "/data" ] 23 | 24 | # XXX $HOME Does not get expanded in the entry point 25 | ENTRYPOINT ["java","-jar","/lsq/lsq-cli.jar"] 26 | CMD ["-h"] 27 | 28 | # Usage: 29 | # docker run -it -v $(pwd):/data ghcr.io/aksw/lsq rx rdfize --endpoint=http://dbpedia.org/sparql virtuoso.dbpedia.log 30 | 31 | -------------------------------------------------------------------------------- /docker-from-uberjar/README.md: -------------------------------------------------------------------------------- 1 | Helper docker file to build the image from an uberjar and thus avoid a full build. 2 | 3 | For example, this git repository might be checked out on a remote server with only docker available. 4 | In this case, the lsq jar could be built locally and copied to the remote server using `scp`. 5 | 6 | The filename must match the pattern: `lsq-pkg-uberjar-cli-*-jar-with-dependencies.jar` 7 | This is also the file generated when running `make distjar` on the project root. 8 | 9 | Once the uberjar is present, run `docker build .` to build the image. 10 | 11 | -------------------------------------------------------------------------------- /docker-from-uberjar/copy-uberjar.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # In the project root, run "make distjar" to build the uberjar 4 | 5 | cp ../lsq-pkg-parent/lsq-pkg-uberjar-cli/target/lsq-pkg-uberjar-cli-*-jar-with-dependencies.jar . 6 | 7 | -------------------------------------------------------------------------------- /docs/CNAME: -------------------------------------------------------------------------------- 1 | lsq.aksw.org -------------------------------------------------------------------------------- /docs/_config.yml: -------------------------------------------------------------------------------- 1 | title: "Linked SPARQL Queries (LSQ)" 2 | remote_theme: pmarsceill/just-the-docs 3 | search_enabled: true 4 | logo: "assets/images/lsq-logo.png" 5 | markdown: kramdown 6 | kramdown: 7 | parse_block_html: true 8 | 9 | 10 | exclude: 11 | - lsq* 12 | 13 | # Footer last edited timestamp 14 | last_edit_timestamp: true # show or hide edit time - page must have `last_modified_date` defined in the frontmatter 15 | last_edit_time_format: "%b %e %Y at %I:%M %p" # uses ruby's time format: https://ruby-doc.org/stdlib-2.7.0/libdoc/time/rdoc/Time.html 16 | 17 | # Footer "Edit this page on GitHub" link text 18 | gh_edit_link: true # show or hide edit this page link 19 | gh_edit_link_text: "Edit this page on GitHub." 20 | gh_edit_repository: "https://github.com/AKSW/LSQ" # the github URL for your repo 21 | gh_edit_branch: "develop" # the branch that your docs is served from 22 | gh_edit_source: "docs" # the source that your files originate from 23 | gh_edit_view_mode: "tree" # "tree" or "edit" if you want the user to jump into the editor immediately 24 | 25 | -------------------------------------------------------------------------------- /docs/legacy.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Legacy 3 | #has_children: true 4 | nav_order: 7 5 | --- 6 | 7 | * [LSQ V1](v1/index.html) 8 | 9 | -------------------------------------------------------------------------------- /docs/v1/images/background.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/background.png -------------------------------------------------------------------------------- /docs/v1/images/bg-ramp.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/bg-ramp.jpg -------------------------------------------------------------------------------- /docs/v1/images/bg_hr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/bg_hr.png -------------------------------------------------------------------------------- /docs/v1/images/blacktocat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/blacktocat.png -------------------------------------------------------------------------------- /docs/v1/images/blockquote-gfx-2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/blockquote-gfx-2x.png -------------------------------------------------------------------------------- /docs/v1/images/blockquote-gfx.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/blockquote-gfx.png -------------------------------------------------------------------------------- /docs/v1/images/body-background.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/body-background.png -------------------------------------------------------------------------------- /docs/v1/images/body-bg.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/body-bg.jpg -------------------------------------------------------------------------------- /docs/v1/images/bullet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/bullet.png -------------------------------------------------------------------------------- /docs/v1/images/chevron-2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/chevron-2x.png -------------------------------------------------------------------------------- /docs/v1/images/chevron.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/chevron.png -------------------------------------------------------------------------------- /docs/v1/images/download-button.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/download-button.png -------------------------------------------------------------------------------- /docs/v1/images/download-fallback-bg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/download-fallback-bg.png -------------------------------------------------------------------------------- /docs/v1/images/download-sprite.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/download-sprite.png -------------------------------------------------------------------------------- /docs/v1/images/footer-ramp.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/footer-ramp.jpg -------------------------------------------------------------------------------- /docs/v1/images/fork-sprite.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/fork-sprite.png -------------------------------------------------------------------------------- /docs/v1/images/github-button.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/github-button.png -------------------------------------------------------------------------------- /docs/v1/images/header-bg.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/header-bg.jpg -------------------------------------------------------------------------------- /docs/v1/images/highlight-bg.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/highlight-bg.jpg -------------------------------------------------------------------------------- /docs/v1/images/hr-2x.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/hr-2x.jpg -------------------------------------------------------------------------------- /docs/v1/images/hr.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/hr.gif -------------------------------------------------------------------------------- /docs/v1/images/hr.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/hr.jpg -------------------------------------------------------------------------------- /docs/v1/images/icon_download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/icon_download.png -------------------------------------------------------------------------------- /docs/v1/images/octocat-2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/octocat-2x.png -------------------------------------------------------------------------------- /docs/v1/images/octocat-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/octocat-logo.png -------------------------------------------------------------------------------- /docs/v1/images/octocat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/octocat.png -------------------------------------------------------------------------------- /docs/v1/images/ribbon-tail-sprite-2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/ribbon-tail-sprite-2x.png -------------------------------------------------------------------------------- /docs/v1/images/ribbon-tail-sprite.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/ribbon-tail-sprite.png -------------------------------------------------------------------------------- /docs/v1/images/shield-fallback.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/shield-fallback.png -------------------------------------------------------------------------------- /docs/v1/images/shield.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/shield.png -------------------------------------------------------------------------------- /docs/v1/images/sidebar-bg.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/sidebar-bg.jpg -------------------------------------------------------------------------------- /docs/v1/images/site-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/site-2.png -------------------------------------------------------------------------------- /docs/v1/images/small-ribbon-tail-sprite-2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/small-ribbon-tail-sprite-2x.png -------------------------------------------------------------------------------- /docs/v1/images/small-ribbon-tail-sprite.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/small-ribbon-tail-sprite.png -------------------------------------------------------------------------------- /docs/v1/images/sprite_download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/images/sprite_download.png -------------------------------------------------------------------------------- /docs/v1/lsq-images/lsq-vocab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v1/lsq-images/lsq-vocab.png -------------------------------------------------------------------------------- /docs/v1/stylesheets/core.css: -------------------------------------------------------------------------------- 1 | @import url("screen.css"); 2 | @import url("non-screen.css") handheld; 3 | @import url("non-screen.css") only screen and (max-device-width:640px); -------------------------------------------------------------------------------- /docs/v2/Erroneous-Queries.md: -------------------------------------------------------------------------------- 1 | This is a collection of queries with various errors. It is useful for further improvement of the test suite and LSQ2 as a whole. 2 | 3 | :warning: LSQ2 confusingly yields 'internal error; continuing anyway' for the following type of error: 4 | ```sparql 5 | SELECT DISTINCT ?var1 ?var1Label ?var2 ( SAMPLE ( ?var3 ) AS ?var3 ) ?var4 6 | WHERE { 7 | ?var1 . 8 | OPTIONAL { 9 | ?var1 ?var3 . 10 | } 11 | OPTIONAL { 12 | ?var4 ?var1 . 13 | ?var4 . 14 | } 15 | SERVICE { 16 | "[AUTO_LANGUAGE],en,fr,es,de,ru,it,nl,ja,zh,pl,cs". 17 | } 18 | } 19 | GROUP BY ?var1 ?var1Label ?var2 ?var4 20 | LIMIT 50 21 | ``` 22 | 23 | ``` 24 | org.apache.jena.query.QueryParseException: Variable used when already in-scope: ?var3 in ((AGG ?.0 SAMPLE(?var3)) AS ?var3) 25 | ``` 26 | -------------------------------------------------------------------------------- /docs/v2/concepts/concepts.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Concepts 3 | has_children: true 4 | nav_order: 6 5 | --- 6 | 7 | -------------------------------------------------------------------------------- /docs/v2/concepts/data-model.md: -------------------------------------------------------------------------------- 1 | --- 2 | parent: Concepts 3 | title: Data Model 4 | nav_order: 1 5 | --- 6 | 7 | 8 | # LSQ Data Model 9 | 10 | **This is the data model of version 2 of LSQ** 11 | 12 | ![Depiction of the LSQ2 Data Model](https://raw.githubusercontent.com/AKSW/LSQ/develop/docs/v2/images/lsq2-datamodel.png) 13 | -------------------------------------------------------------------------------- /docs/v2/concepts/named-graph-streams.md: -------------------------------------------------------------------------------- 1 | --- 2 | parent: Concepts 3 | title: Named Graph Streams 4 | nav_order: 3 5 | --- 6 | 7 | 8 | ngs 9 | -------------------------------------------------------------------------------- /docs/v2/development/development.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Development 3 | has_children: true 4 | nav_order: 7 5 | --- 6 | 7 | -------------------------------------------------------------------------------- /docs/v2/development/modules.md: -------------------------------------------------------------------------------- 1 | --- 2 | parent: Development 3 | title: Modules 4 | nav_order: 1 5 | --- 6 | 7 | 8 | # Modules of LSQ 9 | 10 | LSQ consists of the following modules, ordered from lower to higher level. 11 | 12 | * [lsq-vocab-jena](https://github.com/AKSW/LSQ/tree/develop/lsq-vocab-jena): The vocabulary of lsq for use with [Apache Jena](https://jena.apache.org/). 13 | * [lsq-model](https://github.com/AKSW/LSQ/tree/develop/lsq-model): The LSQ domain classes. Useful for working with existing LSQ models from Java. 14 | * [lsq-web-log-parser](https://github.com/AKSW/LSQ/tree/develop/lsq-parser): Parser and RDFizer for web log files. 15 | * [lsq-core](https://github.com/AKSW/LSQ/tree/develop/lsq-core): The core of LSQ, most prominently the RDFizer. 16 | * [lsq-enrichers](https://github.com/AKSW/LSQ/tree/develop/lsq-enrichers): Classes for enriching LSQ models and benchmarking SPARQL queries. 17 | * [lsq-spark-core](https://github.com/AKSW/LSQ/tree/develop/lsq-spark-core): Binding of LSQ for [Apache Spark](https://spark.apache.org/). 18 | * [lsq-cli](https://github.com/AKSW/LSQ/tree/develop/lsq-cli): [Picocli](https://picocli.info/)-based Command line interface (cli) for LSQ. 19 | * [lsq-debian-cli](https://github.com/AKSW/LSQ/tree/develop/lsq-debian-cli): Debian packaging of LSQ's cli. 20 | 21 | -------------------------------------------------------------------------------- /docs/v2/images/lsq2-datamodel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AKSW/LSQ/8b1a01fa9728c155f447a528fc77d0771715763c/docs/v2/images/lsq2-datamodel.png -------------------------------------------------------------------------------- /docs/v2/news/2020-08-13.md: -------------------------------------------------------------------------------- 1 | --- 2 | parent: News 3 | title: 2020-08-13 LSQ2 Pre-Release 4 | nav_order: 1 5 | --- 6 | 7 | ### Pre-Release Announcement 2020-08-13 8 | 9 | ANN: Linked SPARQL Queries (LSQ) Version 2.0 Software Pre-Release 10 | 11 | We are happy to announce a completely revamped software framework for RDFizing and benchmarking SPARQL queries from 12 | various Web servers log formats. 13 | 14 | * Automatic log format probing 15 | * Easy-to-use command line interface 16 | * Java API to read/write the LSQ RDF model 17 | * Greatly improved LSQ data model (not compatible with the prior version) 18 | * Greatly simplified log RDFization 19 | * Greatly improved benchmarking process; added new configuration options to control scalability aspects 20 | * Greatly improved documentation (https://github.com/AKSW/LSQ/wiki/) 21 | * Improved hashing to reduce chance of collisions (w.r.t. to prior version) 22 | 23 | Comments, suggestions and (constructive) criticism is always welcome :) 24 | 25 | Best regards, 26 | 27 | Mohammed Saleem and Claus Stadler 28 | 29 | On behalf of 30 | * Institute of Applied Informatics (InfAI)- http://infai.org/ 31 | * SmartDataAnalytics (SDA) - https://sda.tech/ 32 | * Data Science Group (DICE) - https://dice-research.org/ 33 | 34 | -------------------------------------------------------------------------------- /docs/v2/news/news.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: News 3 | has_children: true 4 | nav_order: 2 5 | --- 6 | 7 | -------------------------------------------------------------------------------- /docs/v2/resources/downloads/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | parent: Resources 3 | title: Downloads 4 | has_children: false 5 | nav_order: 1 6 | --- 7 | 8 | The LSQ ontology file is available at: 9 | 10 | * [http://lsq.aksw.org/vocab](http://lsq.aksw.org/vocab) 11 | 12 | LSQ datasets can be downloaded from the following file server: 13 | 14 | * [https://hobbitdata.informatik.uni-leipzig.de/lsqv2/dumps/](https://hobbitdata.informatik.uni-leipzig.de/lsqv2/dumps/) 15 | 16 | -------------------------------------------------------------------------------- /docs/v2/resources/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Resources 3 | has_children: true 4 | nav_order: 3 5 | --- 6 | 7 | -------------------------------------------------------------------------------- /docs/v2/resources/sparql-endpoint.md: -------------------------------------------------------------------------------- 1 | --- 2 | parent: Resources 3 | title: Sparql Endpoint 4 | has_children: false 5 | nav_order: 2 6 | --- 7 | 8 | LSQ can be queried online at the following public SPARQL endpoint: 9 | 10 | * [https://lsq.data.dice-research.org/sparql](https://lsq.data.dice-research.org/sparql) 11 | 12 | -------------------------------------------------------------------------------- /docs/v2/setup.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Setup 3 | has_children: false 4 | nav_order: 4 5 | --- 6 | 7 | ## Building from Source 8 | 9 | LSQ requires Maven to build. 10 | 11 | If you want a self-contained runnable jar then build lsq using: 12 | ```bash 13 | mvn -Pdist,standalone clean install 14 | ``` 15 | 16 | In order to reduce build times for specific tasks there exist different profiles. 17 | The following table summarizes those profiles: 18 | 19 | 20 | | Build task | Required profile(s) | Example | 21 | |------------------------------------------|---------------------|---------------------------------------| 22 | | Libraries only | (none) | `mvn clean install` | 23 | | Jar bundle (uber jar) | dist | `mvn -Pdist clean install` | 24 | | Embed Apache Spark | standalone | `mvn -Pdist,standalone clean install` | 25 | | Debian package (implies dist,standalone) | deb | `mvn -Pdeb clean install` | 26 | 27 | ### Building the Jar Bundle (jar-with-dependencies) 28 | 29 | The jar bundle contains a main class manifest and can thus be run with `java -jar`: 30 | ```bash 31 | mvn -Pdist,standalone clean install 32 | 33 | java -jar lsq-cli/target/lsq-cli-${VERSON}-jar-with-dependencies.jar 34 | ``` 35 | (Don't forget to substitute `${VERSION}` with its proper value) 36 | 37 | 38 | ### Building, Installing and Running the Debian Package 39 | 40 | Installing the debian package makes the `lsq` command available (located at `/usr/bin/lsq`). 41 | 42 | 43 | * You may first want to completely remove (i.e. purge) a prior installation 44 | ```bash 45 | sudo apt-get purge lsq-cli 46 | ``` 47 | 48 | * The debian package is built with the `deb` profile 49 | ```bash 50 | mvn -Pdeb clean install 51 | ``` 52 | 53 | * Install the deb package 54 | The convenience script `./reinstall-deb.sh` searches for generated debian packages and installs them. 55 | It assumes that only a single built package exists. 56 | 57 | ```bash 58 | ./reinstall-deb.sh 59 | 60 | ``` 61 | 62 | The script is a shorthand for: 63 | 64 | ``` 65 | sudo dpkg -i `find 'lsq-debian-cli/target/' -name 'lsq-cli_*.deb'` 66 | ``` 67 | 68 | 69 | -------------------------------------------------------------------------------- /docs/v2/usage/probing-logs.md: -------------------------------------------------------------------------------- 1 | --- 2 | parent: Usage 3 | title: Probing Log Files 4 | nav_order: 20 5 | --- 6 | 7 | Probing is performed with the `lsq probe` subcommand. Multiple files can be processed at once using globbing. 8 | 9 | It attempts to automatically detect the format of an input log in order to recognize the semantics of the contained records. 10 | By default, up to the first 1000 lines of a log file are parsed against all known log formats. The format for which the highest score could be obtained wins. 11 | 12 | In detail, for a given sample of _n_ lines, the score _s_ is `s := parseRatio * avgNumberOfImmediatePredicates` with 13 | `parseRatio = number of successfully parsed lines / n` and avgNumberOfImmediatePredicates is the average number of RDF predicates obtained from successfully parsed lines with the log format being probed. 14 | 15 | The following is the output on the test resources of the lsq project, and at the time of writing all resources were correctly classified: 16 | 17 | ``` 18 | ➜ lsq probe lsq-core/src/test/resources/logs/* 19 | lsq-core/src/test/resources/logs/bio2rdf.unknown.log [bio2rdf=7.0, sparql2=2.0] 20 | lsq-core/src/test/resources/logs/combined.swdf.log [combined=10.0, common=9.0, distributed=9.0, bio2rdf=7.0, sparql2=2.0] 21 | lsq-core/src/test/resources/logs/combined.swdf.single-query.log [combined=10.0, common=9.0, distributed=9.0, bio2rdf=7.0, sparql2=2.0] 22 | lsq-core/src/test/resources/logs/combined.swdf.single-triple-pattern.log [combined=10.0, common=9.0, distributed=9.0, bio2rdf=7.0, sparql2=2.0] 23 | lsq-core/src/test/resources/logs/commonVirtuoso.dbpedia.20151025-1.log [commonVirtuoso=9.0, virtuoso=5.0, sparql2=2.0] 24 | lsq-core/src/test/resources/logs/commonVirtuoso.dbpedia.20160411-1000.log [commonVirtuoso=9.0, virtuoso=5.036, sparql2=2.0] 25 | lsq-core/src/test/resources/logs/sparql2.debug.log [sparql2=2.0, sparql=1.0] 26 | lsq-core/src/test/resources/logs/sparql2.debug.ttl [sparql2=2.0] 27 | lsq-core/src/test/resources/logs/virtuoso.dbpedia351.log [virtuoso=6.0, sparql2=2.0] 28 | lsq-core/src/test/resources/logs/virtuoso.dbpedia.log [virtuoso=6.0, sparql2=2.0] 29 | lsq-core/src/test/resources/logs/wikidata.wikidata.20170612-10.log [wikidata=4.0, sparql2=2.0] 30 | ``` 31 | 32 | LSQ's database of default known log formats is stored in the file [default-log-formats.ttl](https://github.com/AKSW/LSQ/blob/master/lsq-parser/src/main/resources/default-log-formats.ttl). 33 | The patterns are processed by the [lsq-parser](https://github.com/AKSW/LSQ/blob/master/lsq-parser/) module which implements a subset of Apache's [mod_log_config options](http://httpd.apache.org/docs/current/mod/mod_log_config.html). 34 | -------------------------------------------------------------------------------- /docs/v2/usage/processing-lsq-data.md: -------------------------------------------------------------------------------- 1 | --- 2 | parent: Usage 3 | title: Processing Lsq Data 4 | nav_order: 5 5 | --- 6 | 7 | 8 | How to work with LSQ ; sparql-integrate tool (sit) and named graph streams (ngs) 9 | -------------------------------------------------------------------------------- /docs/v2/usage/rx-vs-spark.md: -------------------------------------------------------------------------------- 1 | --- 2 | parent: Usage 3 | title: Rx vs Spark 4 | nav_order: 1 5 | --- 6 | 7 | # Rx vs Spark 8 | 9 | Several LSQ commands come in two flavours: `rx` and `spark`. 10 | [RxJava](https://github.com/ReactiveX/RxJava) is a framework for building workflows over streaming data. 11 | [Apache Spark](https://spark.apache.org/) is framework for processing large amounts of (input) data in parallel, both locally and on a cluster. 12 | It is possible to (re-)use rx-based functions within spark's `rdd.mapPartition` feature - and this is what `lsq spark` does. 13 | 14 | Concretely, LSQ builds on [SANSA-Stack](https://github.com/SANSA-Stack/SANSA-Stack) which provides a foundation for working with RDF/SPARQL in Apache Spark. 15 | 16 | 17 | ## Differences 18 | * `rx` commands are more lightweight and support immediate streaming from stdin (see the note on sorting below). For small input data they may perform faster because there is no overhead in initializing spark. 19 | * `spark` commands can read input sources in parallel and thus **significantly** outperform plain `rx` ones on larger input data. 20 | 21 | ### A Note on Sorting 22 | `lsq {spark|rx} rdfize` by default tries inverts log entries into a query centric named graphs and then sorts and merges them. 23 | For this task the `rx` implementation used to rely on platform-specific sorting via `/usr/bin/sort` and will be updated to use a cross-platform Spark-based implementation. 24 | -------------------------------------------------------------------------------- /docs/v2/usage/usage.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Usage 3 | has_children: true 4 | nav_order: 4 5 | --- 6 | 7 | -------------------------------------------------------------------------------- /lsq-cli/src/main/java/org/aksw/simba/lsq/cli/cmd/base/CmdLsqAnalyzeBase.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.cli.cmd.base; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Iterator; 5 | import java.util.List; 6 | 7 | import org.aksw.simba.lsq.cli.main.MainCliLsq; 8 | import org.aksw.simba.lsq.enricher.core.LsqEnricherRegistry; 9 | 10 | import picocli.CommandLine.ArgGroup; 11 | import picocli.CommandLine.Option; 12 | import picocli.CommandLine.Parameters; 13 | 14 | public class CmdLsqAnalyzeBase { 15 | @Option(names = { "-h", "--help" }, usageHelp = true) 16 | public boolean help = false; 17 | 18 | @ArgGroup(exclusive = true, multiplicity = "0..1") 19 | public EnricherSpec enricherSpec = new EnricherSpec(); 20 | 21 | public static class EnricherSpec { 22 | @Option(names = { "--only" }, completionCandidates = CompletionCandidatesEnrichers.class) 23 | public List inclusions = null; 24 | 25 | @Option(names = { "--exclude" }, completionCandidates = CompletionCandidatesEnrichers.class) 26 | public List exclusions = null; 27 | 28 | public boolean isWhitelist() { 29 | return inclusions != null; 30 | } 31 | 32 | public List getRawList() { 33 | return inclusions != null 34 | ? inclusions 35 | : exclusions != null 36 | ? exclusions 37 | : List.of(); 38 | } 39 | 40 | public List getEffectiveList() { 41 | List rawList = getRawList(); 42 | boolean isWhitelist = isWhitelist(); 43 | List result = MainCliLsq.effectiveList(rawList, isWhitelist, new ArrayList<>(LsqEnricherRegistry.get().getKeys())); 44 | return result; 45 | } 46 | } 47 | 48 | public static class CompletionCandidatesEnrichers 49 | implements Iterable { 50 | @Override 51 | public Iterator iterator() { 52 | return LsqEnricherRegistry.get().getKeys().iterator(); 53 | } 54 | } 55 | 56 | @Parameters(arity = "1..*", description = "file-list to probe") 57 | public List nonOptionArgs = new ArrayList<>(); 58 | } 59 | -------------------------------------------------------------------------------- /lsq-cli/src/main/java/org/aksw/simba/lsq/cli/cmd/base/CmdLsqMain.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.cli.cmd.base; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | import java.util.Properties; 6 | 7 | import org.aksw.commons.picocli.VersionProviderFromClasspathProperties; 8 | import org.aksw.simba.lsq.cli.cmd.rx.api.CmdLsqRxParent; 9 | import org.aksw.simba.lsq.cli.cmd.spark.api.CmdLsqSparkParent; 10 | 11 | import picocli.CommandLine.Command; 12 | import picocli.CommandLine.Option; 13 | 14 | 15 | @Command(name="lsq", version="LSQ version", versionProvider = CmdLsqMain.LsqVersionProvider.class, description = "LSQ Subcommands", subcommands = { 16 | CmdLsqRxParent.class, 17 | CmdLsqSparkParent.class 18 | // CmdLsqSparkRehash.class 19 | }) 20 | public class CmdLsqMain { 21 | public static class LsqVersionProvider extends VersionProviderFromClasspathProperties { 22 | public @Override String getResourceName() { return "lsq-core.properties"; } 23 | public @Override List getStrings(Properties p) { return Arrays.asList(p.get("lsq-core.version") + " built at " + p.get("lsq-core.build.timestamp")); } 24 | } 25 | 26 | @Option(names={"-h", "--help"}, usageHelp = true, description = "Show general help") 27 | public boolean help = false; 28 | 29 | @Option(names={"-v", "--version"}, versionHelp = true, description = "Show version") 30 | public boolean version = false; 31 | 32 | } 33 | -------------------------------------------------------------------------------- /lsq-cli/src/main/java/org/aksw/simba/lsq/cli/cmd/base/CmdOutputSpecBase.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.cli.cmd.base; 2 | 3 | import picocli.CommandLine.Option; 4 | 5 | public class CmdOutputSpecBase { 6 | @Option(names = { "-o", "--out-format" }, description = "Output format", defaultValue = "trig/blocks") 7 | public String outFormat = null; 8 | 9 | @Option(names = { "-f", "--out-file" }, description = "Output file") 10 | public String outFile = null; 11 | } 12 | -------------------------------------------------------------------------------- /lsq-cli/src/main/java/org/aksw/simba/lsq/cli/cmd/rx/api/CmdLsqRxAnalyze.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.cli.cmd.rx.api; 2 | 3 | import java.util.concurrent.Callable; 4 | 5 | import org.aksw.simba.lsq.cli.cmd.base.CmdLsqAnalyzeBase; 6 | import org.aksw.simba.lsq.cli.main.MainCliLsq; 7 | 8 | import picocli.CommandLine.Command; 9 | 10 | /** 11 | * Static analysis of queries 12 | * 13 | * @author raven 14 | * 15 | */ 16 | @Command(name = "analyze", description = "Analyze queries and emit structural features") 17 | public class CmdLsqRxAnalyze 18 | extends CmdLsqAnalyzeBase 19 | implements Callable 20 | { 21 | @Override 22 | public Integer call() throws Exception { 23 | MainCliLsq.analyze(this); 24 | return 0; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /lsq-cli/src/main/java/org/aksw/simba/lsq/cli/cmd/rx/api/CmdLsqRxBenchmarkParent.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.cli.cmd.rx.api; 2 | 3 | import picocli.CommandLine.Command; 4 | 5 | @Command(name="benchmark", description = "Manage benchmarks of query logs", subcommands = { 6 | CmdLsqRxBenchmarkCreate.class, 7 | CmdLsqRxBenchmarkPrepare.class, 8 | CmdLsqRxBenchmarkRun.class, 9 | }) 10 | public class CmdLsqRxBenchmarkParent { 11 | 12 | } 13 | -------------------------------------------------------------------------------- /lsq-cli/src/main/java/org/aksw/simba/lsq/cli/cmd/rx/api/CmdLsqRxBenchmarkPrepare.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.cli.cmd.rx.api; 2 | 3 | import java.util.concurrent.Callable; 4 | 5 | import org.aksw.simba.lsq.cli.main.MainCliLsq; 6 | 7 | import picocli.CommandLine.Command; 8 | import picocli.CommandLine.Option; 9 | 10 | @Command(name = "prepare", description = "Prepare a benchmark run") 11 | public class CmdLsqRxBenchmarkPrepare implements Callable { 12 | @Option(names = { "-h", "--help" }, usageHelp = true) 13 | public boolean help = false; 14 | 15 | @Option(names = { "-o", "--stdout" }, description = "Output generated config to STDOUT instead of a file") 16 | public boolean stdout = false; 17 | 18 | @Option(names = { "-c", "--config" }, required = true, description = "Configuration file (RDF)") 19 | public String config = null; 20 | 21 | // @Parameters(arity = "1..*", paramLabel="FILE", description="Log files to process") 22 | // public List logSources = new ArrayList<>(); 23 | 24 | @Override 25 | public Integer call() throws Exception { 26 | MainCliLsq.benchmarkPrepare(this); 27 | return 0; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /lsq-cli/src/main/java/org/aksw/simba/lsq/cli/cmd/rx/api/CmdLsqRxBenchmarkRun.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.cli.cmd.rx.api; 2 | 3 | import java.nio.file.Path; 4 | import java.nio.file.Paths; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | import java.util.concurrent.Callable; 8 | 9 | import org.aksw.simba.lsq.cli.cmd.base.CmdLsqAnalyzeBase.EnricherSpec; 10 | import org.aksw.simba.lsq.cli.main.MainCliLsq; 11 | 12 | import com.google.common.base.StandardSystemProperty; 13 | 14 | import picocli.CommandLine.ArgGroup; 15 | import picocli.CommandLine.Command; 16 | import picocli.CommandLine.Help.Visibility; 17 | import picocli.CommandLine.Option; 18 | import picocli.CommandLine.Parameters; 19 | 20 | 21 | @Command(name = "run", description = "Execute a benchmark run") 22 | public class CmdLsqRxBenchmarkRun 23 | implements Callable 24 | { 25 | @Option(names={"-h", "--help"}, help=true) 26 | public boolean help = false; 27 | 28 | @Option(names= {"-c", "--config"}, required=true, description="Configuration file (RDF)") 29 | public String config = null; 30 | 31 | @Option(names= {"--tdb"}, description="Base path to the TDB2 database directory for indexing benchmark results", showDefaultValue = Visibility.ALWAYS) 32 | public Path tdb2BasePath = Paths.get(StandardSystemProperty.JAVA_IO_TMPDIR.value()); //.resolve("lsq"); 33 | 34 | @Parameters(arity = "1..*", paramLabel="FILE", description="Log files to process") 35 | public List logSources = new ArrayList<>(); 36 | 37 | @ArgGroup(exclusive = true, multiplicity = "0..1") 38 | public EnricherSpec enricherSpec = new EnricherSpec(); 39 | 40 | @Override 41 | public Integer call() throws Exception { 42 | MainCliLsq.benchmarkExecute(this); 43 | return 0; 44 | } 45 | 46 | 47 | } 48 | 49 | -------------------------------------------------------------------------------- /lsq-cli/src/main/java/org/aksw/simba/lsq/cli/cmd/rx/api/CmdLsqRxParent.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.cli.cmd.rx.api; 2 | 3 | import picocli.CommandLine.Command; 4 | 5 | @Command(name="rx", version="LSQ version", description = "Tasks based on rx", subcommands = { 6 | CmdLsqRxAnalyze.class, 7 | CmdLsqRxRdfize.class, 8 | CmdLsqRxBenchmarkParent.class, 9 | CmdLsqRxProbe.class, 10 | }) 11 | public class CmdLsqRxParent { 12 | } 13 | -------------------------------------------------------------------------------- /lsq-cli/src/main/java/org/aksw/simba/lsq/cli/cmd/rx/api/CmdLsqRxProbe.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.cli.cmd.rx.api; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.concurrent.Callable; 6 | 7 | import org.aksw.simba.lsq.cli.main.MainCliLsq; 8 | 9 | import picocli.CommandLine.Command; 10 | import picocli.CommandLine.Option; 11 | import picocli.CommandLine.Parameters; 12 | 13 | @Command(name="probe", description = "Probe log files for known formats") 14 | public class CmdLsqRxProbe 15 | implements Callable 16 | { 17 | @Option(names = {"-h", "--help"}, usageHelp = true) 18 | public boolean help = false; 19 | 20 | // @Parameter(names={"-m", "--format"}, description="Input log format") 21 | // public String inputLogFormat; 22 | // 23 | // @Parameter(names={"-n", "--namespaces"}, description="Namespace prefix sources") 24 | // public List prefixSources = new ArrayList<>(); 25 | 26 | @Parameters(arity="1..*", description="file-list to probe") 27 | public List nonOptionArgs = new ArrayList<>(); 28 | 29 | @Override 30 | public Integer call() throws Exception { 31 | MainCliLsq.probe(this); 32 | return 0; 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /lsq-cli/src/main/java/org/aksw/simba/lsq/cli/cmd/rx/api/CmdLsqRxRdfize.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.cli.cmd.rx.api; 2 | 3 | import java.util.concurrent.Callable; 4 | 5 | import org.aksw.simba.lsq.cli.cmd.base.CmdLsqRdfizeBase; 6 | import org.aksw.simba.lsq.cli.main.MainCliLsq; 7 | 8 | import picocli.CommandLine.Command; 9 | 10 | @Command(name = "rdfize", description = "RDFize query logs into query centric named graphs") 11 | public class CmdLsqRxRdfize 12 | extends CmdLsqRdfizeBase 13 | implements Callable 14 | { 15 | @Override 16 | public Integer call() throws Exception { 17 | MainCliLsq.rdfize(this); 18 | return 0; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /lsq-cli/src/main/java/org/aksw/simba/lsq/cli/cmd/rx/api/CmdRdfIoBaseTmp.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.cli.cmd.rx.api; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Arrays; 5 | import java.util.List; 6 | 7 | import org.aksw.jenax.annotation.reprogen.IriNs; 8 | import org.aksw.jenax.annotation.reprogen.ResourceView; 9 | import org.aksw.simba.lsq.spark.cmd.impl.CmdRdfIo; 10 | import org.apache.jena.rdf.model.Resource; 11 | 12 | import picocli.CommandLine.Option; 13 | import picocli.CommandLine.Parameters; 14 | 15 | @ResourceView 16 | public interface CmdRdfIoBaseTmp 17 | extends CmdRdfIo, Resource 18 | { 19 | @IriNs("eg") 20 | @Parameters(arity="1..*", description="file-list to probe") 21 | @Override 22 | List getNonOptionArgs(); 23 | 24 | @IriNs("eg") 25 | @Override 26 | @Parameters(arity="1..*", description="file-list to probe") 27 | String getOutFile(); 28 | 29 | 30 | @IriNs("eg") 31 | @Option(names= {"-m", "--out-format"}, required=true, description="Output RDF format") 32 | @Override 33 | String getOutFormat(); 34 | 35 | 36 | @IriNs("eg") 37 | @Option(names={"-n", "--namespaces"}, description="Namespace prefix sources") 38 | @Override 39 | List getPrefixSources(); 40 | 41 | 42 | @IriNs("eg") 43 | @Override 44 | @Option(names={"-d", "--defer"}, description="Namespace prefix sources") 45 | long getDeferOutputForUsedPrefixes(); 46 | 47 | } -------------------------------------------------------------------------------- /lsq-cli/src/main/java/org/aksw/simba/lsq/cli/cmd/spark/api/CmdLsqSparkParent.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.cli.cmd.spark.api; 2 | 3 | import picocli.CommandLine.Command; 4 | 5 | @Command(name="spark", description = "LSQ Spark Subcommands", subcommands = { 6 | CmdLsqSparkProbe.class, 7 | CmdLsqSparkRdfize.class, 8 | CmdLsqSparkAnalyze.class 9 | }) 10 | public class CmdLsqSparkParent { 11 | 12 | } 13 | -------------------------------------------------------------------------------- /lsq-cli/src/main/java/org/aksw/simba/lsq/cli/cmd/spark/api/CmdLsqSparkProbe.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.cli.cmd.spark.api; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.concurrent.Callable; 6 | 7 | import org.aksw.simba.lsq.cli.cmd.spark.impl.LsqCmdSparkImpls; 8 | import org.aksw.simba.lsq.cli.main.MainCliLsq; 9 | 10 | import picocli.CommandLine.Command; 11 | import picocli.CommandLine.Option; 12 | import picocli.CommandLine.Parameters; 13 | 14 | @Command(name="probe", description = "Probe log files for known formats") 15 | public class CmdLsqSparkProbe 16 | implements Callable 17 | { 18 | @Option(names = {"-h", "--help"}, usageHelp = true) 19 | public boolean help = false; 20 | 21 | // @Parameter(names={"-m", "--format"}, description="Input log format") 22 | // public String inputLogFormat; 23 | // 24 | // @Parameter(names={"-n", "--namespaces"}, description="Namespace prefix sources") 25 | // public List prefixSources = new ArrayList<>(); 26 | 27 | @Parameters(arity="1..*", description="file-list to probe") 28 | public List nonOptionArgs = new ArrayList<>(); 29 | 30 | @Override 31 | public Integer call() throws Exception { 32 | LsqCmdSparkImpls.probe(this); 33 | return 0; 34 | } 35 | 36 | } -------------------------------------------------------------------------------- /lsq-cli/src/main/java/org/aksw/simba/lsq/cli/cmd/spark/api/CmdLsqSparkRdfize.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.cli.cmd.spark.api; 2 | 3 | import java.util.concurrent.Callable; 4 | 5 | import org.aksw.commons.rx.function.RxFunction; 6 | import org.aksw.jenax.arq.dataset.api.DatasetOneNg; 7 | import org.aksw.simba.lsq.cli.cmd.base.CmdLsqRdfizeBase; 8 | import org.aksw.simba.lsq.cli.cmd.base.CmdOutputSpecBase; 9 | import org.aksw.simba.lsq.cli.util.spark.LsqCliSparkUtils; 10 | 11 | import picocli.CommandLine.Command; 12 | import picocli.CommandLine.Mixin; 13 | 14 | @Command(name = "rdfize", description = "RDFize query logs") 15 | public class CmdLsqSparkRdfize 16 | extends CmdLsqRdfizeBase 17 | implements Callable 18 | { 19 | @Mixin 20 | public CmdOutputSpecBase outputSpec; 21 | 22 | 23 | @Override 24 | public Integer call() throws Exception { 25 | LsqCliSparkUtils.runSparkJob(this, outputSpec, RxFunction.identity()); 26 | return 0; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /lsq-cli/src/main/java/org/aksw/simba/lsq/cli/cmd/spark/impl/LsqCmdSparkImpls.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.cli.cmd.spark.impl; 2 | 3 | import java.util.List; 4 | import java.util.Map; 5 | import java.util.Map.Entry; 6 | 7 | import org.aksw.simba.lsq.cli.cmd.spark.api.CmdLsqSparkProbe; 8 | import org.apache.spark.sql.SparkSession; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import net.sansa_stack.rdf.spark.io.LsqRegistrySparkAdapter; 13 | import net.sansa_stack.rdf.spark.io.LsqSparkIo; 14 | import net.sansa_stack.rdf.spark.io.LsqSparkUtils; 15 | import net.sansa_stack.rdf.spark.io.SourceOfRddOfResources; 16 | 17 | public class LsqCmdSparkImpls { 18 | private static final Logger logger = LoggerFactory.getLogger(LsqCmdSparkImpls.class); 19 | 20 | public static void probe(CmdLsqSparkProbe cmdProbe) { 21 | List nonOptionArgs = cmdProbe.nonOptionArgs; 22 | if(nonOptionArgs.isEmpty()) { 23 | logger.error("No arguments provided."); 24 | logger.error("Argument must be one or more log files which will be probed against all registered LSQ log formats"); 25 | } 26 | 27 | 28 | SparkSession ss = LsqSparkUtils.createSparkSession(conf -> {}); 29 | 30 | Map registry = LsqRegistrySparkAdapter.createDefaultLogFmtRegistry(ss); 31 | 32 | for(int i = 0; i < nonOptionArgs.size(); ++i) { 33 | String filename = nonOptionArgs.get(i); 34 | 35 | List> bestCands = LsqSparkIo.probeLogFormat(registry, filename); 36 | 37 | System.out.println(filename + "\t" + bestCands); 38 | } 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /lsq-cli/src/main/java/org/aksw/simba/lsq/cli/util/spark/LsqCliSparkUtils.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.cli.util.spark; 2 | 3 | import java.util.Optional; 4 | 5 | import org.aksw.commons.rx.function.RxFunction; 6 | import org.aksw.jenax.arq.dataset.api.DatasetOneNg; 7 | import org.aksw.simba.lsq.cli.cmd.base.CmdOutputSpecBase; 8 | import org.aksw.simba.lsq.cli.main.MainCliLsq; 9 | import org.aksw.simba.lsq.core.LsqRdfizeSpec; 10 | import org.apache.jena.shared.PrefixMapping; 11 | import org.apache.jena.shared.impl.PrefixMappingImpl; 12 | import org.apache.spark.api.java.JavaRDD; 13 | import org.apache.spark.sql.SparkSession; 14 | 15 | import net.sansa_stack.rdf.spark.io.LsqSparkIo; 16 | import net.sansa_stack.rdf.spark.io.LsqSparkUtils; 17 | import net.sansa_stack.spark.io.rdf.output.RddRdfWriterFactory; 18 | import net.sansa_stack.spark.rdd.op.rx.JavaRddRxOps; 19 | 20 | public class LsqCliSparkUtils { 21 | 22 | public static void runSparkJob( 23 | LsqRdfizeSpec inputSpec, 24 | CmdOutputSpecBase outputSpec, 25 | RxFunction processor) { 26 | 27 | PrefixMapping prefixes = MainCliLsq.addLsqPrefixes(new PrefixMappingImpl()); 28 | 29 | RddRdfWriterFactory rddRdfWriterFactory = RddRdfWriterFactory.create() 30 | .setGlobalPrefixMapping(prefixes) 31 | .setOutputFormat(outputSpec.outFormat) 32 | .setDeferOutputForUsedPrefixes(100) 33 | // .setOutputFormat(cmd.getOutFormat()) 34 | .setMapQuadsToTriplesForTripleLangs(true) 35 | // .setAllowOverwriteFiles(true) 36 | /// .setPartitionFolder(cmd.getOutFolder()) 37 | .setTargetFile(outputSpec.outFile) 38 | // .setUseElephas(true) 39 | .setAllowOverwriteFiles(true) 40 | .setDeletePartitionFolderAfterMerge(true) 41 | .validate(); 42 | 43 | SparkSession sc = LsqSparkUtils.createSparkSession(conf -> { 44 | Optional.ofNullable(inputSpec.getTemporaryDirectory()).ifPresent(v -> conf.set("spark.local.dir", v)); 45 | }); 46 | 47 | JavaRDD baseRdd = LsqSparkIo.createLsqRdfFlow(sc, inputSpec); 48 | JavaRDD outRdd = JavaRddRxOps.mapPartitions(baseRdd, processor); 49 | 50 | 51 | rddRdfWriterFactory.forDataset(outRdd).runUnchecked(); 52 | } 53 | 54 | // public static RxFunction, Resource> namedModelToResource() { 55 | // return RxFunction.>identity() 56 | // .andThenMap(t -> t._2().createResource(t._1())); 57 | // } 58 | } 59 | -------------------------------------------------------------------------------- /lsq-cli/src/main/resources/log4j2.properties: -------------------------------------------------------------------------------- 1 | dest = err 2 | name = LsqLoggerConfig 3 | 4 | appender.console.type = Console 5 | appender.console.name = STDERR 6 | appender.console.target = SYSTEM_ERR 7 | appender.console.layout.type = PatternLayout 8 | appender.console.layout.pattern = [%p] %m%n 9 | #appender.console.layout.pattern = %d{HH:mm:ss} %t %-5p %-15c{1} :: %m%n 10 | 11 | 12 | rootLogger.level = info 13 | rootLogger.appenderRef.stderr.ref = STDERR 14 | 15 | logger.jetty.name = org.eclipse.jetty 16 | logger.jetty.level = warn 17 | 18 | logger.spring.name = org.springframework 19 | logger.spring.level = warn 20 | 21 | logger.cors.name = org.aksw.jena_sparql_api.web.filters.CorsFilter 22 | logger.cors.level = warn 23 | 24 | logger.riot.name = org.apache.jena.riot.resultset.rw 25 | logger.riot.level = off 26 | 27 | logger.syntax.name = org.apache.jena.sparql.syntax.syntaxtransform.ElementTransformer 28 | logger.syntax.level = off 29 | 30 | logger.fs.name = org.aksw.jena_sparql_api.sparql.ext.fs.QueryIterServiceOrFile 31 | logger.fs.level = info 32 | -------------------------------------------------------------------------------- /lsq-cli/src/test/resources/lsq-example-config.ttl: -------------------------------------------------------------------------------- 1 | @prefix dct: . 2 | @prefix xsd: . 3 | @prefix lsq: . 4 | 5 | 6 | 7 | lsq:baseIri ; 8 | lsq:connectionTimeoutForRetrieval "60"^^xsd:decimal ; 9 | lsq:executionTimeoutForRetrieval "300"^^xsd:decimal ; 10 | lsq:connectionTimeoutForCounting "60"^^xsd:decimal ; 11 | lsq:executionTimeoutForCounting "300"^^xsd:decimal ; 12 | lsq:maxByteSizeForCounting "-1"^^xsd:long ; 13 | lsq:maxByteSizeForSerialization 14 | "1000000"^^xsd:long ; 15 | lsq:maxItemCountForCounting "1000000"^^xsd:long ; 16 | lsq:maxItemCountForSerialization 17 | "-1"^^xsd:long ; 18 | lsq:maxCount "1000000"^^xsd:long ; 19 | lsq:datasetLabel "mydataset" ; 20 | lsq:datasetSize "2005615"^^xsd:long ; 21 | lsq:endpoint [ a ; 22 | 23 | 24 | ] ; 25 | lsq:userAgent "Linked SPARQL Queries (LSQ) Client" ; 26 | dct:identifier "xc-mydataset_2020-07-25" . 27 | -------------------------------------------------------------------------------- /lsq-cli/src/test/resources/lsq-example-run.ttl: -------------------------------------------------------------------------------- 1 | @prefix dct: . 2 | @prefix xsd: . 3 | @prefix lsq: . 4 | @prefix lsqr: . 5 | @prefix sp: . 6 | @prefix rdf: . 7 | @prefix rdfs: . 8 | @prefix prov: . 9 | 10 | 11 | lsq:connectionTimeoutForRetrieval "0.01"^^xsd:decimal ; 12 | lsq:executionTimeoutForRetrieval "0.01"^^xsd:decimal ; 13 | lsq:connectionTimeoutForCounting "0.01"^^xsd:decimal ; 14 | lsq:executionTimeoutForCounting "0.01"^^xsd:decimal ; 15 | lsq:maxByteSizeForCounting "-1"^^xsd:long ; 16 | lsq:maxByteSizeForSerialization 17 | "1000000"^^xsd:long ; 18 | lsq:maxResultCountForCounting "1000"^^xsd:long ; 19 | lsq:maxResultCountForSerialization 20 | "-1"^^xsd:long ; 21 | lsq:maxCount "-1"^^xsd:long ; 22 | lsq:baseIri ; 23 | lsq:datasetIri ; 24 | lsq:datasetLabel "dbpedia.org-somedata" ; 25 | lsq:datasetSize "2005615"^^ ; 26 | lsq:endpoint [ a ; 27 | 28 | 29 | ] ; 30 | lsq:userAgent "Linked SPARQL Queries (LSQ) Client" ; 31 | 32 | "xc-dbpedia.org-somedata_2020-07-25" . 33 | 34 | 35 | lsq:config ; 36 | dct:identifier "xc-mydataset_2020-07-25_2020-08-05T00:24:05.487Z" ; 37 | 38 | "2020-08-05T00:24:05.487Z"^^xsd:dateTime . 39 | -------------------------------------------------------------------------------- /lsq-core/src/main/java/org/aksw/simba/lsq/core/LsqRdfizeSpec.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.core; 2 | 3 | import java.util.List; 4 | 5 | public interface LsqRdfizeSpec { 6 | 7 | String getBaseIri(); 8 | 9 | String getInputLogFormat(); 10 | 11 | List getPrefixSources(); 12 | 13 | boolean isNoMerge(); 14 | 15 | boolean isNoHostHash(); 16 | 17 | String getHostSalt(); 18 | 19 | // Consolidate "Slim mode" and "query only" into a class that captures what features to rdfize 20 | 21 | boolean isSlimMode(); 22 | boolean isQueryOnly(); 23 | 24 | String getEndpointUrl(); 25 | 26 | List getNonOptionArgs(); 27 | 28 | String getBufferSize(); 29 | 30 | String getTemporaryDirectory(); 31 | 32 | int getParallel(); 33 | 34 | } 35 | -------------------------------------------------------------------------------- /lsq-core/src/main/java/org/aksw/simba/lsq/core/ResourceParser.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.core; 2 | 3 | import java.io.InputStream; 4 | import java.util.concurrent.Callable; 5 | 6 | import org.aksw.jenax.arq.dataset.api.ResourceInDataset; 7 | 8 | import io.reactivex.rxjava3.core.Flowable; 9 | 10 | public interface ResourceParser { 11 | Flowable parse(Callable inSupp); 12 | } 13 | -------------------------------------------------------------------------------- /lsq-core/src/main/java/org/aksw/simba/lsq/core/SinkIO.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.core; 2 | 3 | import java.io.PrintStream; 4 | import java.util.function.BiConsumer; 5 | 6 | import org.apache.jena.atlas.lib.Sink; 7 | 8 | /** 9 | * Jena Sink implementation that forwards data to a printstream, such as STDOUT 10 | * 11 | * @author raven Mar 21, 2018 12 | * 13 | * @param 14 | */ 15 | public class SinkIO 16 | implements Sink 17 | { 18 | protected PrintStream out; 19 | protected boolean doClose; 20 | protected BiConsumer consumer; 21 | 22 | public static boolean isStdStream(PrintStream stream) { 23 | boolean result = System.out == stream || System.err == stream; 24 | return result; 25 | } 26 | 27 | public SinkIO(BiConsumer consumer) { 28 | this(System.out, consumer); 29 | } 30 | 31 | public SinkIO(PrintStream out, BiConsumer consumer) { 32 | // By default, close streams unless we are dealing with stdout or stderr 33 | this(out, !isStdStream(out), consumer); 34 | } 35 | 36 | public SinkIO(PrintStream out, boolean doClose, BiConsumer consumer) { 37 | this.out = out; 38 | this.doClose = doClose; 39 | this.consumer = consumer; 40 | } 41 | 42 | @Override 43 | public void close() { 44 | if(doClose) { 45 | out.close(); 46 | } 47 | } 48 | 49 | @Override 50 | public void send(T item) { 51 | consumer.accept(out, item); 52 | } 53 | 54 | @Override 55 | public void flush() { 56 | out.flush(); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /lsq-core/src/main/java/org/aksw/simba/lsq/core/io/input/registry/LsqInputFormatRegistry.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.core.io.input.registry; 2 | 3 | import java.util.AbstractMap.SimpleEntry; 4 | import java.util.LinkedHashMap; 5 | import java.util.Map; 6 | import java.util.Map.Entry; 7 | import java.util.stream.Collectors; 8 | 9 | import org.aksw.simba.lsq.core.ResourceParser; 10 | import org.aksw.simba.lsq.core.rx.io.input.LsqRxIo; 11 | import org.aksw.simba.lsq.parser.Mapper; 12 | import org.aksw.simba.lsq.parser.WebLogParser; 13 | import org.apache.jena.riot.Lang; 14 | import org.apache.jena.riot.RDFDataMgr; 15 | 16 | public class LsqInputFormatRegistry { 17 | 18 | public static Map createDefaultLogFmtRegistry() { 19 | Map result = new LinkedHashMap<>(); 20 | 21 | // Load line based log formats 22 | result.putAll( 23 | wrap(WebLogParser.loadRegistry(RDFDataMgr.loadModel("default-log-formats.ttl")))); 24 | 25 | // Add custom RDF based log format(s) 26 | result.put("rdf", in -> LsqRxIo.createResourceStreamFromRdf(in, Lang.NTRIPLES, "http://example.org/")); 27 | 28 | // Add multi-line sparql format 29 | result.put("sparql", in -> LsqRxIo.createSparqlStream(in)); 30 | 31 | return result; 32 | } 33 | 34 | 35 | 36 | public static Map wrap(Map webLogParserRegistry) { 37 | Map result = webLogParserRegistry.entrySet().stream().map(e -> { 38 | String name = e.getKey(); 39 | ResourceParser r = inSupp -> LsqRxIo.createResourceStreamFromMapperRegistry(inSupp, webLogParserRegistry::get, name); 40 | return new SimpleEntry<>(name, r); 41 | }).collect(Collectors.toMap(Entry::getKey, Entry::getValue)); 42 | 43 | return result; 44 | } 45 | 46 | 47 | 48 | } 49 | -------------------------------------------------------------------------------- /lsq-core/src/main/java/org/aksw/simba/lsq/core/rx/io/input/LsqLogRecordRdfizer.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.core.rx.io.input; 2 | 3 | import java.util.function.Function; 4 | 5 | import org.aksw.jenax.stmt.core.SparqlStmt; 6 | import org.aksw.simba.lsq.core.LsqRdfizer; 7 | import org.apache.jena.rdf.model.Resource; 8 | 9 | public class LsqLogRecordRdfizer 10 | implements Function 11 | { 12 | protected Function sparqlStmtParser; 13 | protected String baseIri; 14 | protected String hostHashSalt; 15 | protected String serviceUrl; 16 | protected Function hashFn; 17 | 18 | public LsqLogRecordRdfizer(Function sparqlStmtParser, String baseIri, String hostHashSalt, 19 | String serviceUrl, Function hashFn) { 20 | super(); 21 | this.sparqlStmtParser = sparqlStmtParser; 22 | this.baseIri = baseIri; 23 | this.hostHashSalt = hostHashSalt; 24 | this.serviceUrl = serviceUrl; 25 | this.hashFn = hashFn; 26 | } 27 | 28 | @Override 29 | public Resource apply(Resource logEntry) { 30 | return LsqRdfizer.rdfizeLogRecord(sparqlStmtParser, baseIri, hostHashSalt, serviceUrl, hashFn, logEntry).orElse(null); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /lsq-core/src/main/java/org/aksw/simba/lsq/core/util/SpinUtils.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.core.util; 2 | 3 | import org.apache.jena.query.Query; 4 | import org.apache.jena.rdf.model.Model; 5 | import org.apache.jena.rdf.model.ModelFactory; 6 | import org.apache.jena.rdf.model.Resource; 7 | import org.spinrdf.arq.ARQ2SPIN; 8 | 9 | public class SpinUtils { 10 | public static org.spinrdf.model.Query createSpinModel( 11 | Query query, 12 | Model tgtModel 13 | ) { 14 | 15 | return createSpinModel(query, tgtModel.createResource()); 16 | } 17 | public static org.spinrdf.model.Query createSpinModel( 18 | Query query, 19 | Resource spinRes 20 | // BiFunction lsqResToIri 21 | ) { 22 | query = query.cloneQuery(); 23 | query.getGraphURIs().clear(); 24 | 25 | // queryNo++; 26 | // .. generate the spin model ... 27 | //Model spinModel = queryRes.getModel(); 28 | // Model spinModel = ModelFactory.createDefaultModel(); 29 | Model spinModel = spinRes == null ? null : spinRes.getModel(); 30 | if(spinModel == null) { 31 | spinModel = ModelFactory.createDefaultModel(); 32 | } 33 | ARQ2SPIN arq2spin = new ARQ2SPIN(spinModel); 34 | org.spinrdf.model.Query tmpSpinRes = arq2spin.createQuery(query, spinRes == null ? null : spinRes.getURI()); 35 | 36 | // ... and rename the blank node of the query 37 | // ResourceUtils.renameResource(tmpSpinRes, spinRes.getURI()); 38 | 39 | // ... and skolemize the rest 40 | //Skolemize.skolemize(spinRes); 41 | 42 | return tmpSpinRes; 43 | 44 | // System.out.println("TEST {"); 45 | // SpinUtils.indexTriplePatterns2(tmpSpinRes.getModel()).forEach(System.out::println); 46 | // SpinUtils.itp(tmpSpinRes).forEach(System.out::println); 47 | // System.out.println("}"); 48 | //tmpSpinRes.as(org.topbraid.spin.model.Query.class).getWhereElements().forEach(e -> { 49 | //System.out.println("XXElement: " + e.asResource().getId() + ": " + e); 50 | //}); 51 | 52 | 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /lsq-core/src/main/java/org/aksw/simba/lsq/util/NestedResource.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.util; 2 | import org.apache.jena.rdf.model.Model; 3 | import org.apache.jena.rdf.model.Resource; 4 | 5 | /** 6 | * Helper class to conveniently create nested resources 7 | * 8 | * Resources in the given model are only created upon calling .get(). 9 | * 10 | * For that reason, 'current' merely acts as a cache 11 | * current.getModel() should always equal 'model'. 12 | * 13 | * @author raven 14 | * 15 | */ 16 | public class NestedResource { 17 | protected String str; 18 | protected Resource current = null; 19 | protected Model model; 20 | protected NestedResource parent; 21 | 22 | public NestedResource(Model model, String str) { 23 | this(null, model, str, null); 24 | } 25 | 26 | // Use NestedResource.from(..) instead 27 | public NestedResource(Resource current) { 28 | this(null, current.getModel(), current.getURI(), current); 29 | } 30 | 31 | // Avoid calling this ctor directly 32 | public NestedResource(NestedResource parent, Model model, String str, Resource current) { 33 | this.parent = parent; 34 | this.model = model; 35 | this.str = str; 36 | this.current = current; 37 | } 38 | 39 | /** 40 | * A resource instance is only added to the model when .get() is called 41 | * 42 | * @return 43 | */ 44 | public Resource get() { 45 | if(current == null) { 46 | current = model.createResource(str); 47 | } 48 | return current; 49 | } 50 | 51 | public String str() { 52 | return str; 53 | } 54 | 55 | public NestedResource nest(String name) { 56 | NestedResource result = name.isEmpty() 57 | ? this 58 | : new NestedResource(this, model, str + name, null); 59 | 60 | return result; 61 | } 62 | 63 | 64 | public static NestedResource from(Resource r) { 65 | return new NestedResource(r); 66 | } 67 | 68 | public static NestedResource from(Model model, String baseUri) { 69 | return new NestedResource(model, baseUri); 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /lsq-core/src/main/resources-filtered/lsq-core.properties: -------------------------------------------------------------------------------- 1 | lsq-core.version=${project.version} 2 | lsq-core.build.timestamp=${timestamp} 3 | -------------------------------------------------------------------------------- /lsq-core/src/main/resources/lsq-invert-rdfized-log.sparql: -------------------------------------------------------------------------------- 1 | # Return the unique queries in lsq data 2 | # Use sparql-integrate with '-u' flag to scan over named graphs 3 | 4 | PREFIX lsq: 5 | # PREFIX eg: 6 | CONSTRUCT { 7 | GRAPH ?s { 8 | ?s 9 | lsq:text ?str ; 10 | lsq:hash ?hash ; 11 | lsq:hasRemoteExec ?x . 12 | 13 | ?x 14 | ?p ?o 15 | } 16 | } 17 | { 18 | # Wrapping with superfluous projection because of https://issues.apache.org/jira/browse/JENA-1844 19 | { SELECT ?x ?s ?str ?hash { 20 | GRAPH ?x { 21 | ?e lsq:text ?str 22 | } 23 | 24 | # Computing the hash twice by inlining the expression because of https://issues.apache.org/jira/browse/JENA-1843 25 | BIND(SHA256(?str) AS ?hash) 26 | BIND(IRI(CONCAT('urn:lsq:query:sha256:', SHA256(?str))) AS ?s) 27 | } } 28 | OPTIONAL { 29 | GRAPH ?x { 30 | ?x ?p ?o 31 | FILTER(?p != lsq:text) 32 | } 33 | } 34 | } 35 | 36 | -------------------------------------------------------------------------------- /lsq-core/src/main/resources/lsq-prefixes.json: -------------------------------------------------------------------------------- 1 | { 2 | "@context": { 3 | "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", 4 | "lsqr": "http://lsq.aksw.org/res/", 5 | "lsqv": "http://lsq.aksw.org/vocab#", 6 | "sp": "http://spinrdf.org/sp#", 7 | "void": "http://rdfs.org/ns/void#", 8 | "dct": "http://purl.org/dc/terms/", 9 | "prov": "http://www.w3.org/ns/prov#", 10 | "xsd": "http://www.w3.org/2001/XMLSchema#", 11 | "sd": "http://www.w3.org/ns/sparql-service-description#" 12 | } 13 | } -------------------------------------------------------------------------------- /lsq-core/src/main/resources/lsq-prefixes.ttl: -------------------------------------------------------------------------------- 1 | @prefix rdf: . 2 | @prefix lsqr: . 3 | @prefix lsqv: . 4 | @prefix sp: . 5 | @prefix void: . 6 | @prefix dct: . 7 | @prefix prov: . 8 | @prefix xsd: . 9 | @prefix sd: . 10 | -------------------------------------------------------------------------------- /lsq-core/src/main/resources/lsq-slimify.sparql: -------------------------------------------------------------------------------- 1 | # From log records, remove all but host, timestamp and query 2 | # Use sparql-integrate with '-u' flag to scan over named graphs 3 | 4 | PREFIX lsq: 5 | PREFIX prov: 6 | # PREFIX eg: 7 | 8 | DELETE { 9 | GRAPH ?g { 10 | ?s ?p ?o . 11 | ?o ?x ?y 12 | } 13 | } 14 | WHERE { 15 | { SELECT DISTINCT ?g ?s { 16 | GRAPH ?g { 17 | ?s lsq:query [] . 18 | } 19 | } } 20 | GRAPH ?g { 21 | ?s ?p ?o 22 | # Optional in order to delete orphaned lsq:header resources (such as Referer, User-agent) 23 | OPTIONAL{ ?o ?x ?y } 24 | FILTER(?p NOT IN (lsq:host, lsq:hostHash, lsq:uri, lsq:endpoint, prov:atTime)) 25 | 26 | 27 | # Remove lsq:text if ?s is referenced by remote execution with that that attribute 28 | # FILTER(?p != lsq:text || EXISTS { 29 | # GRAPH ?g { 30 | # [] 31 | # lsq:hasRemoteExec ?s ; 32 | # lsq:text [] 33 | # } 34 | # }) 35 | 36 | } 37 | } 38 | 39 | 40 | CONSTRUCT WHERE { GRAPH ?g { ?s ?p ?o } } 41 | 42 | -------------------------------------------------------------------------------- /lsq-core/src/main/resources/rdfa11.json: -------------------------------------------------------------------------------- 1 | { 2 | "@context": { 3 | "as": "https://www.w3.org/ns/activitystreams#", 4 | "dqv": "http://www.w3.org/ns/dqv#", 5 | "duv": "https://www.w3.org/TR/vocab-duv#", 6 | "cat": "http://www.w3.org/ns/dcat#", 7 | "qb": "http://purl.org/linked-data/cube#", 8 | "grddl": "http://www.w3.org/2003/g/data-view#", 9 | "ldp": "http://www.w3.org/ns/ldp#", 10 | "oa": "http://www.w3.org/ns/oa#", 11 | "ma": "http://www.w3.org/ns/ma-ont#", 12 | "owl": "http://www.w3.org/2002/07/owl#", 13 | "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", 14 | "rdfa": "http://www.w3.org/ns/rdfa#", 15 | "rdfs": "http://www.w3.org/2000/01/rdf-schema#", 16 | "rif": "http://www.w3.org/2007/rif#", 17 | "rr": "http://www.w3.org/ns/r2rml#", 18 | "skos": "http://www.w3.org/2004/02/skos/core#", 19 | "skosxl": "http://www.w3.org/2008/05/skos-xl#", 20 | "wdr": "http://www.w3.org/2007/05/powder#", 21 | "void": "http://rdfs.org/ns/void#", 22 | "wdrs": "http://www.w3.org/2007/05/powder-s#", 23 | "xhv": "http://www.w3.org/1999/xhtml/vocab#", 24 | "xml": "http://www.w3.org/XML/1998/namespace", 25 | "xsd": "http://www.w3.org/2001/XMLSchema#", 26 | "prov": "http://www.w3.org/ns/prov#", 27 | "sd": "http://www.w3.org/ns/sparql-service-description#", 28 | "org": "http://www.w3.org/ns/org#", 29 | "gldp": "http://www.w3.org/ns/people#", 30 | "cnt": "http://www.w3.org/2008/content#", 31 | "dcat": "http://www.w3.org/ns/dcat#", 32 | "earl": "http://www.w3.org/ns/earl#", 33 | "ht": "http://www.w3.org/2006/http#", 34 | "ptr": "http://www.w3.org/2009/pointers#", 35 | "cc": "http://creativecommons.org/ns#", 36 | "ctag": "http://commontag.org/ns#", 37 | "dc": "http://purl.org/dc/terms/", 38 | "dc11": "http://purl.org/dc/elements/1.1/", 39 | "dcterms": "http://purl.org/dc/terms/", 40 | "foaf": "http://xmlns.com/foaf/0.1/", 41 | "gr": "http://purl.org/goodrelations/v1#", 42 | "ical": "http://www.w3.org/2002/12/cal/icaltzd#", 43 | "og": "http://ogp.me/ns#", 44 | "rev": "http://purl.org/stuff/rev#", 45 | "sioc": "http://rdfs.org/sioc/ns#", 46 | "v": "http://rdf.data-vocabulary.org/#", 47 | "vcard": "http://www.w3.org/2006/vcard/ns#", 48 | "schema": "http://schema.org/", 49 | "describedby": "http://www.w3.org/2007/05/powder-s#describedby", 50 | "license": "http://www.w3.org/1999/xhtml/vocab#license", 51 | "role": "http://www.w3.org/1999/xhtml/vocab#role", 52 | "ssn": "http://www.w3.org/ns/ssn/", 53 | "sosa":"http://www.w3.org/ns/sosa/", 54 | "time":"http://www.w3.org/2006/time#" 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /lsq-core/src/test/java/org/aksw/simba/lsq/TestLsqFeatureExtraction.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq; 2 | 3 | import java.util.Map; 4 | 5 | import org.aksw.simba.lsq.util.ElementVisitorFeatureExtractor; 6 | import org.aksw.simba.lsq.vocab.LSQ; 7 | import com.google.common.collect.ImmutableMap; 8 | import org.apache.jena.query.QueryFactory; 9 | import org.apache.jena.rdf.model.Resource; 10 | import org.junit.Assert; 11 | import org.junit.Test; 12 | 13 | 14 | public class TestLsqFeatureExtraction { 15 | 16 | @Test 17 | public void testFeatureExtraction() { 18 | Map actual = ElementVisitorFeatureExtractor.getFeatures( 19 | QueryFactory.create( 20 | "SELECT DISTINCT ?s WHERE { { ?s a/a ?o . ?s ?p ?x } UNION { ?x a ?z } } LIMIT 10 OFFSET 1")); 21 | 22 | Map expected = ImmutableMap.builder() 23 | .put(LSQ.TriplePath, 1) 24 | .put(LSQ.LinkPath, 2) 25 | .put(LSQ.SeqPath, 1) 26 | .put(LSQ.TriplePattern, 2) 27 | .put(LSQ.Group, 3) 28 | .put(LSQ.Union, 1) 29 | .put(LSQ.Select, 1) 30 | .put(LSQ.Distinct, 1) 31 | .put(LSQ.Limit, 1) 32 | .put(LSQ.Offset, 1) 33 | .build(); 34 | 35 | Assert.assertEquals(expected, actual); 36 | 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /lsq-core/src/test/java/org/aksw/simba/lsq/TestLsqHashId.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq; 2 | 3 | import org.junit.Test; 4 | 5 | public class TestLsqHashId { 6 | 7 | @Test 8 | public void testHashId() { 9 | // Model m = ModelFactory.createDefaultModel(); 10 | // m.createResource().as(LocalExecution.class) 11 | // .setQueryExec(queryExec) 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /lsq-core/src/test/java/org/aksw/simba/lsq/TestLsqResourceViews.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq; 2 | 3 | import org.aksw.simba.lsq.model.LsqQuery; 4 | import org.aksw.simba.lsq.model.LsqStructuralFeatures; 5 | import org.aksw.simba.lsq.model.RemoteExecution; 6 | import org.apache.jena.rdf.model.Model; 7 | import org.apache.jena.rdf.model.ModelFactory; 8 | import org.junit.Assert; 9 | import org.junit.Test; 10 | 11 | 12 | public class TestLsqResourceViews { 13 | 14 | @Test 15 | public void testLsqResourceViews() { 16 | Model m = ModelFactory.createDefaultModel(); 17 | 18 | LsqQuery q = m.createResource().as(LsqQuery.class); 19 | String queryStr = "SELECT * { ?s a ?o }"; 20 | q.setText(queryStr); 21 | 22 | 23 | // q.getRemoteExecutions(RemoteExecution.class) 24 | q.getRemoteExecutions() 25 | .add(m.createResource().as(RemoteExecution.class) 26 | .setHost("host") 27 | .setSequenceId(0l) 28 | ); 29 | LsqStructuralFeatures summary = m.createResource().as(LsqStructuralFeatures.class); 30 | summary.setProjectVarCount(2); 31 | 32 | q.setStructuralFeatures(summary); 33 | 34 | 35 | Assert.assertEquals(q.getText(), queryStr); 36 | Assert.assertEquals((long)q.getStructuralFeatures().getProjectVarCount(), 2l); 37 | Assert.assertEquals(q.getRemoteExecutions().iterator().next().getHost(), "host"); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /lsq-core/src/test/java/org/aksw/simba/lsq/TestLsqValidationWithShacl.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq; 2 | 3 | import org.apache.jena.rdf.model.Model; 4 | import org.apache.jena.riot.RDFDataMgr; 5 | import org.apache.jena.riot.RDFFormat; 6 | import org.apache.jena.shacl.ShaclValidator; 7 | import org.apache.jena.shacl.ValidationReport; 8 | import org.junit.Assert; 9 | import org.junit.Test; 10 | 11 | public class TestLsqValidationWithShacl { 12 | 13 | 14 | @Test 15 | public void testLsqValidationWithShacl() { 16 | Model shaclModel = RDFDataMgr.loadModel("lsq.shacl.ttl"); 17 | 18 | Model dataModel = RDFDataMgr.loadModel("2019-02-14-single-resource.lsq.ttl"); 19 | 20 | // Perform the validation of everything, using the data model 21 | // also as the shapes model - you may have them separated 22 | ValidationReport report; 23 | try { 24 | report = ShaclValidator.get().validate(shaclModel.getGraph(), dataModel.getGraph()); 25 | } catch (Exception e) { 26 | RDFDataMgr.write(System.err, shaclModel, RDFFormat.NTRIPLES); 27 | throw new RuntimeException("Internal error during shacl validation - model printed to stderr", e); 28 | } 29 | 30 | boolean conforms = report.conforms(); 31 | 32 | // 33 | // Resource report = ValidationUtil.validateModel(dataModel, shaclModel, true); 34 | // 35 | // boolean conforms = report.getProperty(SH.conforms).getBoolean(); 36 | // 37 | // if(!conforms) { 38 | // // Print violations 39 | // RDFDataMgr.write(System.err, report.getModel(), RDFFormat.TURTLE_PRETTY); 40 | // } 41 | // 42 | Assert.assertTrue(conforms); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /lsq-core/src/test/java/org/aksw/simba/lsq/TestTripleRemoval.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq; 2 | 3 | import java.util.Random; 4 | 5 | import org.aksw.jenax.arq.util.node.NodeTransformLib2; 6 | import org.apache.jena.graph.Graph; 7 | import org.apache.jena.graph.Node; 8 | import org.apache.jena.graph.NodeFactory; 9 | import org.apache.jena.graph.Triple; 10 | import org.apache.jena.mem.GraphMem; 11 | import org.apache.jena.riot.RDFDataMgr; 12 | import org.apache.jena.riot.RDFFormat; 13 | import org.apache.jena.util.iterator.ExtendedIterator; 14 | import org.apache.jena.vocabulary.RDF; 15 | import org.junit.Test; 16 | 17 | 18 | public class TestTripleRemoval { 19 | @Test 20 | public void test() { 21 | Graph graph = new GraphMem(); // GraphFactory.createDefaultGraph(); // new GraphMem(); 22 | RDFDataMgr.read(graph, "broken-triple-removal-01.ttl"); 23 | RDFDataMgr.write(System.out, graph, RDFFormat.NTRIPLES); 24 | ExtendedIterator it = graph.find(); 25 | try { 26 | while (it.hasNext()) { 27 | Triple t = it.next(); 28 | it.remove(); 29 | } 30 | } finally { 31 | it.close(); 32 | } 33 | } 34 | 35 | // @Test 36 | public void test2() { 37 | Random r = new Random(0); 38 | for (int j = 0; j < 100000; ++j) { 39 | System.out.println(j); 40 | Graph graph = new GraphMem(); // new NamedGraphWrapper(null, new GraphMem()); //GraphFactory.createGraphMem(); 41 | for (int i = 0; i < 1000; ++i) { 42 | String str = "urn:foobar" + (i / 3); 43 | Node n = NodeFactory.createURI(str.repeat(100)); 44 | Triple t = Triple.create(n, n, n); 45 | // System.out.println(t); 46 | graph.add(t); 47 | ExtendedIterator it = graph.find(); 48 | while (it.hasNext()) { 49 | it.next(); 50 | if (r.nextFloat() > 0.95) { 51 | it.remove(); 52 | } 53 | } 54 | it.close(); 55 | } 56 | 57 | Graph after = NodeTransformLib2.applyNodeTransform(x -> r.nextFloat() > 1 ? x : RDF.Nodes.type, graph); 58 | // RDFDataMgr.write(System.out, DatasetFactory.wrap(DatasetGraphFactory.wrap(after)), RDFFormat.TRIG_PRETTY); 59 | } 60 | // ExtendedIterator it = graph.find(); 61 | // try { 62 | // while (it.hasNext()) { 63 | // it.next(); 64 | // it.remove(); 65 | // } 66 | // } finally { 67 | // it.close(); 68 | // } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/logs/combined.swdf.single-query.log: -------------------------------------------------------------------------------- 1 | 140.203.154.206 - - [16/May/2014:03:22:51 +0100] "GET /sparql?query=PREFIX+swc%3A+%3Chttp%3A%2F%2Fdata.semanticweb.org%2Fns%2Fswc%2Fontology%23%3E+PREFIX+rdfs%3A+%3Chttp%3A%2F%2Fwww.w3.org%2F2000%2F01%2Frdf-schema%23%3E++++++++++++SELECT+DISTINCT+%3Fconf_uri+%3Fconf_name++%3Fconf_acronym+WHERE+%7B+%3Fconf_uri+a+swc%3AConferenceEvent+.+%3Fconf_uri+rdfs%3Alabel+%3Fconf_name+.%3Fconf_uri+swc%3AhasAcronym+%3Fconf_acronym+.%7D+ORDER+BY+%3Fconf_acronym HTTP/1.0" 200 15287 "-" "-" 2 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/logs/combined.swdf.single-triple-pattern.log: -------------------------------------------------------------------------------- 1 | 152.78.189.43 - - [16/May/2014:02:35:42 +0100] "GET /sparql?query=SELECT+%3Fs+%3Fp+%3Fo+WHERE+%7B+%3Fs+%3Fp+%3Fo+%7D+LIMIT+1 HTTP/1.1" 200 521 "-" "-" -------------------------------------------------------------------------------- /lsq-core/src/test/resources/logs/commonVirtuoso.dbpedia.20151025-1.log: -------------------------------------------------------------------------------- 1 | ba3d2de051ffe4dedb0482bbe400686f - - [24/Oct/2015 03:00:00 +0200] "GET /sparql?query=PREFIX++dbpedia%3A+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2F%3E%0A%0ACONSTRUCT+%0A++%7B+%3Fobj+%3Frel+dbpedia%3AMontreal_Carsteel+.%7D%0AWHERE%0A++%7B+%7B+%3Fobj+%3Frel+dbpedia%3AMontreal_Carsteel%7D%0A++++FILTER+regex%28%3Frel%2C+%22%28%2F%7C%23%29%28parent%7Coccupation%7Cowners%7CparentOrganization%7Cdivisions%7Cdivision%7Cinstitution%7Ccompanies%7CkeyFigures%7CowningOrganisation%7Cworkplaces%7ClocationCountry%7Cengines%7Cemployer%7CgoverningBody_of%7Ctenant_of%7Cbrand%7Csector%7CkeyPerson%7CregionServed%7CworkInstitutions%7Cauthor%7Cconstructor%7Ccurrentowner%7CowningOrganisation_of%7Csubsidiary%7Cinstitutions%7CparentCompany%7Cconstructors%7Clicense%7Cheadquarters%7CworkInstitution%7CknownFor%7Cdesignfirm%7CengineName%7ClocationCity%7Csuccessor%7CkeyPeople%7CcurrentTenants_of%7Claterwork%7Caffiliations%7Cboards%7Cindustry%7Cfoundation%7Cpredecessor%7Ctitle%7Cmanufacturer%7Cproducts%7CwebsiteType%7CRelation%7Csubsid%7Cengine%7Cheadquarter%7Cleadership%7Cbuilder%7Ccompany%7CparentOrganisation%7CfoundedBy%7Cowner%7Ccreator%7Cproduct%7CfoundationPlace%7Cmanufacturers%7CareaServed%7Cfounder%7CwikiPageRevisionID%7CowningCompany%7Cchairman%7Cdesigner%7Cservices%7Cownership%7Cpartners%7Cservice%7CwikiPageID%7Cdeveloper%7Clocation%7Clocations%7Cpreviousowners%7Cboard%7Chomepage%29%24%22%29%0A++++FILTER+%28+%21+regex%28%3Fobj%2C+%22%E2%80%93%22%29+%29%0A++++FILTER+%28+%21+regex%28%3Fobj%2C+%22%C3%97%22%29+%29%0A++++FILTER+%28+%21+regex%28%3Fobj%2C+%22%E2%80%99%22%29+%29%0A++++FILTER+%28+%21+regex%28%3Fobj%2C+%22%C2%B0%22%29+%29%0A++++FILTER+%28+%21+regex%28%3Fobj%2C+%22%E2%80%94%22%29+%29%0A++++FILTER+%28+%21+regex%28%3Fobj%2C+%22%C2%BF%22%29+%29%0A++++FILTER+%28+%21+regex%28%3Fobj%2C+%22%C2%A1%22%29+%29%0A++%7D%0A HTTP/1.1" 200 15 "-" "R" "-" 2 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/logs/sparql2.debug.log: -------------------------------------------------------------------------------- 1 | PREFIX ex: SELECT * { { ?s a ex:Person ; ex:firstName ?n } UNION { ?s a ex:Organization ; ex:label ?l } } 2 | ASK WHERE { } 3 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/logs/sparql2.parseError.log: -------------------------------------------------------------------------------- 1 | SELECT * { ? ? ? } 2 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/logs/virtuoso.dbpedia.duplicates.log: -------------------------------------------------------------------------------- 1 | 9c6a991dbf3332fdc973c5b8461ba79f [30/Apr/2010 00:00:00 -0600] "R" "/sparql?default-graph-uri=http%3A%2F%2Fdbpedia.org&should-sponge=&query=SELECT+DISTINCT+%3Fcity+%3Flatd%0D%0AFROM+%3Chttp%3A%2F%2Fdbpedia.org%3E%0D%0AWHERE+%7B%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2FsubdivisionName%3E+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FNetherlands%3E.%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2Flatd%3E+%3Flatd.%0D%0A%7D&format=text%2Fhtml&debug=on&timeout=2200" 2 | 9c6a991dbf3332fdc973c5b8461ba79f [30/Apr/2010 00:00:00 -0600] "R" "/sparql?default-graph-uri=http%3A%2F%2Fdbpedia.org&should-sponge=&query=SELECT+DISTINCT+%3Fcity+%3Flatd+%3Flongd%0D%0AFROM+%3Chttp%3A%2F%2Fdbpedia.org%3E%0D%0AWHERE+%7B%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2FsubdivisionName%3E+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FNetherlands%3E.%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2Flatd%3E+%3Flatd.%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2Flong%3E+%3Flongd.%0D%0A%7D&format=text%2Fhtml&debug=on&timeout=2200" 3 | 9c6a991dbf3332fdc973c5b8461ba79f [30/Apr/2010 00:00:00 -0600] "R" "/sparql?default-graph-uri=http%3A%2F%2Fdbpedia.org&should-sponge=&query=SELECT+DISTINCT+%3Fcity+%3Flatd%0D%0AFROM+%3Chttp%3A%2F%2Fdbpedia.org%3E%0D%0AWHERE+%7B%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2FsubdivisionName%3E+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FNetherlands%3E.%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2Flatd%3E+%3Flatd.%0D%0A%7D&format=text%2Fhtml&debug=on&timeout=2200" 4 | 9c6a991dbf3332fdc973c5b8461ba79f [30/Apr/2010 00:00:00 -0600] "R" "/sparql?default-graph-uri=http%3A%2F%2Fdbpedia.org&should-sponge=&query=SELECT+DISTINCT+%3Fcity+%3Flatd+%3Flongd%0D%0AFROM+%3Chttp%3A%2F%2Fdbpedia.org%3E%0D%0AWHERE+%7B%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2FsubdivisionName%3E+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FNetherlands%3E.%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2Flatd%3E+%3Flatd.%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2Flong%3E+%3Flongd.%0D%0A%7D&format=text%2Fhtml&debug=on&timeout=2200" 5 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/logs/virtuoso.dbpedia.log: -------------------------------------------------------------------------------- 1 | 9c6a991dbf3332fdc973c5b8461ba79f [30/Apr/2010 00:00:00 -0600] "R" "/sparql?default-graph-uri=http%3A%2F%2Fdbpedia.org&should-sponge=&query=SELECT+DISTINCT+%3Fcity+%3Flatd%0D%0AFROM+%3Chttp%3A%2F%2Fdbpedia.org%3E%0D%0AWHERE+%7B%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2FsubdivisionName%3E+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FNetherlands%3E.%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2Flatd%3E+%3Flatd.%0D%0A%7D&format=text%2Fhtml&debug=on&timeout=2200" 2 | 9c6a991dbf3332fdc973c5b8461ba79f [30/Apr/2010 00:00:00 -0600] "R" "/sparql?default-graph-uri=http%3A%2F%2Fdbpedia.org&should-sponge=&query=SELECT+DISTINCT+%3Fcity+%3Flatd+%3Flongd%0D%0AFROM+%3Chttp%3A%2F%2Fdbpedia.org%3E%0D%0AWHERE+%7B%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2FsubdivisionName%3E+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FNetherlands%3E.%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2Flatd%3E+%3Flatd.%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2Flong%3E+%3Flongd.%0D%0A%7D&format=text%2Fhtml&debug=on&timeout=2200" -------------------------------------------------------------------------------- /lsq-core/src/test/resources/logs/virtuoso.dbpedia351.log: -------------------------------------------------------------------------------- 1 | ccce1ececbe46aa7536e94102533f784 [12/Jul/2010 04:00:00 +0200] "R" "/sparql?query=PREFIX++rdfs%3A+%3Chttp%3A%2F%2Fwww.w3.org%2F2000%2F01%2Frdf-schema%23%3E%0APREFIX++foaf%3A+%3Chttp%3A%2F%2Fxmlns.com%2Ffoaf%2F0.1%2F%3E%0APREFIX++owl%3A++%3Chttp%3A%2F%2Fwww.w3.org%2F2002%2F07%2Fowl%23%3E%0A%0ASELECT+DISTINCT++%3Ftitle+%3Fdescription+%3Fsameas%0AWHERE%0A++%7B+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FQI%3E%0A++++++++++++++foaf%3Aname+++++%3Ftitle+%3B%0A++++++++++++++rdfs%3Acomment++%3Fdescription+.%0A++++FILTER+%28+lang%28%3Fdescription%29+%3D+%22en%22+%29%0A++++OPTIONAL%0A++++++%7B+++%7B+%3Fsameas++owl%3AsameAs++%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FQI%3E+.%7D%0A++++++++UNION%0A++++++++++%7B+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FQI%3E%0A++++++++++++++++++++++owl%3AsameAs++%3Fsameas+.%0A++++++++++%7D%0A++++++%7D%0A++%7D%0A&default-graph-uri=http%3A%2F%2Fdbpedia.org" 2 | ccce1ececbe46aa7536e94102533f784 [12/Jul/2010 04:00:00 +0200] "R" "/sparql?query=PREFIX++dbpp%3A+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2F%3E%0A%0ASELECT+DISTINCT++%3FcontainedIn%0AWHERE%0A++%7B+++%7B+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FQI%3E%0A++++++++++++++++++dbpp%3Awriter++%3FcontainedIn+.%0A++++++%7D%0A++++UNION%0A++++++%7B+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FQI%3E%0A++++++++++++++++++dbpp%3AexecutiveProducer++%3FcontainedIn+.%0A++++++%7D%0A++++UNION%0A++++++%7B+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FQI%3E%0A++++++++++++++++++dbpp%3Acreator++%3FcontainedIn+.%0A++++++%7D%0A++++UNION%0A++++++%7B+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FQI%3E%0A++++++++++++++++++dbpp%3Astarring++%3FcontainedIn+.%0A++++++%7D%0A++++UNION%0A++++++%7B+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FQI%3E%0A++++++++++++++++++dbpp%3AexecutiveProducer++%3FcontainedIn+.%0A++++++%7D%0A++++UNION%0A++++++%7B+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FQI%3E%0A++++++++++++++++++dbpp%3Aguest++%3FcontainedIn+.%0A++++++%7D%0A++++UNION%0A++++++%7B+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FQI%3E%0A++++++++++++++++++dbpp%3Adirector++%3FcontainedIn+.%0A++++++%7D%0A++++UNION%0A++++++%7B+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FQI%3E%0A++++++++++++++++++dbpp%3Aproducer++%3FcontainedIn+.%0A++++++%7D%0A++++UNION%0A++++++%7B+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FQI%3E%0A++++++++++++++++++dbpp%3Aseries++%3FcontainedIn+.%0A++++++%7D%0A++%7D%0A&default-graph-uri=http%3A%2F%2Fdbpedia.org" 3 | 96b67ee7d2871208d99bf7e621b2908b [12/Jul/2010 04:00:00 +0200] "R" "/sparql/?query=SELECT+%3Fabstract+WHERE+{+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FErwin_Leo_Himmel%3E+%3Chttp%3A%2F%2Fdbpedia.org%2Fontology%2Fabstract%3E+%3Fabstract.+FILTER+langMatches(lang(%3Fabstract)%2C+%27en%27)+}&format=json" -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-00b105817b1c47b78f23032aeffa91d4.dat: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 4 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-00b105817b1c47b78f23032aeffa91d4.sparql: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?c 2 | WHERE 3 | { { SELECT (count(*) AS ?c) 4 | WHERE 5 | { SELECT DISTINCT ?x0 6 | WHERE 7 | { ?x0 . 8 | ?x0 ?x1 9 | } 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-112748535ddab11bda9a3ec76e01eebc.dat: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 4 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-112748535ddab11bda9a3ec76e01eebc.sparql: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?c 2 | WHERE 3 | { { SELECT (count(*) AS ?c) 4 | WHERE 5 | { 6 | . 7 | 8 | . 9 | ?x0 . 10 | ?x0 ?x1 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-20f6ffc0970367f255dfbcced95cf0ed.dat: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 8 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-20f6ffc0970367f255dfbcced95cf0ed.sparql: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?c 2 | WHERE 3 | { { SELECT (count(*) AS ?c) 4 | WHERE 5 | { SELECT DISTINCT ?x0 6 | WHERE 7 | { ?x0 } 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-33f876148694a00482188bf8c9ccd144.dat: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 4 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-33f876148694a00482188bf8c9ccd144.sparql: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?c 2 | WHERE 3 | { { SELECT (count(*) AS ?c) 4 | WHERE 5 | { SELECT DISTINCT ?x0 ?x1 6 | WHERE 7 | { 8 | . 9 | 10 | . 11 | ?x0 . 12 | ?x0 ?x1 13 | } 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-34d6fd2cf4bb93cc0d2a774af2823375.dat: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 124582 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-34d6fd2cf4bb93cc0d2a774af2823375.sparql: -------------------------------------------------------------------------------- 1 | SELECT (count(*) AS ?c) 2 | WHERE 3 | { ?x0 ?x1} 4 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-3a4ad858f543f82daa50f02eb438c218.dat: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 113487 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-3a4ad858f543f82daa50f02eb438c218.sparql: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?c 2 | WHERE 3 | { { SELECT (count(*) AS ?c) 4 | WHERE 5 | { SELECT DISTINCT ?x0 6 | WHERE 7 | { ?x0 ?x1} 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-411d9d59eef750e34db25b9509148c67.dat: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 124582 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-411d9d59eef750e34db25b9509148c67.sparql: -------------------------------------------------------------------------------- 1 | SELECT (count(*) AS ?_c_) 2 | WHERE 3 | { ?x0 ?x1} 4 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-49179a37bf1e93da3d5ada5aa8db0a79.dat: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 1 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-49179a37bf1e93da3d5ada5aa8db0a79.sparql: -------------------------------------------------------------------------------- 1 | SELECT (count(*) AS ?c) 2 | WHERE 3 | { 4 | } 5 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-63abcdf4f59fb131f75f24175e60fa7b.dat: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 8 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-63abcdf4f59fb131f75f24175e60fa7b.sparql: -------------------------------------------------------------------------------- 1 | SELECT (count(*) AS ?c) 2 | WHERE 3 | { ?x0 } 4 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-71c172100429809f406cf0907383e679.dat: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 1 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-71c172100429809f406cf0907383e679.sparql: -------------------------------------------------------------------------------- 1 | SELECT (count(*) AS ?_c_) 2 | WHERE 3 | { 4 | } 5 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-7c028cda1ee6d80a4a10a32b9ca03f19.dat: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 438336517 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-7c028cda1ee6d80a4a10a32b9ca03f19.sparql: -------------------------------------------------------------------------------- 1 | SELECT (count(*) AS ?c) 2 | WHERE 3 | { ?s ?p ?o } 4 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-af4137d91c3abc348243c2946207b7a2.dat: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | http://dbpedia.org/resource/Alex_Cross_(film) 11 | 12 | 13 | http://dbpedia.org/resource/Rob_Cohen 14 | 15 | 16 | 17 | 18 | http://dbpedia.org/resource/Bone_Tomahawk 19 | 20 | 21 | http://dbpedia.org/resource/S._Craig_Zahler 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-af4137d91c3abc348243c2946207b7a2.sparql: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?x0 ?x1 2 | WHERE 3 | { ?x0 ?x1 ; 4 | . 5 | 6 | . 7 | 8 | 9 | } 10 | OFFSET 0 11 | LIMIT 2 12 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-b5d6d24f3c3b57170beb4cf6a58e728c.dat: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 4 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-b5d6d24f3c3b57170beb4cf6a58e728c.sparql: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?c 2 | WHERE 3 | { { SELECT (count(*) AS ?c) 4 | WHERE 5 | { SELECT DISTINCT ?x0 6 | WHERE 7 | { 8 | . 9 | 10 | . 11 | ?x0 . 12 | ?x0 ?x1 13 | } 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-ecc0934f84813da15414fa919fc59ec0.dat: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 1 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-ecc0934f84813da15414fa919fc59ec0.sparql: -------------------------------------------------------------------------------- 1 | SELECT (count(*) AS ?c) 2 | WHERE 3 | { 4 | } 5 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-fcb889011c1f52dd453f34fb8c9fbbcf.dat: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 8 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-fcb889011c1f52dd453f34fb8c9fbbcf.sparql: -------------------------------------------------------------------------------- 1 | SELECT (count(*) AS ?_c_) 2 | WHERE 3 | { ?x0 } 4 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-fdcd3a7f5482a3b90bc3ab1a4afc1e95.dat: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 1 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests-sparql-cache/http%3A%2F%2Fdbpedia.org%2Fsparql%23%5BdefaultGraphs+%3D+http%3A%2F%2Fdbpedia.org%5D%5BnamedGraphs+%3D+%5D-fdcd3a7f5482a3b90bc3ab1a4afc1e95.sparql: -------------------------------------------------------------------------------- 1 | SELECT (count(*) AS ?_c_) 2 | WHERE 3 | { 4 | } 5 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests/01/data.ttl: -------------------------------------------------------------------------------- 1 | @prefix : . 2 | 3 | :Anne 4 | a :Person ; 5 | :age 20 ; 6 | :knows :Bob ; 7 | . 8 | 9 | :Bob 10 | a :Person ; 11 | :age 22 ; 12 | :knows :Colin ; 13 | . 14 | 15 | 16 | :Colin 17 | a :Person ; 18 | :age 24 ; 19 | # :knows :Anne ; 20 | . 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests/01/query.sparql.log: -------------------------------------------------------------------------------- 1 | PREFIX : SELECT * { ?s a :Person . ?s :knows ?o . ?o a :Person . ?o :age ?a . } 2 | 3 | 4 | PREFIX : SELECT * { ?s a :Person . ?s :knows ?o . ?o a :Person . ?o :age ?a . } 5 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/lsq-tests/triple-pattern-selectivity/tpsel01.sparql: -------------------------------------------------------------------------------- 1 | SELECT DISTINCT ?x0 ?x1 2 | WHERE { 3 | ?x0 4 | ?x1 ; 5 | . 6 | 7 | 8 | . 9 | 10 | 11 | 12 | } 13 | OFFSET 0 14 | LIMIT 2 15 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/rdf-envs/swdf-desc.ttl: -------------------------------------------------------------------------------- 1 | @prefix rdf: 2 | @prefix rdfs: 3 | @prefix owl: 4 | @prefix lsqo: 5 | @prefix lsqr: 6 | 7 | @prefix se: 8 | @prefix dcat: . 9 | 10 | @prefix mvn: 11 | 12 | # There is related work for modeling computing infrastructure: https://ivi.fnwi.uva.nl/sne/ndl/ 13 | @prefix sd: 14 | @prefix hd: 15 | @prefix id: 16 | 17 | 18 | data:org.semanticweb.swdf_swdf-full_latest 19 | a dcat:Dataset . 20 | dcat:theme theme:primary ; 21 | mvn:group "org.semanticweb.swdf" ; 22 | mvn:artifact "swdf-full" ; 23 | mvn:version "latest" ; 24 | dcat:keyword "swdf" ; 25 | dcat:primaryDistribution dist:org.semanticweb.swdf_swdf-full_latest_public ; 26 | rdfs:label "The complete Semantic Web Conference Corpus" ; 27 | rdfs:comment "The complete Semantic Web Conference Corpus a.k.a. Semantic Web Dog Food Corpus" ; 28 | foaf:depiction ; 29 | . 30 | 31 | dist:org.semanticweb.swdf_swdf-full_latest_public 32 | a dcat:Distribution ; 33 | o:priority 1 ; 34 | dcat:distributionOf data:org.semanticweb.swdf_swdf-full_latest ; 35 | dcat:accessURL ; 36 | . 37 | 38 | service:org.semanticweb.swdf_swdf-full_latest_public_sparql 39 | a sd:Service ; 40 | sd:endpoint ; 41 | id:provider infra:org.semanticweb ; 42 | . 43 | 44 | # An infrastructure is comprised of a set of services powered by physical and virtual hosts, that have physical or virtual hardware specifications 45 | infra:org.semanticweb.swdf_swdf_sparql 46 | a id:SingleNodeInfrastructure ; 47 | id:app app:com.openlink_virtuoso_7.2.1 ; 48 | id:host host:org.semanticweb.swdf_swdf-host ; 49 | . 50 | 51 | app:com.openlink_virtuoso_7.2.1 52 | a doap:Project ; 53 | doap:name "Virtuoso" ; 54 | doap:vendor "OpenLink" ; 55 | doap:version "7.2.1" ; 56 | . 57 | 58 | host:org.semanticweb.swdf_swdf-host 59 | a hd:Host ; 60 | # a hd:PhysicalHost ; 61 | hd:cpuType "i7" ; 62 | hd:totalRam "8" ; # in GB ; 63 | hd:avgCpuSpeed "4000" ; 64 | hd:componentSpec cpu:intel-core-i7-6700K ; 65 | . 66 | 67 | cpu:intel-core-i7-6700K 68 | a hd:CPU ; 69 | hd:product "Intel(R) Core(TM) i7-6700K CPU @ 4.00GHz" ; 70 | hd:cpuSize "4000" ; # in Mhz ; 71 | . 72 | 73 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/rdf-envs/trash.ttl: -------------------------------------------------------------------------------- 1 | # I don't think we need an environment, but instead we can use the distribution: 2 | # A 3 | env:org.aksw_lsq-swdf-official_2016 4 | a envo:Environment ; 5 | envo:distribution dist:org.semanticweb_swdf_2016_official ; 6 | envo:systemSpecs sys:org.aksw_lsq-swdf-official_2016 ; 7 | . 8 | -------------------------------------------------------------------------------- /lsq-core/src/test/resources/rdf-envs/virt.log: -------------------------------------------------------------------------------- 1 | 9c6a991dbf3332fdc973c5b8461ba79f [30/Apr/2010 00:00:00 -0600] "R" "/sparql?default-graph-uri=http%3A%2F%2Fdbpedia.org&should-sponge=&query=SELECT+DISTINCT+%3Fcity+%3Flatd%0D%0AFROM+%3Chttp%3A%2F%2Fdbpedia.org%3E%0D%0AWHERE+%7B%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2FsubdivisionName%3E+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FNetherlands%3E.%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2Flatd%3E+%3Flatd.%0D%0A%7D&format=text%2Fhtml&debug=on&timeout=2200" 2 | 9c6a991dbf3332fdc973c5b8461ba79f [30/Apr/2010 00:00:00 -0600] "R" "/sparql?default-graph-uri=http%3A%2F%2Fdbpedia.org&should-sponge=&query=SELECT+DISTINCT+%3Fcity+%3Flatd+%3Flongd%0D%0AFROM+%3Chttp%3A%2F%2Fdbpedia.org%3E%0D%0AWHERE+%7B%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2FsubdivisionName%3E+%3Chttp%3A%2F%2Fdbpedia.org%2Fresource%2FNetherlands%3E.%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2Flatd%3E+%3Flatd.%0D%0A+%3Fcity+%3Chttp%3A%2F%2Fdbpedia.org%2Fproperty%2Flong%3E+%3Flongd.%0D%0A%7D&format=text%2Fhtml&debug=on&timeout=2200" -------------------------------------------------------------------------------- /lsq-core/src/test/resources/test-data.ttl: -------------------------------------------------------------------------------- 1 | @prefix ex: . 2 | 3 | ex:s1 4 | ex:p1 ex:o1 ; 5 | ex:p2 ex:o2, ex:o3 ; 6 | . 7 | 8 | ex:s2 9 | ex:p1 ex:o1 ; 10 | ex:p2 ex:o2, ex:o3 ; 11 | . 12 | 13 | -------------------------------------------------------------------------------- /lsq-debian-cli/CHANGES.txt: -------------------------------------------------------------------------------- 1 | release date=15:00 03.06.2016,version=0.9,urgency=low,by=Claus Stadler ,distribution=ldstack-nightly 2 | * Initial version 3 | -------------------------------------------------------------------------------- /lsq-enrichers/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | lsq-enrichers 8 | jar 9 | 10 | 11 | org.aksw.simba.lsq 12 | lsq-parent 13 | 2.0.0-RC5-SNAPSHOT 14 | 15 | 16 | 17 | 18 | 19 | org.aksw.simba.lsq 20 | lsq-core 21 | 22 | 23 | 24 | org.aksw.jenax 25 | jenax-models-geosparql 26 | 27 | 28 | 29 | junit 30 | junit 31 | test 32 | 33 | 34 | 36 | 37 | 38 | org.apache.logging.log4j 39 | log4j-slf4j-impl 40 | test 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /lsq-enrichers/src/main/java/org/aksw/simba/lsq/enricher/benchmark/opcache/OpExtKey.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.enricher.benchmark.opcache; 2 | 3 | import java.util.Objects; 4 | 5 | import org.apache.jena.atlas.io.IndentedWriter; 6 | import org.apache.jena.sparql.algebra.Op; 7 | import org.apache.jena.sparql.algebra.op.OpExt; 8 | import org.apache.jena.sparql.engine.ExecutionContext; 9 | import org.apache.jena.sparql.engine.QueryIterator; 10 | import org.apache.jena.sparql.serializer.SerializationContext; 11 | import org.apache.jena.sparql.util.NodeIsomorphismMap; 12 | 13 | /** 14 | * An operator that references a result set by key, such as a key of a cache entry. 15 | * The evaluation depends on the execution context or the executor 16 | * 17 | * @author raven 18 | * 19 | */ 20 | @Deprecated // Migrate to OpVar 21 | public class OpExtKey 22 | extends OpExt 23 | { 24 | protected Object key; 25 | 26 | public OpExtKey(Object key) { 27 | super(OpExtKey.class.getSimpleName()); 28 | this.key = key; 29 | } 30 | 31 | @Override 32 | public Op effectiveOp() { 33 | return null; 34 | } 35 | 36 | @Override 37 | public QueryIterator eval(QueryIterator input, ExecutionContext execCxt) { 38 | // ExprEvalException? 39 | throw new RuntimeException("This class requires an xecutor that can handle " + getClass().getName()); 40 | } 41 | 42 | @Override 43 | public void outputArgs(IndentedWriter out, SerializationContext sCxt) { 44 | out.print(key); 45 | } 46 | 47 | @Override 48 | public int hashCode() { 49 | //int result = tag.hashCode() * 13 + Objects.hashCode(key) * 11; 50 | int result = Objects.hash(tag, key); 51 | return result; 52 | } 53 | 54 | @Override 55 | public boolean equalTo(Op other, NodeIsomorphismMap labelMap) { 56 | return other instanceof OpExtKey && Objects.equals(key, ((OpExtKey)other).key); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /lsq-enrichers/src/main/java/org/aksw/simba/lsq/enricher/benchmark/opcache/OpExtTableToMultiset.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.enricher.benchmark.opcache; 2 | 3 | import org.apache.jena.atlas.io.IndentedWriter; 4 | import org.apache.jena.sparql.algebra.Op; 5 | import org.apache.jena.sparql.algebra.op.OpExt; 6 | import org.apache.jena.sparql.engine.ExecutionContext; 7 | import org.apache.jena.sparql.engine.QueryIterator; 8 | import org.apache.jena.sparql.serializer.SerializationContext; 9 | import org.apache.jena.sparql.util.NodeIsomorphismMap; 10 | 11 | public class OpExtTableToMultiset 12 | extends OpExt 13 | { 14 | protected Op subOp; 15 | 16 | 17 | public OpExtTableToMultiset(Op subOp) { 18 | super(OpExtTableToMultiset.class.getSimpleName()); 19 | this.subOp = subOp; 20 | } 21 | 22 | public Op getSubOp() { 23 | return subOp; 24 | } 25 | 26 | @Override 27 | public Op effectiveOp() { 28 | return null; 29 | } 30 | 31 | @Override 32 | public QueryIterator eval(QueryIterator input, ExecutionContext execCxt) { 33 | return null; 34 | } 35 | 36 | @Override 37 | public void outputArgs(IndentedWriter out, SerializationContext sCxt) { 38 | } 39 | 40 | @Override 41 | public int hashCode() { 42 | return 12345; 43 | } 44 | 45 | @Override 46 | public boolean equalTo(Op other, NodeIsomorphismMap labelMap) { 47 | return other instanceof OpExtTableToMultiset; 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /lsq-enrichers/src/main/java/org/aksw/simba/lsq/enricher/benchmark/opcache/TableMgr.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.enricher.benchmark.opcache; 2 | 3 | import java.io.ByteArrayInputStream; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.nio.charset.StandardCharsets; 7 | import java.util.Objects; 8 | 9 | import org.apache.jena.query.ResultSet; 10 | import org.apache.jena.riot.Lang; 11 | import org.apache.jena.riot.ResultSetMgr; 12 | import org.apache.jena.sparql.algebra.Table; 13 | import org.apache.jena.sparql.algebra.TableFactory; 14 | import org.apache.jena.sparql.engine.iterator.QueryIteratorResultSet; 15 | 16 | public class TableMgr { 17 | 18 | /** 19 | * Parse a table from a string. 20 | * 21 | * @param str 22 | * @param lang A Lang for a result set such as ResultSetLang.SPARQLResultSetJSON 23 | * @return 24 | */ 25 | public static Table parseTableFromString(String str, Lang lang) { 26 | Objects.requireNonNull(str); 27 | Table result = null; 28 | System.err.println("Parsing result set from string of length " + str.length()); 29 | try(InputStream in = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8))) { 30 | ResultSet rs = ResultSetMgr.read(in, lang); 31 | result = TableFactory.create(new QueryIteratorResultSet(rs)); 32 | } catch (IOException e) { 33 | // ByteArrayInputStream should never raise an IOException 34 | throw new RuntimeException(e); 35 | } 36 | 37 | return result; 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /lsq-enrichers/src/main/java/org/aksw/simba/lsq/enricher/core/LsqEnricher.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.enricher.core; 2 | 3 | import org.aksw.commons.lambda.serializable.SerializableFunction; 4 | import org.aksw.simba.lsq.model.LsqQuery; 5 | 6 | public interface LsqEnricher 7 | extends SerializableFunction //LsqQuery, LsqQuery? 8 | { 9 | } 10 | -------------------------------------------------------------------------------- /lsq-enrichers/src/main/java/org/aksw/simba/lsq/enricher/core/LsqEnricherFactory.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.enricher.core; 2 | 3 | import org.aksw.commons.lambda.serializable.SerializableSupplier; 4 | 5 | public interface LsqEnricherFactory 6 | extends SerializableSupplier 7 | { 8 | } 9 | -------------------------------------------------------------------------------- /lsq-enrichers/src/main/java/org/aksw/simba/lsq/enricher/core/LsqEnricherRegistry.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.enricher.core;import java.util.Collections; 2 | import java.util.Map; 3 | import java.util.Set; 4 | import java.util.concurrent.ConcurrentSkipListMap; 5 | 6 | /** A registry for {@link LsqEnricherFactory} instances. */ 7 | public class LsqEnricherRegistry { 8 | 9 | // Synchronized and insert-order-retaining 10 | protected Map registry = new ConcurrentSkipListMap<>(); 11 | 12 | private static LsqEnricherRegistry INSTANCE = null; 13 | 14 | /** Return the global instance */ 15 | public static LsqEnricherRegistry get() { 16 | if (INSTANCE == null) { 17 | synchronized (LsqEnricherRegistry.class) { 18 | if (INSTANCE == null) { 19 | INSTANCE = LsqEnricherRegistry.createDefault(); 20 | } 21 | } 22 | } 23 | return INSTANCE; 24 | } 25 | 26 | public void register(String name, LsqEnricherFactory factory) { 27 | registry.put(name, factory); 28 | } 29 | 30 | public LsqEnricherFactory get(String name) { 31 | return registry.get(name); 32 | } 33 | 34 | public LsqEnricherFactory getOrThrow(String name) { 35 | LsqEnricherFactory result = get(name); 36 | if (result == null) { 37 | throw new IllegalArgumentException("No enricher factory found for " + name); 38 | } 39 | return result; 40 | } 41 | 42 | public Set getKeys() { 43 | return Collections.unmodifiableSet(registry.keySet()); 44 | } 45 | 46 | public static LsqEnricherRegistry createDefault() { 47 | LsqEnricherRegistry result = new LsqEnricherRegistry(); 48 | initDefaults(result); 49 | return result; 50 | } 51 | 52 | public static LsqEnricherRegistry initDefaults(LsqEnricherRegistry registry) { 53 | registry.register("none", () -> lsqQuery -> lsqQuery); // Identity mapping 54 | registry.register("spin", () -> LsqEnrichments::enrichWithFullSpinModelCore); 55 | registry.register("static", () -> LsqEnrichments::enrichWithStaticAnalysis); 56 | registry.register("bbox", () -> LsqEnrichments::enrichWithBBox); 57 | return registry; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /lsq-enrichers/src/test/java/org/aksw/simba/lsq/TestLsqModel.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq; 2 | 3 | import org.aksw.jenax.arq.util.syntax.QueryHash; 4 | import org.aksw.jenax.reprogen.core.MapperProxyUtils; 5 | import org.aksw.simba.lsq.enricher.core.LsqEnrichments; 6 | import org.aksw.simba.lsq.model.LsqQuery; 7 | import org.apache.jena.query.Query; 8 | import org.apache.jena.query.QueryFactory; 9 | import org.apache.jena.rdf.model.Model; 10 | import org.apache.jena.rdf.model.ModelFactory; 11 | import org.apache.jena.riot.RDFDataMgr; 12 | import org.apache.jena.riot.RDFFormat; 13 | import org.junit.Test; 14 | 15 | public class TestLsqModel { 16 | @Test 17 | public void test01() { 18 | Model model = ModelFactory.createDefaultModel(); 19 | // Query query = QueryFactory.create("SELECT ?o ?p ?s { ?s ?p ?o }"); 20 | Query query = QueryFactory.create("SELECT ?p ?o { ?s ?p ?o } OFFSET 5 LIMIT 10"); 21 | LsqQuery before = model.createResource().as(LsqQuery.class); 22 | QueryHash hash = QueryHash.createHash(query); 23 | System.out.println(hash); 24 | 25 | if (true) { 26 | return; 27 | } 28 | 29 | before.setHash(hash.toString()); 30 | before.setQueryAndHash(query); 31 | before = MapperProxyUtils.skolemize("http://lsq.aksw.org/", before).as(LsqQuery.class); 32 | // before.setHash("--hash--"); 33 | before = LsqEnrichments.enrichWithFullSpinModelCore(before); 34 | before = LsqEnrichments.enrichWithStaticAnalysis(before); 35 | LsqQuery after = MapperProxyUtils.skolemize("http://lsq.aksw.org/", before).as(LsqQuery.class); 36 | RDFDataMgr.write(System.out, after.getModel(), RDFFormat.TURTLE_BLOCKS); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /lsq-legacy/README.md: -------------------------------------------------------------------------------- 1 | Components that are no longer in use but kept here for reference 2 | -------------------------------------------------------------------------------- /lsq-legacy/examples/lsq-example-swdf-todo-add-dataset/lsq-example-swdf.sparql: -------------------------------------------------------------------------------- 1 | PREFIX rdf: PREFIX swdf: SELECT DISTINCT ?prop WHERE { ?obj rdf:type swdf:SessionEvent . ?obj ?prop ?targetObj . FILTER (isLiteral(?targetObj)) } ORDER BY ?prop LIMIT 150 2 | -------------------------------------------------------------------------------- /lsq-legacy/examples/lsq-example-swdf-todo-add-dataset/run-lsq-example-swdf.sh: -------------------------------------------------------------------------------- 1 | lsq \ 2 | -f lsq-example-swdf.sparql \ 3 | -m sparql \ 4 | -e http://localhost:1900/sparql \ 5 | -g http://aksw.org/benchmark \ 6 | -l swdf \ 7 | -b http://lsq.aksw.org/res/ \ 8 | -p http://data.semanticweb.org/sparql \ 9 | -h 10 \ 10 | -t 60000 \ 11 | -r qel | rapper -i turtle -o ntriples - http://foo | sort -u > lsq-example-swdf.nt 12 | -------------------------------------------------------------------------------- /lsq-legacy/examples/lsqify-wikidata.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | lsq \ 4 | ../lsq-core/src/test/resources/logs/wikidata.wikidata.20170612-10.log \ 5 | -m wikidata \ 6 | -e https://query.wikidata.org/sparql \ 7 | -l wikidata \ 8 | -b http://lsq.aksw.org/res/ \ 9 | -p https://query.wikidata.org/sparql \ 10 | -a 'Linked Sparql Queries (lsq.aksw.org) ' \ 11 | -y 1000 \ 12 | -r qel \ 13 | -t 10000 14 | 15 | -------------------------------------------------------------------------------- /lsq-legacy/fixes/fix-datetime.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sed -r 's|"(.*)\^\^(.*)"\^\^|"\1"^^<\2>|g' 3 | #"2015-10-24T01:00:00Z^^http://www.w3.org/2001/XMLSchema#dateTime"^^ . 4 | -------------------------------------------------------------------------------- /lsq-legacy/fixes/fix-datetime.ttl: -------------------------------------------------------------------------------- 1 | 2 | "2015-10-24T01:00:00Z^^http://www.w3.org/2001/XMLSchema#dateTime"^^ . 3 | 4 | -------------------------------------------------------------------------------- /lsq-legacy/lsq-eval/endpoints.ttl: -------------------------------------------------------------------------------- 1 | ###SWDF 2 | lsqr:SWDF-2016-03-06 3 | lsqv:engine 4 | [ lsqv:vendor lsqr:Virtuoso ; lsqv:version "Virtuoso v.7.2" ; lsqv:processor "2.5GHz i7" ; lsqv:ram "8GB"] ; 5 | lsqv:dataset 6 | [ prov:hadPrimarySource ; 7 | prov:atTime "2016-03-06T23:09:10+01:00"^^xsd:dateTimeStamp ] ; 8 | prov:startedAtTime "2016-03-06T23:09:02+01:00"^^xsd:dateTimeStamp ; 9 | prov:endAtTime "2016-03-06T23:09:10+01:00"^^xsd:dateTimeStamp . 10 | 11 | 12 | 13 | ###DBpedia 14 | lsqr:DBpedia-2016-03-06 15 | lsqv:engine 16 | [ lsqv:vendor lsqr:Virtuoso ; lsqv:version "Virtuoso v.7.2" ; lsqv:processor "2.5GHz i7" ; lsqv:ram "8GB"] ; 17 | lsqv:dataset 18 | [ prov:hadPrimarySource ; 19 | prov:atTime "2016-03-06T23:09:10+01:00"^^xsd:dateTimeStamp ] ; 20 | prov:startedAtTime "2016-03-06T23:09:02+01:00"^^xsd:dateTimeStamp ; 21 | prov:endAtTime "2016-03-06T23:09:10+01:00"^^xsd:dateTimeStamp . 22 | 23 | -------------------------------------------------------------------------------- /lsq-legacy/lsq-eval/executions-per-agent.rq: -------------------------------------------------------------------------------- 1 | PREFIX lsqv: 2 | PREFIX sd: 3 | SELECT * { 4 | SELECT ?g ?endpoint (COUNT(DISTINCT ?query) AS ?queryCount) 5 | WHERE { 6 | GRAPH ?g { 7 | ?query lsqv:hasRemoteExec ?rexec . 8 | ?rexec lsqv:hostHash ?hash . 9 | ?rexec lsqv:endpoint ?endpoint . 10 | } 11 | } 12 | GROUP BY ?g ?endpoint ?hash 13 | } 14 | ORDER BY ASC(?g) ASC(?endpoint) DESC(?queryCount) 15 | 16 | -------------------------------------------------------------------------------- /lsq-legacy/lsq-eval/hosts-per-endpoint.rq: -------------------------------------------------------------------------------- 1 | PREFIX lsqv: 2 | 3 | SELECT ?g ?endpoint (COUNT(DISTINCT ?host) AS ?hostCount) 4 | WHERE { 5 | 6 | # { SELECT DISTINCT ?query { GRAPH ?g { ?query lsqv:hasRemoteExec [] } } limit 10 } 7 | 8 | GRAPH ?g { 9 | ?query 10 | lsqv:hasRemoteExec [ 11 | lsqv:hostHash ?host ; 12 | lsqv:endpoint ?endpoint 13 | ] 14 | } 15 | } 16 | GROUP BY ?g ?endpoint 17 | 18 | -------------------------------------------------------------------------------- /lsq-legacy/lsq-eval/query-count-per-host-count.rq: -------------------------------------------------------------------------------- 1 | PREFIX lsqv: 2 | 3 | SELECT ?g ?endpoint ?queryCount (COUNT(DISTINCT ?hostHash) AS ?hostCountPerQueryCount) { 4 | SELECT ?g ?endpoint ?hostHash (COUNT(DISTINCT ?query) AS ?queryCount) { 5 | 6 | # { SELECT DISTINCT ?query { ?query lsqv:hasRemoteExec ?re } limit 10 } 7 | 8 | GRAPH ?g { 9 | ?query 10 | lsqv:hasRemoteExec [ 11 | lsqv:endpoint ?endpoint ; 12 | lsqv:hostHash ?hostHash 13 | ] 14 | } 15 | 16 | 17 | } GROUP BY ?g ?endpoint ?hostHash 18 | } GROUP BY ?g ?endpoint ?queryCount 19 | ORDER BY ?g ?endpoint DESC(?queryCount) DESC(?hostCountPerQueryCount) 20 | 21 | 22 | -------------------------------------------------------------------------------- /lsq-legacy/lsq-eval/query-count-per-host.rq: -------------------------------------------------------------------------------- 1 | PREFIX lsqv: 2 | 3 | SELECT * { 4 | SELECT ?g ?endpoint ?hostHash (COUNT(DISTINCT ?query) AS ?queryCount) { 5 | 6 | # { SELECT DISTINCT ?query { ?query lsqv:hasRemoteExec ?re } limit 10 } 7 | 8 | GRAPH ?g { 9 | ?query 10 | lsqv:hasRemoteExec [ 11 | lsqv:endpoint ?endpoint ; 12 | lsqv:hostHash ?hostHash 13 | ] 14 | } 15 | 16 | 17 | } GROUP BY ?g ?endpoint ?hostHash 18 | } 19 | ORDER BY ?g ?endpoint DESC(?queryCount) ?hostHash 20 | 21 | 22 | -------------------------------------------------------------------------------- /lsq-legacy/lsq-eval/saleem-environment.ttl: -------------------------------------------------------------------------------- 1 | ###SWDF 2 | @prefix lsqr: . 3 | @prefix lsqv: . 4 | 5 | 6 | lsqr:swdf 7 | lsqv:engine 8 | [ lsqv:vendor lsqr:Virtuoso ; lsqv:version "Virtuoso v.7.2" ; lsqv:processor "2.5GHz i7" ; lsqv:ram "8GB"] . 9 | 10 | 11 | ###DBpedia 12 | lsqr:dbpedia 13 | lsqv:engine 14 | [ lsqv:vendor lsqr:Virtuoso ; lsqv:version "Virtuoso v.7.2" ; lsqv:processor "2.5GHz i7" ; lsqv:ram "8GB"] . 15 | -------------------------------------------------------------------------------- /lsq-legacy/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | lsq-legacy 8 | jar 9 | 10 | 11 | org.aksw.simba.lsq 12 | lsq-parent 13 | 2.0.0-SNAPSHOT 14 | 15 | 16 | 18 | 19 | https://github.com/dice-group/LSQ 20 | scm:git:git@github.com:dice-group/LSQ.git 21 | scm:git:git@github.com:dice-group/LSQ.git 22 | HEAD 23 | 24 | 25 | 26 | 27 | 28 | org.aksw.simba.lsq 29 | lsq-cli 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /lsq-legacy/run-example-from-deb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | file=`find lsq-cli -name '*-jar-with-dependencies.jar'` 3 | 4 | command_exists () { command -v "$1" >/dev/null 2>&1; } 5 | 6 | if ! command_exists "lsq"; then 7 | ( mvn clean install && ./reinstall-deb.sh ) 8 | fi 9 | 10 | # -f lsq-core/src/test/resources/logs/combined.swdf.log \ 11 | lsq \ 12 | -e http://localhost:8890/sparql \ 13 | -l swdf \ 14 | -b http://lsq.aksw.org/res/ \ 15 | -p http://localhost/service/org.semanticweb.swdf_swdf-full_latest_public_sparql \ 16 | -h 100 \ 17 | -r qel \ 18 | -t 10000 \ 19 | lsq-core/src/test/resources/logs/combined.swdf.log | rapper -i turtle -o turtle - http://foo 20 | 21 | -------------------------------------------------------------------------------- /lsq-legacy/run-example-from-jar.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | file=`find lsq-cli -name '*-jar-with-dependencies.jar'` 3 | 4 | if [ ! -f "$file" ]; then 5 | ( mvn clean install && cd lsq-cli && mvn assembly:assembly ) 6 | fi 7 | 8 | file=`find lsq-cli -name '*-jar-with-dependencies.jar'` 9 | 10 | #echo "$file" 11 | 12 | java -cp "$file" org.aksw.simba.lsq.cli.main.MainLSQ \ 13 | -e http://localhost:8890/sparql \ 14 | -l swdf \ 15 | -b http://lsq.aksw.org/res/ \ 16 | -p http://localhost/service/org.semanticweb.swdf_swdf-full_latest_public_sparql \ 17 | -h 100 \ 18 | -r qel \ 19 | -t 10000 \ 20 | lsq-core/src/test/resources/logs/combined.swdf.log | rapper -i turtle -o turtle - http://foo 21 | 22 | -------------------------------------------------------------------------------- /lsq-legacy/run-example.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | file=`find lsq-cli -name '*-jar-with-dependencies.jar'` 3 | 4 | if [ ! -f "$file" ]; then 5 | ( mvn clean install && cd lsq-cli && mvn assembly:assembly ) 6 | fi 7 | 8 | file=`find lsq-cli -name '*-jar-with-dependencies.jar'` 9 | 10 | #echo "$file" 11 | 12 | java -cp "$file" org.aksw.simba.lsq.cli.main.MainLSQ \ 13 | -f lsq-core/src/test/resources/logs/combined.swdf.log \ 14 | -e http://localhost:8890/sparql \ 15 | -l swdf \ 16 | -b http://lsq.aksw.org/res/ \ 17 | -p http://localhost/service/org.semanticweb.swdf_swdf-full_latest_public_sparql \ 18 | -h 100 \ 19 | -r qel \ 20 | -t 10000 | rapper -i turtle -o turtle - http://foo 21 | 22 | -------------------------------------------------------------------------------- /lsq-legacy/src/main/java/org/aksw/simba/lsq/TestLsqQueryJoinType.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq; 2 | 3 | import org.aksw.simba.lsq.enrich.core.QueryStatistics2; 4 | import org.aksw.simba.lsq.spinx.model.SpinQueryEx; 5 | import org.aksw.simba.lsq.util.SpinUtilsOld; 6 | import org.apache.jena.query.Query; 7 | import org.apache.jena.query.QueryFactory; 8 | import org.apache.jena.rdf.model.Model; 9 | import org.apache.jena.rdf.model.ModelFactory; 10 | import org.apache.jena.rdf.model.Resource; 11 | import org.apache.jena.riot.RDFDataMgr; 12 | import org.apache.jena.riot.RDFFormat; 13 | import org.spinrdf.arq.ARQ2SPIN; 14 | 15 | public class TestLsqQueryJoinType { 16 | //@Test 17 | public void test() { 18 | String queryStr = "PREFIX tcga: SELECT ?expValue WHERE { { ?s tcga:bcr_patient_barcode . tcga:result ?results. ?results tcga:RPKM ?expValue. } UNION { ?uri tcga:bcr_patient_barcode . tcga:result ?geneResults. ?geneResults tcga:scaled_estimate ?expValue. } }"; 19 | Query query = QueryFactory.create(queryStr); 20 | 21 | Model model = ModelFactory.createDefaultModel(); 22 | Resource featureRes = model.createResource(); 23 | 24 | ARQ2SPIN arq2spin = new ARQ2SPIN(model); 25 | SpinQueryEx spinRes = arq2spin.createQuery(query, null).as(SpinQueryEx.class); 26 | 27 | 28 | Resource outputRes = model.createResource(); 29 | 30 | SpinUtilsOld.enrichWithHasTriplePattern(spinRes, spinRes); 31 | //QueryStatistics2.enrichResourceWithQueryFeatures(featureRes, query); 32 | QueryStatistics2.getDirectQueryRelatedRDFizedStats(spinRes, spinRes); 33 | RDFDataMgr.write(System.out, spinRes.getModel(), RDFFormat.TURTLE); 34 | 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /lsq-legacy/src/main/java/org/aksw/simba/lsq/cli/cmd/experimental/CmdLsqRehash.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.cli.cmd.experimental; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.concurrent.Callable; 6 | 7 | import org.aksw.simba.lsq.cli.main.MainCliLsq; 8 | 9 | import picocli.CommandLine.Command; 10 | import picocli.CommandLine.Option; 11 | import picocli.CommandLine.Parameters; 12 | 13 | /** 14 | * For a given {@link org.aksw.simba.lsq.model.LsqQuery} read out its hash, then 15 | * generate a new hash using the latest approach. If the new hash differes, then 16 | * replace any occurrences of the hash in any of the underlying model's RDF 17 | * terms with the new hash 18 | * 19 | * @author raven 20 | * 21 | */ 22 | //@Command(name = "rehash", description = "Update IDs of lsq query objects generated with older versions of LSQ") 23 | //public class CmdLsqRehash implements Callable { 24 | // @Option(names = { "-h", "--help" }, usageHelp = true) 25 | // public boolean help = false; 26 | // 27 | // @Parameters(arity = "0..*", description = "file-list to probe") 28 | // public List nonOptionArgs = new ArrayList<>(); 29 | // 30 | // @Override 31 | // public Integer call() throws Exception { 32 | // MainCliLsq.rehash(this); 33 | // return 0; 34 | // } 35 | //} 36 | -------------------------------------------------------------------------------- /lsq-legacy/src/main/java/org/aksw/simba/lsq/cli/cmd/experimental/CmdLsqSparkRehash.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.cli.cmd.experimental; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.concurrent.Callable; 6 | 7 | import org.aksw.simba.lsq.cli.cmd.rx.api.CmdRdfIoBase; 8 | import org.aksw.simba.lsq.cli.main.MainCliLsq; 9 | import org.aksw.simba.lsq.cli.trash.CmdLsqRehashSparkImpl; 10 | 11 | import picocli.CommandLine.Command; 12 | import picocli.CommandLine.Parameters; 13 | 14 | /** 15 | * For a given {@link org.aksw.simba.lsq.model.LsqQuery} read out its hash, then 16 | * generate a new hash using the latest approach. If the new hash differes, then 17 | * replace any occurrences of the hash in any of the underlying model's RDF 18 | * terms with the new hash 19 | * 20 | * @author raven 21 | * 22 | */ 23 | @Command(name = "rehash", description = "Update IDs of lsq query objects generated with older versions of LSQ") 24 | public class CmdLsqSparkRehash 25 | extends CmdRdfIoBase 26 | implements Callable 27 | { 28 | @Override 29 | public Integer call() throws Exception { 30 | CmdLsqRehashSparkImpl.mainSpark(this); 31 | return 0; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /lsq-legacy/test-lsq-cli.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | lsq probe lsq-core/src/test/resources/logs/virtuoso.dbpedia.log 4 | lsq rdfize lsq-core/src/test/resources/logs/virtuoso.dbpedia.log 5 | 6 | 7 | -------------------------------------------------------------------------------- /lsq-model/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | lsq-model 8 | jar 9 | 10 | 11 | org.aksw.simba.lsq 12 | lsq-parent 13 | 2.0.0-RC5-SNAPSHOT 14 | 15 | 16 | 17 | 18 | 19 | org.aksw.jenax 20 | jenax-conjure-core 21 | 22 | 23 | 24 | org.aksw.simba.lsq 25 | lsq-vocab-jena 26 | 27 | 28 | 32 | 33 | 34 | com.google.guava 35 | guava 36 | 37 | 38 | 39 | org.aksw.thirdparty.org.spinrdf 40 | spinrdf 41 | 42 | 43 | 44 | org.aksw.jenax 45 | jenax-reprogen-core 46 | 47 | 48 | 49 | org.aksw.jenax 50 | jenax-models-geosparql 51 | 52 | 53 | 54 | junit 55 | junit 56 | test 57 | 58 | 59 | 60 | org.apache.logging.log4j 61 | log4j-slf4j-impl 62 | test 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/model/ElementExec.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.model; 2 | 3 | import org.aksw.jenax.annotation.reprogen.HashId; 4 | import org.aksw.jenax.annotation.reprogen.Iri; 5 | import org.aksw.simba.lsq.vocab.LSQ; 6 | import org.apache.jena.rdf.model.Resource; 7 | 8 | /** 9 | * This effectively represents an observation of the metrics of a query having 10 | * been executed on some endpoint in some benchmark experiment 11 | * 12 | * @author raven 13 | * 14 | */ 15 | public interface ElementExec 16 | extends Resource 17 | { 18 | // @Iri(LSQ.Strs.benchmarkRun) 19 | // ExperimentRun getBenchmarkRun(); 20 | // ElementExec setBenchmarkRun(Resource benchmarkRun); 21 | 22 | @HashId 23 | @Iri(LSQ.Terms.hasElementExec) 24 | QueryExec getQueryExec(); 25 | ElementExec setQueryExec(QueryExec queryExec); 26 | // We should use seconds (standard SI units) 27 | 28 | // Link to the element whole extension query was executed 29 | // Use element.getExtensionQuery to get the the query itself 30 | // LsqElement getElement(); 31 | } 32 | -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/model/ExperimentExec.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.model; 2 | 3 | import java.text.DateFormat; 4 | import java.text.SimpleDateFormat; 5 | import java.util.Calendar; 6 | import java.util.Objects; 7 | 8 | import org.aksw.jenax.annotation.reprogen.HashId; 9 | import org.aksw.jenax.annotation.reprogen.Iri; 10 | import org.aksw.jenax.annotation.reprogen.ResourceView; 11 | import org.aksw.jenax.annotation.reprogen.StringId; 12 | import org.aksw.jenax.reprogen.hashid.HashIdCxt; 13 | import org.aksw.simba.lsq.vocab.LSQ; 14 | import org.apache.jena.datatypes.xsd.XSDDateTime; 15 | import org.apache.jena.rdf.model.Resource; 16 | 17 | /** A benchmark exec is linked to by 1 or more runs */ 18 | @ResourceView 19 | public interface ExperimentExec extends Resource { 20 | @HashId 21 | @Iri(LSQ.Terms.benchmarkConfig) 22 | ExperimentConfig getConfig(); 23 | ExperimentExec setConfig(Resource experimentConfig); 24 | 25 | // @HashId 26 | // @Iri(LSQ.Terms.runId) 27 | // Integer getRunId(); 28 | // ExperimentRun setRunId(Integer runId); 29 | 30 | // Optionally track start and end time of a run 31 | @HashId 32 | @Iri(LSQ.Terms.atTime) 33 | XSDDateTime getTimestamp(); 34 | ExperimentExec setTimestamp(XSDDateTime calendar); 35 | 36 | // 17/Apr/2011:06:47:47 +0200 37 | public static final DateFormat dateFormat = new SimpleDateFormat("dd-MM-yyyy_HH:mm:ss"); 38 | 39 | default String getIdentifier() { 40 | ExperimentConfig config = getConfig(); 41 | String configId = config.getIdentifier(); 42 | 43 | XSDDateTime dt = getTimestamp(); 44 | Calendar cal = dt.asCalendar(); 45 | 46 | String timestampStr = dateFormat.format(cal.getTime()); 47 | String runId = configId + "_" + timestampStr; 48 | return runId; 49 | } 50 | 51 | @StringId 52 | default String getStringId(HashIdCxt cxt) { 53 | ExperimentConfig config = getConfig(); 54 | String id = config.getIdentifier(); 55 | Calendar cal = Objects.requireNonNull(getTimestamp(), "no timestamp given").asCalendar(); 56 | String timestamp = dateFormat.format(cal.getTime()); 57 | // String prefix = StringUtils.toLowerCamelCase(getClass().getSimpleName()); // 58 | // "" 59 | 60 | String result = id; 61 | // Integer runId = getRunId(); 62 | // if (runId != null) { 63 | // result += "_run" + runId; 64 | // } 65 | 66 | result += "_at_" + timestamp; 67 | return result; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/model/Host.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.model; 2 | 3 | import org.aksw.jenax.annotation.reprogen.Iri; 4 | import org.aksw.jenax.annotation.reprogen.ResourceView; 5 | import org.aksw.simba.lsq.vocab.LSQ; 6 | import org.apache.jena.rdf.model.Resource; 7 | 8 | @ResourceView 9 | public interface Host 10 | extends Resource 11 | { 12 | @Iri(LSQ.Terms.host) 13 | String getHost(); 14 | RemoteExecution setHost(String host); 15 | } -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/model/JoinVertex.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.model; 2 | 3 | import org.aksw.jenax.annotation.reprogen.Iri; 4 | import org.aksw.simba.lsq.vocab.LSQ; 5 | import org.apache.jena.rdf.model.Resource; 6 | import org.spinrdf.model.Variable; 7 | 8 | public interface JoinVertex 9 | extends Resource 10 | { 11 | @Iri(LSQ.Terms.proxyFor) 12 | Variable getSpinVariable(); 13 | 14 | @Iri(LSQ.Terms.joinVertexDegree) 15 | Integer getJoinVertexDegree(); 16 | 17 | @Iri(LSQ.Terms.joinVertexType) 18 | Resource getJoinVertexType(); 19 | JoinVertex setJoinVertexType(Resource joinVertexType); 20 | 21 | //LSQ.joinVertexDegree, degree).addProperty(LSQ.joinVertexType 22 | } 23 | -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/model/QualifiedFeature.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.model; 2 | 3 | import org.aksw.jenax.annotation.reprogen.Iri; 4 | import org.aksw.jenax.annotation.reprogen.ResourceView; 5 | import org.aksw.simba.lsq.vocab.LSQ; 6 | import org.apache.jena.rdf.model.Resource; 7 | 8 | @ResourceView 9 | public interface QualifiedFeature 10 | extends Resource 11 | { 12 | @Iri(LSQ.Terms.feature) 13 | String getFeature(); 14 | QualifiedFeature setFeature(String host); 15 | 16 | @Iri(LSQ.Terms.count) 17 | Integer getCount(); 18 | QualifiedFeature setCount(Integer count); 19 | 20 | // 21 | default int incrementAndGet() { 22 | Integer count = getCount(); 23 | int nextCount = count == null ? 1 : ++count; 24 | setCount(nextCount); 25 | return count; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/model/RemoteExecution.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.model; 2 | 3 | import java.util.Calendar; 4 | 5 | import org.aksw.commons.io.util.UriToPathUtils; 6 | import org.aksw.jenax.annotation.reprogen.HashId; 7 | import org.aksw.jenax.annotation.reprogen.Iri; 8 | import org.aksw.jenax.annotation.reprogen.IriType; 9 | import org.aksw.jenax.annotation.reprogen.ResourceView; 10 | import org.aksw.jenax.annotation.reprogen.StringId; 11 | import org.aksw.jenax.reprogen.hashid.HashIdCxt; 12 | import org.aksw.simba.lsq.vocab.LSQ; 13 | import org.apache.jena.rdf.model.Resource; 14 | 15 | @ResourceView 16 | public interface RemoteExecution 17 | extends Resource 18 | { 19 | static String getLogEntryId(RemoteExecution re) { 20 | String serviceUrl = re.getEndpointUrl(); 21 | 22 | String serviceId = serviceUrl == null 23 | ? "unknown-service" 24 | : UriToPathUtils.resolvePath(serviceUrl).toString() 25 | .replace('/', '-'); 26 | 27 | Long seqId = re.getSequenceId(); 28 | Calendar timestamp = re.getTimestamp(); 29 | // TODO If there is a timestamp then use it 30 | // Otherwise, use sourceFileName + sequenceId 31 | String logEntryId = serviceId + "_" + (timestamp != null 32 | ? timestamp.toInstant().toString() 33 | : seqId); 34 | 35 | return logEntryId; 36 | } 37 | 38 | @StringId 39 | default String getStringId(HashIdCxt cxt) { 40 | String prefix = "remoteExec"; 41 | 42 | String logEntryId = getLogEntryId(this); 43 | String result = prefix + "-" + logEntryId; 44 | 45 | return result; 46 | } 47 | 48 | @Iri(LSQ.Terms.host) 49 | @HashId 50 | String getHost(); 51 | RemoteExecution setHost(String host); 52 | 53 | @Iri(LSQ.Terms.userAgent) 54 | String getUserAgent(); 55 | RemoteExecution setUserAgent(String userAgent); 56 | 57 | @Iri(LSQ.Terms.hostHash) 58 | @HashId 59 | String getHostHash(); 60 | RemoteExecution setHostHash(String hostHash); 61 | 62 | @Iri(LSQ.Terms.atTime) 63 | @HashId 64 | Calendar getTimestamp(); 65 | RemoteExecution setTimestamp(Calendar calendar); 66 | 67 | @Iri(LSQ.Terms.endpoint) 68 | @IriType 69 | @HashId 70 | String getEndpointUrl(); 71 | RemoteExecution setEndpointUrl(String endpointUrl); 72 | 73 | @Iri(LSQ.Terms.sequenceId) 74 | @HashId 75 | Long getSequenceId(); 76 | RemoteExecution setSequenceId(Long value); 77 | 78 | @Iri(LSQ.Terms.headers) 79 | RemoteExecutionHeaders getHeaders(); 80 | } 81 | -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/model/RemoteExecutionHeaders.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.model; 2 | 3 | import org.aksw.jenax.annotation.reprogen.HashId; 4 | import org.aksw.jenax.annotation.reprogen.Inverse; 5 | import org.aksw.jenax.annotation.reprogen.Iri; 6 | import org.aksw.jenax.annotation.reprogen.ResourceView; 7 | import org.aksw.jenax.annotation.reprogen.StringId; 8 | import org.aksw.jenax.reprogen.hashid.HashIdCxt; 9 | import org.aksw.simba.lsq.vocab.LSQ; 10 | import org.apache.jena.rdf.model.Resource; 11 | 12 | @ResourceView 13 | @HashId 14 | public interface RemoteExecutionHeaders 15 | extends Resource 16 | { 17 | @Inverse 18 | @Iri(LSQ.Terms.headers) 19 | @HashId(excludeRdfProperty = true) 20 | RemoteExecution getRemoteExecution(); 21 | 22 | @StringId 23 | default String getStringId(HashIdCxt cxt) { 24 | RemoteExecution remoteExecution = getRemoteExecution(); 25 | String prefix = "remoteExecHeaders"; 26 | String result = prefix + "-" + RemoteExecution.getLogEntryId(remoteExecution); 27 | return result; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/spinx/model/BgpInfo.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.spinx.model; 2 | 3 | import java.util.Set; 4 | 5 | import org.aksw.jenax.annotation.reprogen.Iri; 6 | import org.aksw.jenax.annotation.reprogen.ResourceView; 7 | import org.aksw.simba.lsq.vocab.LSQ; 8 | import org.apache.jena.rdf.model.Resource; 9 | 10 | @ResourceView 11 | public interface BgpInfo 12 | extends Resource 13 | { 14 | @Iri(LSQ.Terms.hasBgp) 15 | Set getBgps(); 16 | } 17 | -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/spinx/model/BgpNodeExec.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.spinx.model; 2 | 3 | import java.math.BigDecimal; 4 | 5 | import org.aksw.commons.util.string.StringUtils; 6 | import org.aksw.jenax.annotation.reprogen.HashId; 7 | import org.aksw.jenax.annotation.reprogen.Inverse; 8 | import org.aksw.jenax.annotation.reprogen.Iri; 9 | import org.aksw.jenax.annotation.reprogen.ResourceView; 10 | import org.aksw.jenax.annotation.reprogen.StringId; 11 | import org.aksw.jenax.reprogen.hashid.HashIdCxt; 12 | import org.aksw.simba.lsq.model.ElementExec; 13 | import org.aksw.simba.lsq.model.ExperimentRun; 14 | import org.aksw.simba.lsq.model.LocalExecution; 15 | import org.aksw.simba.lsq.vocab.LSQ; 16 | 17 | 18 | @ResourceView 19 | public interface BgpNodeExec 20 | extends ElementExec 21 | { 22 | @Iri(LSQ.Terms.hasExec) 23 | @Inverse 24 | @HashId // The id is based on having executed on this node (the link to the queryExec is in the base class) 25 | BgpNode getBgpNode(); 26 | BgpNodeExec setBgpNode(BgpNode bpgNode); 27 | 28 | 29 | @Iri(LSQ.Terms.tpSelJoinVarRestricted) 30 | BigDecimal getBgpRestrictedSelectivitiy(); 31 | BgpNodeExec setBgpRestrictedSelectivitiy(BigDecimal selectivity); 32 | 33 | 34 | @Iri(LSQ.Terms.hasJoinVarExec) 35 | @Inverse 36 | BgpExec getBgpExec(); 37 | BgpNodeExec setBgpExec(BgpExec exec); 38 | 39 | 40 | @Iri(LSQ.Terms.hasSubBgpExec) 41 | BgpExec getSubBgpExec(); 42 | BgpNodeExec setSubBgpExec(BgpExec exec); 43 | 44 | 45 | @StringId 46 | default String getStringId(HashIdCxt cxt) { 47 | 48 | // SpinBgpExec bgpPart = getBgpExec(); // .getBgp(); 49 | // BgpNode bgpNode = getBgpNode(); 50 | // TODO Replace the prefix with e.g. cxt.getClassLabel(SpinBgpExec.class) 51 | // String result = "bgpExec-" + cxt.getHashAsString(bgp) + "-" + getLocalExecution().getBenchmarkRun().getIdentifier(); 52 | 53 | //SpinBgpExec bgpExec = getBgpExec(); 54 | //LocalExecution le = bgpExec.getQueryExec().getLocalExecution(); 55 | LocalExecution le = getQueryExec().getLocalExecution(); 56 | ExperimentRun bmr = le.getBenchmarkRun(); 57 | // String prefix = StringUtils.toLowerCamelCase(getClass().getSimpleName()); // "bgpNodeExec-" 58 | String prefix = StringUtils.toLowerCamelCase(BgpNodeExec.class.getSimpleName()); 59 | String result = prefix + "-" + cxt.getHashAsString(this) + "-" + cxt.getStringId(bmr); 60 | return result; 61 | } 62 | 63 | // 64 | // SpinBgpExec getBgpNodeExec(); 65 | // JoinVertexExec setBgpNodeExec(SpinBgp exec); 66 | 67 | } 68 | -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/spinx/model/DirectedHyperEdge.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.spinx.model; 2 | 3 | import java.util.Set; 4 | 5 | import org.aksw.jenax.annotation.reprogen.HashId; 6 | import org.aksw.jenax.annotation.reprogen.Iri; 7 | import org.aksw.jenax.annotation.reprogen.ResourceView; 8 | import org.aksw.simba.lsq.vocab.LSQ; 9 | import org.apache.jena.rdf.model.Resource; 10 | 11 | @ResourceView 12 | public interface DirectedHyperEdge 13 | extends Resource 14 | { 15 | @Iri(LSQ.Terms.in) 16 | @HashId 17 | Set getInNodes(); 18 | 19 | @Iri(LSQ.Terms.out) 20 | @HashId 21 | Set getOutNodes(); 22 | } 23 | -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/spinx/model/JoinVertex.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.spinx.model; 2 | 3 | import java.util.Set; 4 | 5 | import org.aksw.jenax.annotation.reprogen.ResourceView; 6 | import org.apache.jena.rdf.model.Resource; 7 | 8 | @ResourceView 9 | public interface JoinVertex 10 | extends Resource 11 | { 12 | Set getJoinVertexExecs(); 13 | } 14 | -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/spinx/model/Labeled.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.spinx.model; 2 | 3 | import org.aksw.jenax.annotation.reprogen.Iri; 4 | 5 | public interface Labeled 6 | { 7 | @Iri("rdfs:label") 8 | Labeled setLabel(String label); 9 | String getLabel(); 10 | } 11 | -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/spinx/model/LsqElement.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.spinx.model; 2 | 3 | import org.aksw.jenax.annotation.reprogen.Iri; 4 | import org.aksw.simba.lsq.model.LsqQuery; 5 | import org.aksw.simba.lsq.vocab.LSQ; 6 | import org.apache.jena.rdf.model.Resource; 7 | 8 | /** 9 | * Base interface for all resources that denote SPARQL elements that can be related to a query 10 | * that corresponds to that element's set of solution bindings. 11 | * 12 | * The extension query is always a SPARQL SELECT query. 13 | * 14 | * @author raven 15 | * 16 | */ 17 | public interface LsqElement 18 | extends Resource, Labeled 19 | { 20 | @Iri(LSQ.Terms.extensionQuery) 21 | LsqQuery getExtensionQuery(); 22 | LsqElement setExtensionQuery(LsqQuery r); 23 | } 24 | -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/spinx/model/LsqTriplePattern.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.spinx.model; 2 | 3 | import java.nio.charset.StandardCharsets; 4 | import java.util.Objects; 5 | import java.util.Set; 6 | 7 | import org.aksw.jenax.annotation.reprogen.HashId; 8 | import org.aksw.jenax.annotation.reprogen.Iri; 9 | import org.aksw.jenax.annotation.reprogen.ResourceView; 10 | import org.aksw.jenax.reprogen.hashid.HashIdCxt; 11 | import org.aksw.simba.lsq.model.util.SpinCoreUtils; 12 | import org.aksw.simba.lsq.vocab.LSQ; 13 | import org.apache.jena.enhanced.EnhGraph; 14 | import org.apache.jena.graph.Node; 15 | import org.apache.jena.graph.Triple; 16 | import org.spinrdf.model.impl.TriplePatternImpl; 17 | 18 | import com.google.common.hash.HashCode; 19 | 20 | @ResourceView 21 | public abstract class LsqTriplePattern 22 | extends TriplePatternImpl 23 | implements LsqElement 24 | { 25 | public LsqTriplePattern(Node node, EnhGraph graph) { 26 | super(node, graph); 27 | } 28 | 29 | public Triple toJenaTriple() { 30 | Triple result = SpinCoreUtils.toJenaTriple(this); 31 | return result; 32 | } 33 | 34 | @HashId 35 | public HashCode getHashId(HashIdCxt cxt) { 36 | Triple t = SpinCoreUtils.toJenaTriple(this); 37 | HashCode result = cxt.getHashFunction().hashString(Objects.toString(t), StandardCharsets.UTF_8); 38 | // System.out.println("LsqTriplePattern: " + t + " -> " + result); 39 | return result; 40 | } 41 | 42 | @Iri(LSQ.Terms.hasExec) 43 | public abstract Set getTpExecs(); 44 | 45 | // public TpExec findTpExec(Resource expRun) { 46 | // return null; 47 | // } 48 | // default JoinVertexExec findTpExec(Resource expRun) { 49 | // Resource expRun = getBenchmarkRun(); 50 | // Objects.requireNonNull(expRun); 51 | // 52 | // Set cands = getBgpNodeExecs(); 53 | // JoinVertexExec result = null; 54 | // for(JoinVertexExec cand : cands) { 55 | // if(Objects.equals(cand.getBgpNode(), bgp) && Objects.equals(cand.getQueryExec().getBenchmarkRun(), expRun)) { 56 | // result = cand; 57 | // break; 58 | // } 59 | // } 60 | // 61 | // return result; 62 | // } 63 | } 64 | -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/spinx/model/SpinQueryEx.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.spinx.model; 2 | 3 | import org.aksw.jenax.annotation.reprogen.ResourceView; 4 | 5 | 6 | // Deprecated; the information captured by this class went to {@link LsqStructuralFeatures} 7 | @ResourceView 8 | public interface SpinQueryEx 9 | extends BgpInfo 10 | { 11 | // @Iri(LSQ.Strs.hasBGP) 12 | // Set getBgps(); 13 | // 14 | // @Iri(LSQ.Strs.bgpCount) 15 | // Integer getTotalBgpCount(); 16 | // SpinQueryEx setTotalBgpCount(Integer cnt); 17 | // 18 | // @Iri(LSQ.Strs.tpInBgpCountMin) 19 | // Integer getMinBgpTriples(); 20 | // SpinQueryEx setMinBgpTriples(Integer cnt); 21 | // 22 | // @Iri(LSQ.Strs.tpInBgpCountMax) 23 | // Integer getMaxBgpTriples(); 24 | // SpinQueryEx setMaxBgpTriples(Integer cnt); 25 | // 26 | // @Iri(LSQ.Strs.tpCount) 27 | // Integer getTriplePatternCount(); 28 | // SpinQueryEx setTriplePatternCount(Integer cnt); 29 | } 30 | -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/spinx/model/SpinVarOrLiteral.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.spinx.model; 2 | 3 | import org.aksw.jenax.annotation.reprogen.HashId; 4 | import org.aksw.jenax.annotation.reprogen.IriNs; 5 | import org.aksw.jenax.annotation.reprogen.ResourceView; 6 | import org.apache.jena.rdf.model.RDFNode; 7 | import org.apache.jena.rdf.model.Resource; 8 | import org.spinrdf.vocabulary.SP; 9 | 10 | @ResourceView 11 | @HashId 12 | public interface SpinVarOrLiteral 13 | extends Resource 14 | { 15 | @IriNs(SP.NS) 16 | @HashId 17 | String getVarName(); 18 | SpinVarOrLiteral setvarName(String name); 19 | 20 | @Override 21 | default boolean canAs(Class view) { 22 | String varName = getVarName(); 23 | boolean result = varName != null; 24 | return result; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/spinx/model/TpExec.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.spinx.model; 2 | 3 | import java.math.BigDecimal; 4 | 5 | import org.aksw.commons.util.string.StringUtils; 6 | import org.aksw.jenax.annotation.reprogen.Inverse; 7 | import org.aksw.jenax.annotation.reprogen.Iri; 8 | import org.aksw.jenax.annotation.reprogen.ResourceView; 9 | import org.aksw.jenax.annotation.reprogen.StringId; 10 | import org.aksw.jenax.reprogen.hashid.HashIdCxt; 11 | import org.aksw.simba.lsq.model.ElementExec; 12 | import org.aksw.simba.lsq.model.ExperimentRun; 13 | import org.aksw.simba.lsq.model.LocalExecution; 14 | import org.aksw.simba.lsq.vocab.LSQ; 15 | 16 | @ResourceView 17 | public interface TpExec 18 | extends ElementExec 19 | { 20 | @Iri(LSQ.Terms.hasExec) 21 | @Inverse 22 | LsqTriplePattern getTp(); 23 | TpExec setTp(LsqTriplePattern tp); 24 | 25 | 26 | @Iri(LSQ.Terms.hasTpExec) 27 | @Inverse 28 | TpInBgpExec getTpInBgpExec(); 29 | TpExec setTpInBgpExec(TpInBgpExec tpInBgpExec); 30 | 31 | @Iri(LSQ.Terms.tpSelBGPRestricted) 32 | BigDecimal getBgpRestrictedTpSel(); 33 | TpExec setBgpRestrictedTpSel(BigDecimal bgpRestrictedTpSel); 34 | 35 | /** 36 | * Ratio of resultSetSize(tp) / resultSetSize(dataset) 37 | * 38 | * @return 39 | */ 40 | @Iri(LSQ.Terms.tpToGraphRatio) 41 | BigDecimal getSelectivity(); 42 | TpInBgpExec setSelectivity(BigDecimal value); 43 | 44 | @StringId 45 | default String getStringId(HashIdCxt cxt) { 46 | LocalExecution le = this.getQueryExec().getLocalExecution(); 47 | ExperimentRun bmr = le.getBenchmarkRun(); 48 | String prefix = StringUtils.toLowerCamelCase(TpExec.class.getSimpleName()); // "tpExec" 49 | String result = prefix + "-" + cxt.getHashAsString(this) + "-" + cxt.getStringId(bmr); 50 | return result; 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/spinx/model/TpInBgp.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.spinx.model; 2 | 3 | import java.util.Set; 4 | 5 | import org.aksw.jenax.annotation.reprogen.HashId; 6 | import org.aksw.jenax.annotation.reprogen.Iri; 7 | import org.aksw.jenax.annotation.reprogen.ResourceView; 8 | import org.aksw.simba.lsq.vocab.LSQ; 9 | import org.apache.jena.rdf.model.Resource; 10 | 11 | /** 12 | * A mention of a triple pattern in a bgp 13 | * 14 | * 15 | * @author raven 16 | * 17 | */ 18 | @ResourceView 19 | public interface TpInBgp 20 | extends Resource 21 | { 22 | @HashId 23 | @Iri(LSQ.Terms.hasBgp) 24 | Bgp getBgp(); 25 | TpInBgp setBgp(Resource bgp); 26 | 27 | @HashId 28 | @Iri(LSQ.Terms.hasTp) 29 | LsqTriplePattern getTriplePattern(); 30 | TpInBgp setTriplePattern(Resource tp); 31 | 32 | @Iri(LSQ.Terms.hasTpExec) 33 | Set getExecs(); 34 | 35 | // default Map indexExecs() { 36 | // Set res = getExecs(); 37 | // Map result = res.stream() 38 | // .collect(Collectors.toMap(r -> r.getBenchmarkRun(), r -> r)); 39 | // return result; 40 | // } 41 | 42 | // @SortedBy(TIME.atTime) 43 | //Map getExecutions(); 44 | } 45 | -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/spinx/model/TpInBgpExec.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.spinx.model; 2 | 3 | import java.math.BigDecimal; 4 | 5 | import org.aksw.jenax.annotation.reprogen.HashId; 6 | import org.aksw.jenax.annotation.reprogen.Inverse; 7 | import org.aksw.jenax.annotation.reprogen.Iri; 8 | import org.aksw.jenax.annotation.reprogen.ResourceView; 9 | import org.aksw.simba.lsq.vocab.LSQ; 10 | import org.apache.jena.rdf.model.Resource; 11 | 12 | @ResourceView 13 | public interface TpInBgpExec 14 | extends Resource 15 | { 16 | // ExperimentRun getBenchmarkRun(); 17 | // TpInBgpExec setBenchmarkRun(ExperimentRun exp); 18 | 19 | 20 | // TODO The following attributes should be mapped as they refer to computations from which the selectivity was derived 21 | @HashId 22 | @Iri(LSQ.Terms.hasTpInBgpExec) 23 | @Inverse 24 | BgpExec getBgpExec(); 25 | TpInBgpExec setBgpExec(Resource bgpExec); 26 | 27 | @HashId 28 | @Iri(LSQ.Terms.hasTpExec) 29 | TpExec getTpExec(); 30 | TpInBgpExec setTpExec(TpExec tpExec); 31 | 32 | @Iri(LSQ.Terms.hasTp) 33 | @Inverse 34 | TpInBgp getTpInBgp(); 35 | TpInBgpExec setTpInBgp(Resource tpInBgp); 36 | 37 | @Iri(LSQ.Terms.tpSelBGPRestricted) 38 | BigDecimal getSelectivity(); 39 | TpInBgpExec setSelectivity(BigDecimal value); 40 | 41 | @Iri(LSQ.Terms.tpToBgpRatio) 42 | BigDecimal getTpToBgpRatio(); 43 | TpInBgpExec setTpToBgpRatio(BigDecimal value); 44 | 45 | @Iri(LSQ.Terms.hasJoinVarExec) 46 | BgpNodeExec getJoinVarExec(); 47 | TpInBgpExec setJoinVarExec(Resource joinVar); 48 | } 49 | -------------------------------------------------------------------------------- /lsq-model/src/main/java/org/aksw/simba/lsq/spinx/model/TpInSubBgpExec.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.spinx.model; 2 | 3 | public interface TpInSubBgpExec 4 | extends TpInBgpExec 5 | { 6 | BgpNodeExec getBgpNodeExec(); 7 | TpInSubBgpExec setBgpNodeExec(BgpNodeExec bpgNodeExec); 8 | } 9 | -------------------------------------------------------------------------------- /lsq-model/src/main/resources/META-INF/services/org.apache.jena.sys.JenaSubsystemLifecycle: -------------------------------------------------------------------------------- 1 | org.aksw.simba.lsq.jena.plugin.JenaPluginLsq 2 | 3 | -------------------------------------------------------------------------------- /lsq-parser/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | lsq-parser 8 | jar 9 | 10 | 11 | org.aksw.simba.lsq 12 | lsq-parent 13 | 2.0.0-RC5-SNAPSHOT 14 | 15 | 16 | 17 | 18 | 19 | org.aksw.simba.lsq 20 | lsq-vocab-jena 21 | 22 | 23 | 24 | org.aksw.jenax 25 | jenax-arq-utils 26 | 27 | 28 | 29 | org.aksw.jenax 30 | jenax-arq-rx 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 46 | 48 | 49 | junit 50 | junit 51 | test 52 | 53 | 54 | 56 | 57 | 58 | org.apache.logging.log4j 59 | log4j-slf4j-impl 60 | test 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /lsq-parser/src/main/java/org/aksw/simba/lsq/parser/ConverterChain.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.parser; 2 | 3 | import java.util.Arrays; 4 | import java.util.function.BiFunction; 5 | 6 | import com.google.common.base.Converter; 7 | 8 | 9 | /** 10 | * A converter backed by a sequence of converters. 11 | * The result of the overall conversion is determined by the first 12 | * converter in the sequence that can convert the input. 13 | * 14 | * @author raven 15 | * 16 | * @param 17 | * @param 18 | */ 19 | public class ConverterChain 20 | extends Converter 21 | { 22 | protected Iterable> converters; 23 | 24 | public ConverterChain(Iterable> converters) { 25 | super(); 26 | this.converters = converters; 27 | } 28 | 29 | public static B coalesce(A a, Iterable opSymbols, BiFunction opExecutor) { 30 | B result = null; 31 | 32 | for(X opSymbol : opSymbols) { 33 | try { 34 | result = opExecutor.apply(opSymbol, a); 35 | break; 36 | } catch(Exception e) { 37 | continue; 38 | } 39 | } 40 | 41 | if(result == null) { 42 | throw new IllegalArgumentException("No converter could handle argument " + a); 43 | } 44 | 45 | return result; 46 | } 47 | 48 | @Override 49 | protected B doForward(A a) { 50 | B result = coalesce(a, converters, (c, x) -> c.convert(x)); 51 | return result; 52 | } 53 | 54 | @Override 55 | protected A doBackward(B b) { 56 | A result = coalesce(b, converters, (c, x) -> c.reverse().convert(x)); 57 | return result; 58 | } 59 | 60 | @SafeVarargs 61 | public static ConverterChain create(Converter ... converters) { 62 | return new ConverterChain<>(Arrays.asList(converters)); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /lsq-parser/src/main/java/org/aksw/simba/lsq/parser/FieldSpec.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.parser; 2 | 3 | import java.util.regex.Pattern; 4 | 5 | import org.apache.jena.datatypes.RDFDatatype; 6 | import com.google.common.base.Converter; 7 | 8 | public class FieldSpec { 9 | 10 | protected String fieldType; 11 | protected String fieldId; 12 | 13 | protected boolean isOptional; 14 | 15 | protected Pattern pattern; 16 | 17 | protected Converter normalizer; 18 | protected RDFDatatype rdfTypeMapper; 19 | protected Mapper mapper; 20 | 21 | 22 | public String getFieldType() { 23 | return fieldType; 24 | } 25 | 26 | public FieldSpec setFieldType(String fieldType) { 27 | this.fieldType = fieldType; 28 | return this; 29 | } 30 | 31 | public String getFieldId() { 32 | return fieldId; 33 | } 34 | 35 | public FieldSpec setFieldId(String fieldId) { 36 | this.fieldId = fieldId; 37 | return this; 38 | } 39 | 40 | public boolean isOptional() { 41 | return isOptional; 42 | } 43 | 44 | public FieldSpec setOptional(boolean isOptional) { 45 | this.isOptional = isOptional; 46 | return this; 47 | } 48 | 49 | public Pattern getPattern() { 50 | return pattern; 51 | } 52 | 53 | public FieldSpec setPattern(Pattern pattern) { 54 | this.pattern = pattern; 55 | return this; 56 | } 57 | 58 | public Converter getNormalizer() { 59 | return normalizer; 60 | } 61 | 62 | public FieldSpec setNormalizer(Converter normalizer) { 63 | this.normalizer = normalizer; 64 | return this; 65 | } 66 | 67 | public RDFDatatype getRdfTypeMapper() { 68 | return rdfTypeMapper; 69 | } 70 | 71 | public FieldSpec setRdfTypeMapper(RDFDatatype rdfTypeMapper) { 72 | this.rdfTypeMapper = rdfTypeMapper; 73 | return this; 74 | } 75 | 76 | public Mapper getMapper() { 77 | return mapper; 78 | } 79 | 80 | public FieldSpec setMapper(Mapper mapper) { 81 | this.mapper = mapper; 82 | return this; 83 | } 84 | 85 | } 86 | -------------------------------------------------------------------------------- /lsq-parser/src/main/java/org/aksw/simba/lsq/parser/FixMapper.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.parser; 2 | 3 | import java.util.Objects; 4 | 5 | import org.apache.jena.rdf.model.Resource; 6 | 7 | public class FixMapper 8 | implements Mapper { 9 | 10 | protected Mapper delegate; 11 | protected String prefix; 12 | protected String suffix; 13 | 14 | public FixMapper(Mapper delegate, String prefix, String suffix) { 15 | super(); 16 | Objects.requireNonNull(prefix); 17 | Objects.requireNonNull(suffix); 18 | 19 | this.delegate = delegate; 20 | this.prefix = prefix; 21 | this.suffix = suffix; 22 | //this.prefix = prefix != null ? prefix : ""; 23 | //this.suffix = suffix != null ? suffix : ""; 24 | } 25 | 26 | @Override 27 | public int parse(Resource r, String lexicalForm) { 28 | boolean isPrefixMatch = lexicalForm.startsWith(prefix); 29 | boolean isSuffixMatch = lexicalForm.endsWith(suffix); 30 | 31 | boolean isAccepted = isPrefixMatch && isSuffixMatch; 32 | 33 | int result = isAccepted && delegate != null 34 | ? delegate.parse(r, lexicalForm.substring(prefix.length(), lexicalForm.length() - suffix.length())) 35 | : 0; 36 | 37 | return result; 38 | } 39 | 40 | @Override 41 | public String unparse(Resource r) { 42 | StringBuilder sb = new StringBuilder(); 43 | if(prefix != null) { 44 | sb.append(prefix); 45 | } 46 | 47 | String s = delegate == null ? "" : delegate.unparse(r); 48 | sb.append(s); 49 | 50 | if(suffix != null) { 51 | sb.append(suffix); 52 | } 53 | 54 | String result = sb.toString(); 55 | return result; 56 | } 57 | } -------------------------------------------------------------------------------- /lsq-parser/src/main/java/org/aksw/simba/lsq/parser/Mapper.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.parser; 2 | 3 | import org.apache.jena.rdf.model.Resource; 4 | 5 | /** 6 | * This class differs from Jena's TypeMapper that it does not parse strings into individual 7 | * Nodes but Resources. 8 | * 9 | * @author raven 10 | * 11 | */ 12 | public interface Mapper { 13 | 14 | /** 15 | * 16 | * @param r The resource into which to parse the given string as a property's value. 17 | * @param lexicalForm 18 | * @return The number of properties parsed from the string 19 | */ 20 | int parse(Resource r, String lexicalForm); 21 | String unparse(Resource r); 22 | } -------------------------------------------------------------------------------- /lsq-parser/src/main/java/org/aksw/simba/lsq/parser/MapperConverter.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.parser; 2 | 3 | import org.apache.jena.rdf.model.Resource; 4 | 5 | import com.google.common.base.Converter; 6 | 7 | public class MapperConverter 8 | extends MapperDelegate 9 | { 10 | public MapperConverter(Mapper delegate, Converter converter) { 11 | super(delegate); 12 | this.converter = converter; 13 | } 14 | 15 | protected Converter converter; 16 | 17 | @Override 18 | public int parse(Resource r, String lexicalForm) { 19 | String str = converter.convert(lexicalForm); 20 | int result = super.parse(r, str); 21 | return result; 22 | } 23 | 24 | @Override 25 | public String unparse(Resource r) { 26 | String str = super.unparse(r); 27 | String result = converter.reverse().convert(str); 28 | return result; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /lsq-parser/src/main/java/org/aksw/simba/lsq/parser/MapperDelegate.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.parser; 2 | 3 | import org.apache.jena.rdf.model.Resource; 4 | 5 | public class MapperDelegate 6 | implements Mapper 7 | { 8 | protected Mapper delegate; 9 | 10 | public MapperDelegate(Mapper delegate) { 11 | super(); 12 | this.delegate = delegate; 13 | } 14 | 15 | @Override 16 | public int parse(Resource r, String lexicalForm) { 17 | int result = delegate.parse(r, lexicalForm); 18 | return result; 19 | } 20 | 21 | @Override 22 | public String unparse(Resource r) { 23 | String result = delegate.unparse(r); 24 | return result; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /lsq-parser/src/main/java/org/aksw/simba/lsq/parser/NestedPropertyMapper.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.parser; 2 | 3 | import org.apache.jena.rdf.model.Property; 4 | import org.apache.jena.rdf.model.Resource; 5 | import org.apache.jena.rdf.model.Statement; 6 | 7 | public class NestedPropertyMapper 8 | extends MapperDelegate 9 | { 10 | //protected Directed property; 11 | protected Property property; 12 | 13 | public NestedPropertyMapper(Property property, Mapper delegate) { 14 | super(delegate); 15 | this.property = property; 16 | } 17 | 18 | @Override 19 | public int parse(Resource r, String lexicalForm) { 20 | Resource subR; 21 | Statement stmt = r.getProperty(property); 22 | if(stmt != null) { 23 | subR = stmt.getResource(); 24 | } else { 25 | subR = r.getModel().createResource(); 26 | r.addProperty(property, subR); 27 | } 28 | 29 | int result = super.parse(subR, lexicalForm); 30 | 31 | return result; 32 | } 33 | 34 | @Override 35 | public String unparse(Resource r) { 36 | String result; 37 | 38 | Statement stmt = r.getProperty(property); 39 | if(stmt != null) { 40 | Resource subR = stmt.getResource(); 41 | 42 | result = super.unparse(subR); 43 | } else { 44 | result = null; 45 | } 46 | 47 | return result; 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /lsq-parser/src/main/java/org/aksw/simba/lsq/parser/PatternMatcher.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.parser; 2 | 3 | import java.util.Map; 4 | import java.util.function.Function; 5 | 6 | //@FunctionalInterface 7 | public interface PatternMatcher 8 | extends Function> { 9 | //Map match(String line); 10 | } 11 | -------------------------------------------------------------------------------- /lsq-parser/src/main/java/org/aksw/simba/lsq/parser/PatternMatcherImpl.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.parser; 2 | 3 | import java.util.ArrayList; 4 | import java.util.HashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | import java.util.regex.Matcher; 8 | import java.util.regex.Pattern; 9 | 10 | public class PatternMatcherImpl 11 | implements PatternMatcher 12 | { 13 | protected Pattern pattern; 14 | protected List groupNames; 15 | 16 | // TODO Allow > if escaped 17 | public static final Pattern groupNamePattern = Pattern.compile("\\?<([^>]+)>", Pattern.MULTILINE); 18 | 19 | public static List extractGroupNames(String str) { 20 | List result = new ArrayList<>(); 21 | Matcher m = groupNamePattern.matcher(str); 22 | while(m.find()) { 23 | String v = m.group(1); 24 | result.add(v); 25 | } 26 | return result; 27 | } 28 | 29 | public PatternMatcherImpl(Pattern pattern) { 30 | this(pattern, extractGroupNames(pattern.toString())); 31 | } 32 | 33 | 34 | public PatternMatcherImpl(Pattern pattern, List groupNames) { 35 | super(); 36 | this.pattern = pattern; 37 | this.groupNames = groupNames; 38 | } 39 | 40 | 41 | @Override 42 | public Map apply(String str) { 43 | Map result = matchAsMap(pattern, groupNames, str); 44 | return result; 45 | } 46 | 47 | 48 | public static Map matchAsMap(Pattern pattern, List groupNames, String str) { 49 | Map result = null; 50 | 51 | Matcher m = pattern.matcher(str); 52 | if(m.find()) { 53 | result = new HashMap<>(); 54 | 55 | for(String k : groupNames) { 56 | String v = m.group(k); 57 | result.put(k, v); 58 | } 59 | } 60 | 61 | return result; 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /lsq-parser/src/main/java/org/aksw/simba/lsq/parser/PropertyMapper.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.parser; 2 | 3 | import org.apache.jena.datatypes.RDFDatatype; 4 | import org.apache.jena.datatypes.TypeMapper; 5 | import org.apache.jena.rdf.model.Property; 6 | import org.apache.jena.rdf.model.Resource; 7 | 8 | public class PropertyMapper 9 | implements Mapper 10 | { 11 | protected Property property; 12 | protected RDFDatatype rdfDatatype; 13 | 14 | public PropertyMapper(Property property, Class clazz) { 15 | this(property, TypeMapper.getInstance().getTypeByClass(clazz)); 16 | } 17 | 18 | 19 | public PropertyMapper(Property property, RDFDatatype rdfDatatype) { 20 | super(); 21 | this.property = property; 22 | this.rdfDatatype = rdfDatatype; 23 | } 24 | 25 | public int parse(Resource r, String lexicalForm) { 26 | Object value = rdfDatatype.parse(lexicalForm); 27 | r.addLiteral(property, value); 28 | 29 | return 1; 30 | //return r; 31 | } 32 | 33 | public String unparse(Resource r) { 34 | String result; 35 | if(r.hasProperty(property)) { 36 | Object value = r.getProperty(property).getLiteral().getValue(); 37 | result = rdfDatatype.unparse(value); 38 | } else { 39 | result = ""; 40 | } 41 | return result; 42 | } 43 | 44 | public static PropertyMapper create(Property property, Class clazz) { 45 | RDFDatatype rdfDatatype = TypeMapper.getInstance().getTypeByClass(clazz); 46 | PropertyMapper result = new PropertyMapper(property, rdfDatatype); 47 | return result; 48 | } 49 | } -------------------------------------------------------------------------------- /lsq-parser/src/main/java/org/aksw/simba/lsq/parser/RDFDatatypeDateFormat.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.parser; 2 | 3 | import java.text.DateFormat; 4 | import java.util.Calendar; 5 | import java.util.Date; 6 | import java.util.GregorianCalendar; 7 | 8 | import org.aksw.jenax.arq.util.datatype.RDFDatatypeDelegate; 9 | import org.apache.jena.datatypes.xsd.XSDDateTime; 10 | import org.apache.jena.datatypes.xsd.impl.XSDDateTimeType; 11 | 12 | public class RDFDatatypeDateFormat 13 | extends RDFDatatypeDelegate 14 | { 15 | protected Class clazz; 16 | protected DateFormat dateFormat; 17 | 18 | public RDFDatatypeDateFormat(DateFormat dateFormat) { 19 | super(new XSDDateTimeType("dateTime")); 20 | this.clazz = Date.class; 21 | this.dateFormat = dateFormat; 22 | } 23 | 24 | @Override 25 | public Class getJavaClass() { 26 | return clazz; 27 | } 28 | 29 | public String unparse(Object value) { 30 | XSDDateTime dt = (XSDDateTime)value; 31 | Calendar cal = dt.asCalendar(); 32 | 33 | //Calendar cal = (Calendar)value; 34 | // Calendar cal = new GregorianCalendar(); 35 | // cal.setTime(date); 36 | //Date date = (Date) value; 37 | Date date = cal.getTime(); 38 | String result = dateFormat.format(date); 39 | // XSDDateTime tmp = new XSDDateTime(cal); 40 | // String result = super.unparse(tmp); 41 | return result; 42 | } 43 | 44 | @Override 45 | public Object parse(String lexicalForm) { 46 | try { 47 | 48 | //DateTimeFormatter f = DateTimeFormatter.ofPattern("dd MM uuuu HH:mm:ss.SSS X"); 49 | //OffsetDateTime odt = OffsetDateTime.parse ( input , f ); 50 | 51 | Date date = dateFormat.parse(lexicalForm); 52 | //Object tmp = super.parse(lexicalForm); 53 | //XSDDateTime xsd = (XSDDateTime) tmp; 54 | //Calendar cal = xsd.asCalendar(); 55 | Calendar calendar = new GregorianCalendar();//Calendar.getInstance(); 56 | calendar.setTime(date); 57 | 58 | return calendar; 59 | //Date result = calendar.getTime(); 60 | //return result; 61 | } catch(Exception e) { 62 | throw new RuntimeException(e); 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /lsq-parser/src/main/java/org/aksw/simba/lsq/parser/RDFDatatypeRestricted.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.parser; 2 | 3 | import java.util.regex.Pattern; 4 | 5 | import org.aksw.jenax.arq.util.datatype.RDFDatatypeDelegate; 6 | import org.apache.jena.datatypes.RDFDatatype; 7 | 8 | /** 9 | * Wrapper around an underlying rdf datatype where the lexical value is restricted 10 | * by a regex pattern 11 | * 12 | * @author raven 13 | * 14 | */ 15 | public class RDFDatatypeRestricted 16 | extends RDFDatatypeDelegate 17 | { 18 | protected Pattern pattern; 19 | 20 | public RDFDatatypeRestricted(RDFDatatype delegate, Pattern pattern) { 21 | super(delegate); 22 | this.pattern = pattern; 23 | } 24 | 25 | @Override 26 | public boolean isValid(String lexicalForm) { 27 | boolean result = pattern.matcher(lexicalForm).find(); 28 | result = result && super.isValid(lexicalForm); 29 | return result; 30 | } 31 | } -------------------------------------------------------------------------------- /lsq-parser/src/main/java/org/aksw/simba/lsq/parser/csv/CsvParser.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.parser.csv; 2 | 3 | import java.util.List; 4 | import java.util.Map; 5 | import java.util.stream.Collectors; 6 | 7 | import org.aksw.jena_sparql_api.rx.GraphFactoryEx; 8 | import org.aksw.jenax.sparql.rx.op.FlowOfQuadsOps; 9 | import org.aksw.jenax.sparql.rx.op.QueryFlowOps; 10 | import org.aksw.simba.lsq.vocab.LSQ; 11 | import org.apache.jena.query.Query; 12 | import org.apache.jena.query.QueryFactory; 13 | import org.apache.jena.rdf.model.Model; 14 | import org.apache.jena.rdf.model.ModelFactory; 15 | import org.apache.jena.rdf.model.RDFNode; 16 | import org.apache.jena.rdf.model.Resource; 17 | import org.apache.jena.sparql.core.Quad; 18 | import org.apache.jena.sparql.engine.binding.Binding; 19 | import org.apache.jena.vocabulary.RDF; 20 | 21 | import io.reactivex.rxjava3.core.FlowableTransformer; 22 | 23 | public class CsvParser { 24 | interface BindingToResourceTransform extends FlowableTransformer {} 25 | 26 | 27 | 28 | public static Map loadCsvRegistry(Model model) { 29 | List rs = model.listResourcesWithProperty(RDF.type, LSQ.CsvLogFormat).toList(); 30 | 31 | Map result = rs.stream() 32 | .filter(r -> r.hasProperty(LSQ.pattern)) 33 | .collect(Collectors.toMap( 34 | r -> r.getLocalName(), 35 | r -> create(r.getProperty(LSQ.pattern).getString()) 36 | )); 37 | 38 | return result; 39 | } 40 | 41 | public static BindingToResourceTransform create(String pattern) { 42 | Query query = QueryFactory.create(pattern); 43 | 44 | return upstream -> 45 | upstream 46 | .compose(QueryFlowOps.createMapperQuads(query)) 47 | .compose(FlowOfQuadsOps.groupConsecutiveQuadsToGraph(Quad::getGraph, Quad::asTriple, GraphFactoryEx::createInsertOrderPreservingGraph)) 48 | .map(e -> { 49 | Model m = ModelFactory.createModelForGraph(e.getValue()); 50 | return m.asRDFNode(e.getKey()); 51 | }) 52 | // FIXME Apply node transform that replaces the node with a blank node for further re-skolemization 53 | .map(RDFNode::asResource) 54 | ; 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /lsq-parser/src/main/resources/default-log-formats.ttl: -------------------------------------------------------------------------------- 1 | @prefix lsq: . 2 | @prefix fmt: . 3 | 4 | # TODO Transition to a proper vocabulary 5 | 6 | fmt:common 7 | a lsq:WebAccessLogFormat ; 8 | lsq:pattern "%h %l %u %t \"%r\" %>s %b" ; 9 | . 10 | 11 | fmt:commonVirtuoso 12 | a lsq:WebAccessLogFormat ; 13 | lsq:pattern "%h %l %u %{dd/MMM/yyyy HH:mm:ss Z}t \"%r\" %>s %b" ; 14 | . 15 | 16 | 17 | fmt:commonVHost 18 | a lsq:WebAccessLogFormat ; 19 | lsq:pattern "%v %h %l %u %t \"%r\" %>s %b" ; 20 | . 21 | 22 | fmt:combined 23 | a lsq:WebAccessLogFormat ; 24 | lsq:pattern "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" ; 25 | . 26 | 27 | fmt:virtuoso 28 | a lsq:WebAccessLogFormat ; 29 | lsq:pattern "%h %{dd/MMM/yyyy HH:mm:ss Z}t %u \"%U%q\"" ; 30 | . 31 | 32 | fmt:distributed 33 | a lsq:WebAccessLogFormat ; 34 | lsq:pattern "%h %l %u %t \"%r\" %>s %b" ; 35 | . 36 | 37 | fmt:bio2rdf 38 | a lsq:WebAccessLogFormat ; 39 | lsq:pattern "%h %t \"%r\" %>s" ; 40 | . 41 | 42 | # Format for one sparql query per line 43 | # Note: 'sparql' is a multiline format and should supersede sparql2; however the former may not yet scale to large log files due to the way the parser works 44 | fmt:sparql2 45 | a lsq:WebAccessLogFormat ; 46 | lsq:pattern "%sparql" ; 47 | . 48 | 49 | fmt:wikidata 50 | a lsq:WebAccessLogFormat ; 51 | lsq:pattern "%encsparql\t%{yyyy-MM-dd HH:mm:ss}C\t%l\t%{User-agent}i" ; 52 | . 53 | 54 | fmt:bio2rdfProcessedCsv 55 | a lsq:CsvLogFormat ; 56 | lsq:pattern 57 | """ 58 | PREFIX lsq: 59 | PREFIX prov: 60 | PREFIX xsd: 61 | CONSTRUCT { 62 | GRAPH ?s { 63 | ?s 64 | lsq:query ?query ; 65 | lsq:host ?domain ; 66 | lsq:headers [ ?agent ] ; 67 | prov:atTime ?t 68 | } 69 | } { 70 | BIND(IRI(CONCAT('urn:lsq:', MD5(CONCAT(?query, '-', ?domain, '-', ?timestamp)))) AS ?s) 71 | BIND(STRDT(?timestamp, xsd:dateTime) AS ?t) 72 | } 73 | """ . 74 | 75 | -------------------------------------------------------------------------------- /lsq-pkg-parent/lsq-pkg-deb-cli/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | -------------------------------------------------------------------------------- /lsq-pkg-parent/lsq-pkg-deb-cli/src/deb/control/control: -------------------------------------------------------------------------------- 1 | Package: lsq-cli 2 | Version: [[version]] 3 | Section: web 4 | Priority: optional 5 | Architecture: all 6 | Recommends: default-jre-headless 7 | Maintainer: Claus Stadler 8 | Description: Sparqlify Command Line Interfaces 9 | Distribution: ldstack-nightly 10 | Homepage: http://aksw.org/Projects/LSQ 11 | Vcs-Git: git://github.com/AKSW/LSQ.git 12 | Vcs-Browser: https://github.com/AKSW/LSQ 13 | -------------------------------------------------------------------------------- /lsq-pkg-parent/lsq-pkg-deb-cli/src/deb/control/copyright: -------------------------------------------------------------------------------- 1 | This work was packaged for Debian by: 2 | 3 | Claus Stadler on Tue, 01 Feb 2011 15:37:27 +0100 4 | 5 | It was downloaded from: 6 | 7 | 8 | 9 | Upstream Author(s): 10 | 11 | Claus Stadler 12 | 13 | Copyright: 14 | 15 | 16 | 17 | License: 18 | 19 | This package is free software; you can redistribute it and/or modify 20 | it under the terms of the GNU General Public License version 2 as 21 | published by the Free Software Foundation. 22 | 23 | This package is distributed in the hope that it will be useful, 24 | but WITHOUT ANY WARRANTY; without even the implied warranty of 25 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 26 | GNU General Public License for more details. 27 | 28 | You should have received a copy of the GNU General Public License 29 | along with this program. If not, see 30 | 31 | On Debian systems, the complete text of the GNU General 32 | Public License version 2 can be found in "/usr/share/common-licenses/GPL-2". 33 | 34 | The Debian packaging is: 35 | 36 | Copyright (C) 2013 Claus Stadler 37 | 38 | you can redistribute it and/or modify 39 | it under the terms of the GNU General Public License as published by 40 | the Free Software Foundation; either version 2 of the License, or 41 | (at your option) any later version. 42 | 43 | -------------------------------------------------------------------------------- /lsq-pkg-parent/lsq-pkg-deb-cli/src/deb/resources/usr/bin/lsq: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | #WORK_DIR=`dirname "$0"` 4 | 5 | LIB_DIR="/usr/share/lsq-cli/" 6 | MAIN_CLASS="org.aksw.simba.lsq.cli.main.MainCliLsq" 7 | 8 | JAVA=${JAVA_HOME:+$JAVA_HOME/bin/}java 9 | 10 | # Extra options for Java 17; Source: https://stackoverflow.com/questions/73465937/apache-spark-3-3-0-breaks-on-java-17-with-cannot-access-class-sun-nio-ch-direct 11 | # On Java 11 sansa works without these options althought warnings are shown 12 | EXTRA_OPTS="--add-opens=java.base/java.lang=ALL-UNNAMED \ 13 | --add-opens=java.base/java.lang.invoke=ALL-UNNAMED \ 14 | --add-opens=java.base/java.lang.reflect=ALL-UNNAMED \ 15 | --add-opens=java.base/java.io=ALL-UNNAMED \ 16 | --add-opens=java.base/java.net=ALL-UNNAMED \ 17 | --add-opens=java.base/java.nio=ALL-UNNAMED \ 18 | --add-opens=java.base/java.util=ALL-UNNAMED \ 19 | --add-opens=java.base/java.util.concurrent=ALL-UNNAMED \ 20 | --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED \ 21 | --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \ 22 | --add-opens=java.base/sun.nio.cs=ALL-UNNAMED \ 23 | --add-opens=java.base/sun.security.action=ALL-UNNAMED \ 24 | --add-opens=java.base/sun.util.calendar=ALL-UNNAMED \ 25 | --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED" 26 | 27 | SCRIPTING_OPTS="-Djena:scripting=true -Dnashorn.args=--language=es6" 28 | 29 | $JAVA $EXTRA_OPTS $SCRIPTING_OPTS $JAVA_OPTS -cp "$LIB_DIR:$LIB_DIR/lib/*:$EXTRA_CP" "$MAIN_CLASS" "$@" 30 | 31 | -------------------------------------------------------------------------------- /lsq-pkg-parent/lsq-pkg-docker-cli/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | 8 | org.aksw.simba.lsq 9 | lsq-pkg-parent 10 | 2.0.0-RC5-SNAPSHOT 11 | 12 | 13 | lsq-pkg-docker-cli 14 | jar 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | org.apache.maven.plugins 25 | maven-deploy-plugin 26 | 27 | true 28 | 29 | 30 | 31 | 32 | org.sonatype.plugins 33 | nexus-staging-maven-plugin 34 | 35 | true 36 | 37 | 38 | 39 | 40 | com.google.cloud.tools 41 | jib-maven-plugin 42 | 43 | 44 | docker.io/aksw/lsq 45 | 46 | 47 | eclipse-temurin:11-jre 48 | 49 | 50 | org.aksw.simba.lsq.cli.main.MainCliLsq 51 | USE_CURRENT_TIMESTAMP 52 | 53 | packaged 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | org.aksw.simba.lsq 64 | lsq-cli 65 | 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /lsq-pkg-parent/lsq-pkg-dummy/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | lsq-pkg-dummy 8 | pom 9 | 10 | 11 | org.aksw.simba.lsq 12 | lsq-pkg-parent 13 | 2.0.0-RC5-SNAPSHOT 14 | 15 | 16 | 17 | 18 | org.aksw.simba.lsq 19 | lsq-pkg-uberjar-cli 20 | provided 21 | 22 | 23 | org.aksw.simba.lsq 24 | lsq-pkg-deb-cli 25 | provided 26 | 27 | 28 | org.aksw.simba.lsq 29 | lsq-pkg-rpm-cli 30 | provided 31 | 32 | 33 | org.aksw.simba.lsq 34 | lsq-pkg-docker-cli 35 | provided 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /lsq-pkg-parent/lsq-pkg-rpm-cli/src/rpm/resources/usr/bin/lsq: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | LIB_DIR="/usr/share/lsq-cli/" 4 | MAIN_CLASS="org.aksw.simba.lsq.cli.main.MainCliLsq" 5 | 6 | JAVA=${JAVA_HOME:+$JAVA_HOME/bin/}java 7 | 8 | # Extra options for Java 17; Source: https://stackoverflow.com/questions/73465937/apache-spark-3-3-0-breaks-on-java-17-with-cannot-access-class-sun-nio-ch-direct 9 | # On Java 11 sansa works without these options althought warnings are shown 10 | EXTRA_OPTS="--add-opens=java.base/java.lang=ALL-UNNAMED \ 11 | --add-opens=java.base/java.lang.invoke=ALL-UNNAMED \ 12 | --add-opens=java.base/java.lang.reflect=ALL-UNNAMED \ 13 | --add-opens=java.base/java.io=ALL-UNNAMED \ 14 | --add-opens=java.base/java.net=ALL-UNNAMED \ 15 | --add-opens=java.base/java.nio=ALL-UNNAMED \ 16 | --add-opens=java.base/java.util=ALL-UNNAMED \ 17 | --add-opens=java.base/java.util.concurrent=ALL-UNNAMED \ 18 | --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED \ 19 | --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \ 20 | --add-opens=java.base/sun.nio.cs=ALL-UNNAMED \ 21 | --add-opens=java.base/sun.security.action=ALL-UNNAMED \ 22 | --add-opens=java.base/sun.util.calendar=ALL-UNNAMED \ 23 | --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED" 24 | 25 | SCRIPTING_OPTS="-Djena:scripting=true -Dnashorn.args=--language=es6" 26 | 27 | $JAVA $EXTRA_OPTS $SCRIPTING_OPTS $JAVA_OPTS -cp "$LIB_DIR:$LIB_DIR/lib/*:$EXTRA_CP" "$MAIN_CLASS" "$@" 28 | 29 | -------------------------------------------------------------------------------- /lsq-pkg-parent/lsq-pkg-uberjar-cli/src/main/java/LsqUberJarDummy.java: -------------------------------------------------------------------------------- 1 | 2 | /** Dummy class to trigger creation of a (non-empty) jar file and java doc */ 3 | public class LsqUberJarDummy { 4 | } 5 | -------------------------------------------------------------------------------- /lsq-pkg-parent/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | 8 | org.aksw.simba.lsq 9 | lsq-parent 10 | 2.0.0-RC5-SNAPSHOT 11 | 12 | 13 | 21 | lsq-pkg-parent 22 | pom 23 | 24 | 25 | lsq-pkg-uberjar-cli 26 | lsq-pkg-deb-cli 27 | lsq-pkg-rpm-cli 28 | lsq-pkg-docker-cli 29 | lsq-pkg-dummy 30 | 31 | 32 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /lsq-spark-core/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | lsq-spark-core 8 | jar 9 | 10 | 11 | org.aksw.simba.lsq 12 | lsq-parent 13 | 2.0.0-RC5-SNAPSHOT 14 | 15 | 16 | 17 | 18 | org.aksw.simba.lsq 19 | lsq-core 20 | 21 | 22 | 23 | org.aksw.commons 24 | aksw-commons-utils 25 | 26 | 32 | 33 | org.aksw.jenax 34 | jenax-arq-picocli 35 | 36 | 37 | 38 | net.sansa-stack 39 | sansa-spark-jena-java${scala.version.suffix} 40 | 41 | 42 | 43 | 49 | 50 | 51 | 52 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | com.thoughtworks.paranamer 69 | paranamer 70 | 71 | 72 | 73 | junit 74 | junit 75 | test 76 | 77 | 78 | 80 | 81 | 82 | org.apache.logging.log4j 83 | log4j-slf4j-impl 84 | 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /lsq-spark-core/src/main/java/net/sansa_stack/rdf/spark/io/LsqSparkUtils.java: -------------------------------------------------------------------------------- 1 | package net.sansa_stack.rdf.spark.io; 2 | 3 | import java.util.function.Consumer; 4 | 5 | import org.apache.spark.SparkConf; 6 | import org.apache.spark.api.java.JavaSparkContext; 7 | import org.apache.spark.sql.SparkSession; 8 | 9 | public class LsqSparkUtils { 10 | 11 | public static SparkSession createSparkSession(Consumer confCallback) { 12 | SparkConf sparkConf = new SparkConf() 13 | .setAppName("Lsq") 14 | .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") 15 | .set("spark.kryoserializer.buffer.max", "1000") // MB 16 | .set("spark.kryo.registrator", 17 | String.join(", ", "net.sansa_stack.spark.io.rdf.kryo.JenaKryoRegistrator")) 18 | .set("spark.sql.crossJoin.enabled", "true") 19 | // .set("spark.hadoop.mapred.max.split.size", "" + 4 * 1024 * 1024) 20 | // mapreduce.input.fileinputformat.split.minsize 21 | ; 22 | 23 | confCallback.accept(sparkConf); 24 | sparkConf.setMaster("local[*]"); 25 | 26 | SparkSession ss = SparkSession.builder().config(sparkConf).getOrCreate(); 27 | 28 | //JavaSparkContext sc = JavaSparkContext.fromSparkContext(ss.sparkContext()); 29 | 30 | return ss; 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /lsq-spark-core/src/main/java/net/sansa_stack/rdf/spark/io/SourceOfRddOfResources.java: -------------------------------------------------------------------------------- 1 | package net.sansa_stack.rdf.spark.io; 2 | 3 | import org.apache.jena.rdf.model.Resource; 4 | import org.apache.spark.api.java.JavaRDD; 5 | 6 | public interface SourceOfRddOfResources { 7 | JavaRDD load(String source) throws Exception; 8 | } 9 | -------------------------------------------------------------------------------- /lsq-spark-core/src/main/java/org/aksw/simba/lsq/rdf/conversion/GraphOpsStream.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.rdf.conversion; 2 | 3 | import java.util.Map.Entry; 4 | import java.util.function.Function; 5 | import java.util.function.Supplier; 6 | 7 | import org.aksw.commons.util.stream.CollapseRunsSpec; 8 | import org.aksw.commons.util.stream.StreamOperatorCollapseRuns; 9 | import org.apache.jena.graph.Graph; 10 | import org.apache.jena.graph.Node; 11 | import org.apache.jena.graph.Triple; 12 | import org.apache.jena.sparql.graph.GraphFactory; 13 | 14 | public class GraphOpsStream { 15 | public static StreamTransformer> groupConsecutiveTriplesRaw( 16 | Function grouper, 17 | Supplier graphSupplier) { 18 | 19 | return StreamOperatorCollapseRuns.create(CollapseRunsSpec.create( 20 | grouper::apply, 21 | groupKey -> graphSupplier.get(), 22 | Graph::add))::transform; 23 | } 24 | 25 | public static StreamTransformer> graphsFromConsecutiveSubjectsRaw() { 26 | return graphsFromConsecutiveSubjectsRaw(GraphFactory::createDefaultGraph); 27 | } 28 | 29 | public static StreamTransformer> graphsFromConsecutiveSubjectsRaw(Supplier graphSupplier) { 30 | return groupConsecutiveTriplesRaw(Triple::getSubject, graphSupplier); 31 | } 32 | 33 | public static StreamTransformer graphsFromConsecutiveSubjects() { 34 | return graphsFromConsecutiveSubjects(GraphFactory::createDefaultGraph); 35 | } 36 | 37 | public static StreamTransformer graphsFromConsecutiveSubjects(Supplier graphSupplier) { 38 | return graphFromConsecutiveTriples(Triple::getSubject, graphSupplier); 39 | } 40 | 41 | public static StreamTransformer graphFromConsecutiveTriples( 42 | Function grouper, 43 | Supplier graphSupplier) { 44 | return upstream -> 45 | groupConsecutiveTriplesRaw(grouper, graphSupplier).apply(upstream) 46 | .map(Entry::getValue); 47 | } 48 | } 49 | 50 | -------------------------------------------------------------------------------- /lsq-spark-core/src/main/java/org/aksw/simba/lsq/rdf/conversion/StreamTransformer.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.rdf.conversion; 2 | 3 | import java.util.stream.Stream; 4 | 5 | public interface StreamTransformer { 6 | Stream apply(Stream input); 7 | } -------------------------------------------------------------------------------- /lsq-spark-core/src/main/java/org/aksw/simba/lsq/spark/cmd/impl/CmdRdfIo.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.spark.cmd.impl; 2 | 3 | import java.util.List; 4 | 5 | 6 | /** 7 | * Interface for basic RDF-to-RDF batch processing tasks 8 | * 9 | */ 10 | public interface CmdRdfIo { 11 | List getNonOptionArgs(); 12 | 13 | String getOutFile(); 14 | CmdRdfIo setOutFile(String outFile); 15 | 16 | String getOutFolder(); 17 | CmdRdfIo setOutFolder(String outFolder); 18 | 19 | String getOutFormat(); 20 | CmdRdfIo setOutFormat(String outFormat); 21 | 22 | List getPrefixSources(); 23 | 24 | long getDeferOutputForUsedPrefixes(); 25 | CmdRdfIo setDeferOutputForUsedPrefixes(long n); 26 | 27 | } 28 | -------------------------------------------------------------------------------- /lsq-vocab-jena/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | lsq-vocab-jena 8 | jar 9 | 10 | 11 | org.aksw.simba.lsq 12 | lsq-parent 13 | 2.0.0-RC5-SNAPSHOT 14 | 15 | 16 | 17 | 18 | org.apache.jena 19 | jena-core 20 | 21 | 22 | 23 | org.apache.commons 24 | commons-lang3 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /lsq-vocab-jena/src/main/java/org/aksw/simba/lsq/vocab/PROV.java: -------------------------------------------------------------------------------- 1 | package org.aksw.simba.lsq.vocab; 2 | 3 | import org.apache.jena.rdf.model.Property; 4 | import org.apache.jena.rdf.model.ResourceFactory; 5 | 6 | /** 7 | * Provenance Ontology. 8 | * 9 | * There is probably some class like this one out there as a maven dep for provo. 10 | * Once we find it, we can make this class obsolete. 11 | * 12 | * @author raven 13 | * 14 | */ 15 | public class PROV { 16 | public static final String NS = "http://www.w3.org/ns/prov#"; 17 | 18 | public static Property property(String local) { 19 | return ResourceFactory.createProperty(NS + local); 20 | } 21 | 22 | public static final Property hadPrimarySource = property("hadPrimarySource"); 23 | public static final Property atTime = property("atTime"); 24 | public static final Property startedAtTime = property("startedAtTime"); 25 | public static final Property endedAtTime = property("endedAtTime"); 26 | public static final Property wasGeneratedBy = property("wasGeneratedBy"); 27 | public static final Property wasAssociatedWith = property("wasAssociatedWith"); 28 | 29 | } 30 | -------------------------------------------------------------------------------- /reinstall-deb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | sudo apt-get purge lsq-cli 4 | sudo dpkg -i `find 'lsq-pkg-parent/lsq-pkg-deb-cli/target/' -name 'lsq-cli_*.deb'` 5 | 6 | --------------------------------------------------------------------------------