├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md ├── data ├── bashlog │ └── recursion │ │ ├── arcs │ │ └── datalog.txt ├── lubm │ ├── .gitignore │ ├── queries.sparql │ ├── queries.txt │ ├── tbox.txt │ ├── univ-bench.owl │ └── univ-bench.ttl ├── wikidata │ ├── README.md │ ├── build_simple_wikidata_dump.py │ ├── build_wikidata_people.py │ ├── full_queries.txt │ ├── full_tbox.txt │ ├── people_queries.txt │ ├── people_tbox.txt │ ├── rdfox_tbox.dlog │ └── sparql │ │ ├── query1.sparql │ │ ├── query2.sparql │ │ ├── query3.sparql │ │ ├── query4.sparql │ │ ├── query5.sparql │ │ └── query6.sparql └── yago │ ├── README.md │ ├── full_queries.txt │ ├── full_tbox.txt │ ├── rdfox_tbox.dlog │ └── sparql │ ├── query1.sparql │ ├── query2.sparql │ ├── query3.sparql │ ├── query4.sparql │ ├── query5.sparql │ └── query6.sparql ├── exec.sh ├── module └── bashlog-web │ ├── .gitignore │ ├── WebContent │ ├── META-INF │ │ └── MANIFEST.MF │ ├── WEB-INF │ │ ├── tags │ │ │ └── page.tag │ │ └── web.xml │ ├── about.jsp │ ├── api.jsp │ ├── convert_datalog.jsp │ ├── convert_sparql.jsp │ ├── links.jsp │ ├── reset.css │ └── style.css │ ├── pom.xml │ ├── script │ ├── compare-bashlog-web.sh │ └── queries │ │ ├── .gitignore │ │ ├── all-people.datalog │ │ ├── all-people.owl │ │ ├── all-people.sparql │ │ ├── facts-program.datalog │ │ ├── facts-program.owl │ │ ├── facts-program.sparql │ │ ├── living-people.datalog │ │ ├── living-people.owl │ │ ├── living-people.sparql │ │ ├── same-city.datalog │ │ └── same-city.sparql │ └── src │ └── main │ └── java │ ├── bashlog │ ├── bashlogweb │ ├── API.java │ └── Main.java │ ├── common │ └── rdf ├── pom.xml └── src ├── main ├── java │ ├── bashlog │ │ ├── BashlogCompiler.java │ │ ├── BashlogEvaluator.java │ │ ├── Cmd.java │ │ ├── CmdSparql.java │ │ ├── CompilerInternals.java │ │ ├── command │ │ │ └── Bash.java │ │ ├── plan │ │ │ ├── BashlogOptimizer.java │ │ │ ├── BashlogPlan.java │ │ │ ├── CombinedColumnNode.java │ │ │ ├── SortAntiJoinNode.java │ │ │ ├── SortJoinNode.java │ │ │ ├── SortNode.java │ │ │ └── TSVFileNode.java │ │ └── translation │ │ │ ├── AwkHelper.java │ │ │ ├── BashCmd.java │ │ │ ├── BashTranslator.java │ │ │ ├── CombineColumns.java │ │ │ ├── Fact.java │ │ │ ├── FileInput.java │ │ │ ├── Join.java │ │ │ ├── Materialization.java │ │ │ ├── MultiFilter.java │ │ │ ├── MultiOutput.java │ │ │ ├── ProjectFilter.java │ │ │ ├── Recursion.java │ │ │ ├── Sort.java │ │ │ └── Union.java │ ├── common │ │ ├── CallStack.java │ │ ├── DatalogTools.java │ │ ├── Evaluator.java │ │ ├── FactsSet.java │ │ ├── SimpleFactsSet.java │ │ ├── TSVReader.java │ │ ├── TSVWriter.java │ │ ├── Tools.java │ │ ├── VerifyNodeImplementation.java │ │ ├── compiler │ │ │ ├── AutoIndent.java │ │ │ ├── CompilerInternals.java │ │ │ └── Translator.java │ │ ├── parser │ │ │ ├── Atom.java │ │ │ ├── BashRule.java │ │ │ ├── CompoundTerm.java │ │ │ ├── Constant.java │ │ │ ├── ParseException.java │ │ │ ├── Parseable.java │ │ │ ├── ParserReader.java │ │ │ ├── Program.java │ │ │ ├── Rule.java │ │ │ ├── Term.java │ │ │ ├── TermList.java │ │ │ └── Variable.java │ │ └── plan │ │ │ ├── LogicalPlanBuilder.java │ │ │ ├── node │ │ │ ├── AntiJoinNode.java │ │ │ ├── BashNode.java │ │ │ ├── BuiltinNode.java │ │ │ ├── ConstantEqualityFilterNode.java │ │ │ ├── EqualityFilterNode.java │ │ │ ├── FactNode.java │ │ │ ├── JoinNode.java │ │ │ ├── MaterializationNode.java │ │ │ ├── MultiFilterNode.java │ │ │ ├── MultiOutputNode.java │ │ │ ├── PlaceholderNode.java │ │ │ ├── PlanNode.java │ │ │ ├── ProjectNode.java │ │ │ ├── RecursionNode.java │ │ │ ├── UnionNode.java │ │ │ └── VariableEqualityFilterNode.java │ │ │ └── optimizer │ │ │ ├── CatToFile.java │ │ │ ├── CombineFacts.java │ │ │ ├── CombineFilter.java │ │ │ ├── Materialize.java │ │ │ ├── MultiOutput.java │ │ │ ├── Optimizer.java │ │ │ ├── PlanValidator.java │ │ │ ├── PushDownFilterAndProject.java │ │ │ ├── PushDownJoin.java │ │ │ ├── ReorderJoinLinear.java │ │ │ ├── ReorderJoinTree.java │ │ │ └── SimplifyRecursion.java │ ├── experiments │ │ ├── MainThomas.java │ │ ├── MainThomasOWL.java │ │ ├── MainThomasWikidata.java │ │ ├── MainThomasYago.java │ │ └── lubm │ │ │ ├── BashlogLUBM.java │ │ │ ├── generator │ │ │ ├── DamlWriter.java │ │ │ ├── DatalogWriter.java │ │ │ ├── Generator.java │ │ │ ├── NtlitWriter.java │ │ │ ├── OwlWriter.java │ │ │ ├── RdfWriter.java │ │ │ ├── Tsv2Writer.java │ │ │ ├── Tsv3Writer.java │ │ │ ├── TtlWriter.java │ │ │ └── Writer.java │ │ │ └── readme.txt │ ├── rdf │ │ ├── MainOwl.java │ │ ├── OWL2RLOntologyConverter.java │ │ ├── OntologyConverter.java │ │ ├── RDFSpecificTuplesSerializer.java │ │ ├── RDFTripleTupleSerializer.java │ │ ├── RDFTupleSerializer.java │ │ └── SPARQLConverter.java │ ├── sparqlog │ │ └── SparqlogCompiler.java │ └── sqllog │ │ └── SqllogCompiler.java └── resources │ └── log4j2.xml └── test ├── java ├── bashlog │ ├── BashlogIntegrationTests.java │ ├── BashlogLUBMTest.java │ └── FileTest.java ├── common │ ├── Check.java │ ├── DownloadTools.java │ ├── IntegrationTests.java │ ├── LUBMTest.java │ ├── parser │ │ └── ProgramTest.java │ └── plan │ │ └── PlanNodeTest.java └── sqllog │ └── IntegrationTests.java └── resources ├── test1-extern ├── expected.txt ├── input-data.txt └── main.txt └── test1-intern ├── expected.txt └── main.txt /.gitattributes: -------------------------------------------------------------------------------- 1 | # In general, use LF for text 2 | #* text eol=lf 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.bbl 3 | *.blg 4 | *.log 5 | *.aux 6 | *.out 7 | *.pdf 8 | *.gz 9 | .classpath 10 | .project 11 | .settings 12 | *.iml 13 | /bin/ 14 | /target/ 15 | /data/lubm/abox* 16 | 17 | .idea/ 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bashlog 2 | This tool translates datalog programs to Unix bash scripts. It can be used to preprocess large tabular datasets. 3 | Please have a look at the [technical report](https://www.thomasrebele.org/publications/2018_report_bashlog.pdf) if you are interested how it works. 4 | You can also try it [online](https://www.thomasrebele.org/projects/bashlog/). 5 | 6 | ## How to run it locally 7 | 8 | 1. Download [bashlog-datalog.jar](https://github.com/thomasrebele/bashlog/releases/download/v1.0.1/bashlog-datalog.jar) from the [releases](https://github.com/thomasrebele/bashlog/releases) 9 | 2. Write a datalog program ([examples](https://www.thomasrebele.org/projects/bashlog/datalog)). 10 | 3. Generate the script with `java -jar bashlog-datalog.jar --query-file --query-pred > query.sh` 11 | 4. Execute it with `bash query.sh > result.txt` 12 | 13 | ## References 14 | 15 | If you use bashlog in your research, please cite: 16 | 17 | @inproceedings{bashlog, 18 | author = {Thomas Rebele and Thomas Pellissier Tanon and Fabian M. Suchanek}, 19 | title = {Bash Datalog: Answering Datalog Queries with Unix Shell Commands}, 20 | booktitle = {International Semantic Web Conference}, 21 | pages = {566--582}, 22 | year = {2018} 23 | } 24 | -------------------------------------------------------------------------------- /data/bashlog/recursion/arcs: -------------------------------------------------------------------------------- 1 | A B 2 | B C 3 | C D 4 | D E 5 | E F 6 | F G 7 | G H 8 | -------------------------------------------------------------------------------- /data/bashlog/recursion/datalog.txt: -------------------------------------------------------------------------------- 1 | arcs(X,Y) :~ cat data/bashlog/recursion/arcs 2 | 3 | tc(X,Y) :- arcs(X,Y). 4 | tc(X,Z) :- tc(X,Y), arcs(Y,Z). 5 | -------------------------------------------------------------------------------- /data/lubm/.gitignore: -------------------------------------------------------------------------------- 1 | 1/ 2 | answers/ 3 | -------------------------------------------------------------------------------- /data/lubm/queries.sparql: -------------------------------------------------------------------------------- 1 | PREFIX rdf: 2 | PREFIX ub: 3 | SELECT ?X 4 | WHERE 5 | {?X rdf:type ub:GraduateStudent . 6 | ?X ub:takesCourse 7 | } 8 | 9 | PREFIX rdf: 10 | PREFIX ub: 11 | SELECT ?X ?Y ?Z 12 | WHERE 13 | {?X rdf:type ub:GraduateStudent . 14 | ?Y rdf:type ub:University . 15 | ?Z rdf:type ub:Department . 16 | ?X ub:memberOf ?Z . 17 | ?Z ub:subOrganizationOf ?Y . 18 | ?X ub:undergraduateDegreeFrom ?Y} 19 | 20 | PREFIX rdf: 21 | PREFIX ub: 22 | SELECT ?X 23 | WHERE 24 | {?X rdf:type ub:Publication . 25 | ?X ub:publicationAuthor 26 | } 27 | 28 | PREFIX rdf: 29 | PREFIX ub: 30 | SELECT ?X ?Y1 ?Y2 ?Y3 31 | WHERE 32 | {?X rdf:type ub:Professor . 33 | ?X ub:worksFor . 34 | ?X ub:name ?Y1 . 35 | ?X ub:emailAddress ?Y2 . 36 | ?X ub:telephone ?Y3} 37 | 38 | PREFIX rdf: 39 | PREFIX ub: 40 | SELECT ?X 41 | WHERE 42 | {?X rdf:type ub:Person . 43 | ?X ub:memberOf } 44 | 45 | PREFIX rdf: 46 | PREFIX ub: 47 | SELECT ?X WHERE {?X rdf:type ub:Student} 48 | 49 | PREFIX rdf: 50 | PREFIX ub: 51 | SELECT ?X ?Y 52 | WHERE 53 | {?X rdf:type ub:Student . 54 | ?Y rdf:type ub:Course . 55 | ?X ub:takesCourse ?Y . 56 | 57 | ub:teacherOf ?Y} 58 | 59 | PREFIX rdf: 60 | PREFIX ub: 61 | SELECT ?X ?Y ?Z 62 | WHERE 63 | {?X rdf:type ub:Student . 64 | ?Y rdf:type ub:Department . 65 | ?X ub:memberOf ?Y . 66 | ?Y ub:subOrganizationOf . 67 | ?X ub:emailAddress ?Z} 68 | 69 | PREFIX rdf: 70 | PREFIX ub: 71 | SELECT ?X ?Y ?Z 72 | WHERE 73 | {?X rdf:type ub:Student . 74 | ?Y rdf:type ub:Faculty . 75 | ?Z rdf:type ub:Course . 76 | ?X ub:advisor ?Y . 77 | ?Y ub:teacherOf ?Z . 78 | ?X ub:takesCourse ?Z} 79 | 80 | PREFIX rdf: 81 | PREFIX ub: 82 | SELECT ?X 83 | WHERE 84 | {?X rdf:type ub:Student . 85 | ?X ub:takesCourse 86 | } 87 | 88 | PREFIX rdf: 89 | PREFIX ub: 90 | SELECT ?X 91 | WHERE 92 | {?X rdf:type ub:ResearchGroup . 93 | ?X ub:subOrganizationOf } 94 | 95 | PREFIX rdf: 96 | PREFIX ub: 97 | SELECT ?X ?Y 98 | WHERE 99 | {?X rdf:type ub:Chair . 100 | ?Y rdf:type ub:Department . 101 | ?X ub:worksFor ?Y . 102 | ?Y ub:subOrganizationOf } 103 | 104 | PREFIX rdf: 105 | PREFIX ub: 106 | SELECT ?X 107 | WHERE 108 | {?X rdf:type ub:Person . 109 | ub:hasAlumnus ?X} 110 | 111 | PREFIX rdf: 112 | PREFIX ub: 113 | SELECT ?X 114 | WHERE {?X rdf:type ub:UndergraduateStudent} 115 | -------------------------------------------------------------------------------- /data/lubm/queries.txt: -------------------------------------------------------------------------------- 1 | query1(X) :- GraduateStudent(X),takesCourse(X,"http://www.Department0.University0.edu/GraduateCourse0") . 2 | query2(X,Y,Z) :- GraduateStudent(X),University(Y),Department(Z),memberOf(X,Z),subOrganizationOf(Z,Y),undergraduateDegreeFrom(X,Y) . 3 | query3(X) :- Publication(X),publicationAuthor(X,"http://www.Department0.University0.edu/AssistantProfessor0") . 4 | query4(X,Y1,Y2,Y3) :- Professor(X),worksFor(X,"http://www.Department0.University0.edu"),name(X,Y1),emailAddress(X,Y2),telephone(X,Y3) . 5 | query5(X) :- Person(X),memberOf(X,"http://www.Department0.University0.edu") . 6 | query6(X) :- Student(X) . 7 | query7(X, Y) :- Student(X),Course(Y),takesCourse(X,Y),teacherOf("http://www.Department0.University0.edu/AssociateProfessor0",Y) . 8 | query8(X,Y,Z) :- Student(X),Department(Y),memberOf(X,Y),subOrganizationOf(Y,"http://www.University0.edu"),emailAddress(X,Z) . 9 | query9(X,Y,Z) :- Student(X),Faculty(Y),Course(Z),advisor(X,Y),teacherOf(Y,Z),takesCourse(X,Z) . 10 | query10(X) :- Student(X),takesCourse(X,"http://www.Department0.University0.edu/GraduateCourse0") . 11 | query11(X) :- ResearchGroup(X),subOrganizationOf(X,"http://www.University0.edu") . 12 | query12(X,Y) :- Chair(X),Department(Y),worksFor(X,Y),subOrganizationOf(Y,"http://www.University0.edu") . 13 | query13(X) :- Person(X),hasAlumnus("http://www.University0.edu",X) . 14 | query14(X) :- UndergraduateStudent(X) . 15 | 16 | -------------------------------------------------------------------------------- /data/wikidata/README.md: -------------------------------------------------------------------------------- 1 | This repository provides two experiments. For both of them you need to download a Wikidata truthy dump: 2 | https://dumps.wikimedia.org/other/wikibase/wikidatawiki/latest-truthy.nt.gz 3 | The one from October 27th, 2017 contains 2.1G triples. 4 | 5 | ## Full 6 | 7 | ### Setup 8 | 9 | To preprocess the dataset: `python3 build_simple_wikidata_dump.py latest-truthy.nt.gz` 10 | (replace `latest-truthy.nt.gz` with the name of the downloaded dump if required). 11 | It outputs a file called `all-triples.txt`. 12 | 13 | 14 | ### Execute queries 15 | 16 | #### bashlog 17 | 18 | The Tbox is in `full_tbox.txt` (update the path to the `all-triples.txt` file). 19 | The queries are defined in `full_queries.txt` 20 | 21 | 22 | ### rdfox 23 | 24 | Script to start all queries 25 | ``` 26 | for((i=1;i<=6;i++)); do 27 | rdfox() { 28 | python3 ../../../experiments/competitors/rdfox/run-rdfox.py rdfox_tbox.dlog latest-truthy.nt.gz sparql/query$i.sparql > $tmp/wikidata-result$i-rdfox.txt; 29 | } 30 | run "rdfox: wikidata query $i" rdfox 31 | done 32 | ``` 33 | 34 | ## People 35 | 36 | Outdated: do not run. 37 | 38 | This experiments extracts a few relation from Wikidata centered on people and places and then provides a complex TBox and a few queries on them. 39 | To generate the relation files you should run "python3 build_wikidata_people.py latest-truthy.nt.gz". 40 | 41 | The TBox is in people_tbox.txt and the queries in people_queries.txt 42 | -------------------------------------------------------------------------------- /data/wikidata/build_simple_wikidata_dump.py: -------------------------------------------------------------------------------- 1 | import gzip 2 | 3 | import plac # pip3 install plac 4 | 5 | 6 | # Is going to return this data as a big file {subject}\t{property}\t{object} 7 | 8 | def parse_triples(file_name): 9 | with gzip.open(file_name, 'rt') as file: 10 | for line in file: 11 | parts = [e.strip(' \r') for e in line.strip(' \t\r\n.').split(' ', 2)] 12 | if len(parts) == 3: 13 | yield parts 14 | else: 15 | print(parts) 16 | 17 | 18 | def main(input_file): 19 | with open('all-triples.txt', 'wt') as fp: 20 | for (s, p, o) in parse_triples(input_file): 21 | fp.write('{}\t{}\t{}\n'.format(s, p, o)) 22 | 23 | 24 | if __name__ == '__main__': 25 | plac.call(main) 26 | -------------------------------------------------------------------------------- /data/wikidata/build_wikidata_people.py: -------------------------------------------------------------------------------- 1 | import gzip 2 | 3 | import plac # pip3 install plac 4 | 5 | 6 | # Is going to return this data as folder per relation 7 | # You need first to download Wikidata truthy dump: https://dumps.wikimedia.org/other/wikibase/wikidatawiki/latest-truthy.nt.gz and pass its path as first argument of this script 8 | 9 | def parse_triples(file_name): 10 | with gzip.open(file_name, 'rt') as file: 11 | for line in file: 12 | parts = [e.strip(' \r<>') 13 | .replace('http://www.wikidata.org/prop/direct/', '') 14 | .replace('http://www.wikidata.org/entity/', '') for e in line.strip(' \t\r\n.').split(' ', 2)] 15 | if len(parts) == 3: 16 | yield parts 17 | else: 18 | print(parts) 19 | 20 | 21 | predicates_map = { 22 | 'P17': 'hasCountry', 23 | 'P19': 'hasBirthPlace', 24 | 'P20': 'hasDeathPlace', 25 | 'P21': 'hasGender', 26 | 'P22': 'hasFather', 27 | 'P25': 'hasMother', 28 | 'P26': 'hasSpouse', 29 | 'P27': 'hasNationality', 30 | 'P40': 'hasChild', 31 | 'P131': 'isLocatedIn', 32 | 'P150': 'containsLocation', 33 | 'P3373': 'hasSibling', 34 | 'P3448': 'hasStepParent' 35 | } 36 | 37 | 38 | def main(input_file): 39 | files = {prop: open('people/' + prop, 'wt') for prop in predicates_map.values()} 40 | for (s, p, o) in parse_triples(input_file): 41 | if p in predicates_map: 42 | files[predicates_map[p]].write('{}\t{}\n'.format(s, o)) 43 | 44 | 45 | if __name__ == '__main__': 46 | plac.call(main) 47 | -------------------------------------------------------------------------------- /data/wikidata/full_queries.txt: -------------------------------------------------------------------------------- 1 | query1(X) :- subClassOf(X, "") . 2 | query1(X) :- subClassOf(X, Y), query1(Y) . 3 | query2(X) :- type(X, "") . 4 | query3(X) :- hasFather(X, "") . 5 | query4(X) :- hasAncestor(X, "") . 6 | query5(X) :- hasBirthPlace(X, Y), inCountry(Y, "") . 7 | query6birthPlaceExists(X) :- hasBirthPlace(X,Y) . 8 | query6(X) :- hasDeathPlace(X, Y), not query6birthPlaceExists(X) . 9 | -------------------------------------------------------------------------------- /data/wikidata/full_tbox.txt: -------------------------------------------------------------------------------- 1 | % cat "data/wikidata/all-triples.txt" 2 | 3 | triple(S,P,O) :~ cat $1 4 | 5 | inCountry(X,Y) :- triple(X, "", Y) . 6 | hasBirthPlace(X,Y) :- triple(X, "", Y) . 7 | hasDeathPlace(X,Y) :- triple(X, "", Y) . 8 | Male(X) :- triple(X, "", "") . 9 | Female(X) :- triple(X, "", "") . 10 | hasFather(X,Y) :- triple(X, "", Y) . 11 | hasMother(X,Y) :- triple(X, "", Y) . 12 | hasSpouse(X,Y) :- triple(X, "", Y) . 13 | hasNationality(X,Y) :- triple(X, "", Y) . 14 | type(X,Y) :- triple(X, "", Y) . 15 | hasChild(X,Y) :- triple(X, "", Y) . 16 | isLocatedIn(X,Y) :- triple(X, "", Y) . 17 | containsLocation(X,Y) :- triple(X, "", Y) . 18 | subClassOf(X,Y) :- triple(X, "", Y) . 19 | hasSibling(X,Y) :- triple(X, "", Y) . 20 | hasStepParent(X,Y) :- triple(X, "", Y) . 21 | 22 | type(X,Z) :- type(X,Y), subClassOf(Y,Z) . 23 | hasParent(X,Y) :- hasFather(X,Y) . 24 | hasParent(X,Y) :- hasMother(X,Y) . 25 | hasAncestor(X,Y) :- hasParent(X,Y) . 26 | hasAncestor(X,Z) :- hasAncestor(X,Y), hasParent(Y,Z) . 27 | inCountry(X,Z) :- isLocatedIn(X,Y), inCountry(Y,Z) . 28 | isLocatedIn(X,Y) :- containsLocation(Y,X) . 29 | containsLocation(X,Y) :- isLocatedIn(Y,X) . 30 | isLocatedIn(X,Y) :- isLocatedIn(X,Y), isLocatedIn(Y,Z) . 31 | -------------------------------------------------------------------------------- /data/wikidata/people_queries.txt: -------------------------------------------------------------------------------- 1 | query1(X) :- hasFather(X, "Q7742") . 2 | query2(X) :- hasAncestor(X, "Q7742") . 3 | query3(X) :- hasBirthPlace(X), hasCountry(X, "Q228") . 4 | -------------------------------------------------------------------------------- /data/wikidata/people_tbox.txt: -------------------------------------------------------------------------------- 1 | hasChild(X, Y) :- hasFather(Y, X) . 2 | hasChild(X, Y) :- hasMother(Y, X) . 3 | hasSibling(X,Y) :- hasChild(Z, X), hasChild(Z, Y) . 4 | hasSibling(X,Y) :- hasSibling(Y,X) . 5 | hasSpouse(X,Y) :- hasSpouse(Y,X) . 6 | 7 | hasAncestor(X,Y) :- hasFather(X,Y) . 8 | hasAncestor(X,Y) :- hasMother(X,Y) . 9 | hasAncestor(X,Z) :- hasAncestor(X,Y), hasParent(Y,Z) . 10 | 11 | isLocatedIn(X,Z) :- isLocatedIn(X,Y), isLocatedIn(Y,Z) . 12 | isLocatedIn(X,Y) :- containsLocation(Y,X) . 13 | containsLocation(X,Y) :- isLocatedIn(Y,X) . 14 | isLocatedIn(X,Y) :- hasCountry(X,Y) . 15 | hasCountry(X,Z) :- isLocatedIn(X,Y), hasCountry(Y,Z) . 16 | 17 | hasBirthPlace(X,Z) :- hasBirthPlace(X,Y), isLocatedIn(Y,Z) . 18 | hasDeathPlace(X,Z) :- hasDeathPlace(X,Y), isLocatedIn(Y,Z) . 19 | -------------------------------------------------------------------------------- /data/wikidata/rdfox_tbox.dlog: -------------------------------------------------------------------------------- 1 | PREFIX wd: 2 | PREFIX wdt: 3 | 4 | wdt:P31(X,Z) :- wdt:P31(X,Y), wdt:P279(Y,Z) . 5 | wdt:hasParent(X,Y) :- wdt:P22(X,Y) . 6 | wdt:hasParent(X,Y) :- wdt:P25(X,Y) . 7 | wdt:hasAncestor(X,Y) :- wdt:hasParent(X,Y) . 8 | wdt:hasAncestor(X,Z) :- wdt:hasAncestor(X,Y), wdt:hasParent(Y,Z) . 9 | wdt:P17(X,Z) :- wdt:P131(X,Y), wdt:P17(Y,Z) . 10 | wdt:P131(X,Y) :- wdt:P150(Y,X) . 11 | wdt:P150(X,Y) :- wdt:P131(Y,X) . 12 | wdt:P131(X,Y) :- wdt:P131(X,Y), wdt:P131(Y,Z) . 13 | -------------------------------------------------------------------------------- /data/wikidata/sparql/query1.sparql: -------------------------------------------------------------------------------- 1 | PREFIX wd: 2 | PREFIX wdt: 3 | 4 | SELECT ?X WHERE { ?X wdt:P279+ wd:Q215627 } 5 | -------------------------------------------------------------------------------- /data/wikidata/sparql/query2.sparql: -------------------------------------------------------------------------------- 1 | PREFIX wd: 2 | PREFIX wdt: 3 | 4 | SELECT ?X WHERE { ?X wdt:P31 wd:Q215627 } 5 | -------------------------------------------------------------------------------- /data/wikidata/sparql/query3.sparql: -------------------------------------------------------------------------------- 1 | PREFIX wd: 2 | PREFIX wdt: 3 | 4 | SELECT ?X WHERE { ?X wdt:P22 wd:Q7742 } 5 | -------------------------------------------------------------------------------- /data/wikidata/sparql/query4.sparql: -------------------------------------------------------------------------------- 1 | PREFIX wd: 2 | PREFIX wdt: 3 | 4 | SELECT ?X WHERE { ?X wdt:hasAncestor wd:Q7742 } 5 | -------------------------------------------------------------------------------- /data/wikidata/sparql/query5.sparql: -------------------------------------------------------------------------------- 1 | PREFIX wd: 2 | PREFIX wdt: 3 | 4 | SELECT ?X WHERE { ?X wdt:P19/wdt:P131 wd:Q228 } -------------------------------------------------------------------------------- /data/wikidata/sparql/query6.sparql: -------------------------------------------------------------------------------- 1 | PREFIX wd: 2 | PREFIX wdt: 3 | 4 | SELECT ?X WHERE { ?X wdt:P20 ?Y . FILTER NOT EXISTS { ?X wdt:P19 ?Z } } 5 | -------------------------------------------------------------------------------- /data/yago/README.md: -------------------------------------------------------------------------------- 1 | ## bashlog 2 | 3 | Download the Yago files `yagoFacts.tsv`, `yagoSimpleTaxonomy.tsv` and `yagoSimpleTypes.tsv` 4 | The Tbox is in `full_tbox.txt` (update the path to the TSV files). 5 | The queries are defined in `full_queries.txt` 6 | 7 | 8 | ## rdfox 9 | 10 | Download the Yago files `yagoFacts.ttl`, `yagoSimpleTaxonomy.ttl` and `yagoSimpleTypes.ttl` 11 | Merge `yagoSimpleTaxonomy.ttl` and `yagoSimpleTypes.ttl` in `yagoTypes.ttl` 12 | 13 | Script to start all queries 14 | ``` 15 | for((i=1;i<=2;i++)); do 16 | rdfox() { 17 | python3 ../../../experiments/competitors/rdfox/run-rdfox.py rdfox_tbox.dlog yagoTypes.ttl sparql/query$i.sparql > $tmp/yago-result$i-rdfox.txt; 18 | } 19 | run "rdfox: yago query $i" rdfox 20 | done 21 | for((i=3;i<=6;i++)); do 22 | rdfox() { 23 | python3 ../../../experiments/competitors/rdfox/run-rdfox.py rdfox_tbox.dlog yagoFacts.ttl sparql/query$i.sparql > $tmp/yago-result$i-rdfox.txt; 24 | } 25 | run "rdfox: yago query $i" rdfox 26 | done 27 | ``` 28 | -------------------------------------------------------------------------------- /data/yago/full_queries.txt: -------------------------------------------------------------------------------- 1 | query1(X) :- subClassOf(X, "") . 2 | query1(X) :- subClassOf(X, Y), query1(Y) . 3 | query2(X) :- type(X, "") . 4 | query3(X) :- hasParent(X, "") . 5 | query4(X) :- hasAncestor(X, "") . 6 | query5(X) :- hasBirthPlace(X, Y), isLocatedIn(Y, "") . 7 | query6birthPlaceExists(X) :- hasBirthPlace(X, Y) . 8 | query6(X) :- hasDeathPlace(X, Y), not query6birthPlaceExists(X) . 9 | -------------------------------------------------------------------------------- /data/yago/full_tbox.txt: -------------------------------------------------------------------------------- 1 | typeFile(C,X,Y,Z) :~ cat $2 2 | type(X,Z) :- typeFile(C,X,"rdf:type",Z) . 3 | taxonomy(C,X,Y,Z) :~ cat $1 4 | subClassOf(X,Z) :- taxonomy(C,X,"rdfs:subClassOf",Z) . 5 | type(X,Z) :- type(X,Y), subClassOf(Y,Z) . 6 | 7 | quad(C,S,P,O) :~ cat $3 8 | hasBirthPlace(X,Y) :- quad(C, X, "", Y) . 9 | hasDeathPlace(X,Y) :- quad(C, X, "", Y) . 10 | Male(X) :- quad(C, X, "", "") . 11 | Female(X) :- quad(C, X, "", "") . 12 | hasSpouse(X,Y) :- quad(C, X, "", Y) . 13 | hasNationality(X,Y) :- quad(C, X, "", Y) . 14 | hasChild(X,Y) :- quad(C, X, "", Y) . 15 | isLocatedIn(X,Y) :- quad(C, X, "", Y) . 16 | 17 | hasParent(X,Y) :- hasChild(Y,X) . 18 | hasAncestor(X,Y) :- hasParent(Y,X) . 19 | hasAncestor(X,Z) :- hasAncestor(X,Y), hasParent(Y,Z) . 20 | isLocatedIn(X,Y) :- containsLocation(Y,X) . 21 | containsLocation(X,Y) :- isLocatedIn(Y,X) . 22 | isLocatedIn(X,Y) :- isLocatedIn(X,Y), isLocatedIn(Y,Z) . 23 | -------------------------------------------------------------------------------- /data/yago/rdfox_tbox.dlog: -------------------------------------------------------------------------------- 1 | PREFIX yago: 2 | PREFIX owl: 3 | PREFIX rdf: 4 | PREFIX rdfs: 5 | PREFIX ruleml: 6 | PREFIX swrl: 7 | PREFIX swrlb: 8 | PREFIX swrlx: 9 | PREFIX xsd: 10 | 11 | (?X,?Y) :- (?Y,?X) . 12 | (?X,?Y) :- (?X,?Y) . 13 | (?X,?Z) :- (?X,?Y), (?Y,?Z) . 14 | (?X,?Y) :- (?Y,?X) . 15 | (?X,?Y) :- (?Y,?X) . 16 | (?X,?Y) :- (?X,?Y), (?Y,?Z) . 17 | 18 | # (?X,?Z) :- (?X,?Y), (?Y,?Z) . 19 | 20 | (?X,?Z) :- (?X,?Z) . 21 | (?X,?Z) :- (?X,?Y), (?Y,?Z) . 22 | -------------------------------------------------------------------------------- /data/yago/sparql/query1.sparql: -------------------------------------------------------------------------------- 1 | PREFIX rdf: 2 | PREFIX rdfs: 3 | 4 | SELECT ?X WHERE { ?X } -------------------------------------------------------------------------------- /data/yago/sparql/query2.sparql: -------------------------------------------------------------------------------- 1 | PREFIX rdf: 2 | PREFIX rdfs: 3 | 4 | SELECT ?X WHERE { ?X rdf:type } 5 | -------------------------------------------------------------------------------- /data/yago/sparql/query3.sparql: -------------------------------------------------------------------------------- 1 | PREFIX rdf: 2 | PREFIX rdfs: 3 | 4 | SELECT ?X WHERE { ?X } 5 | -------------------------------------------------------------------------------- /data/yago/sparql/query4.sparql: -------------------------------------------------------------------------------- 1 | PREFIX rdf: 2 | PREFIX rdfs: 3 | 4 | SELECT ?X WHERE { ?X } 5 | -------------------------------------------------------------------------------- /data/yago/sparql/query5.sparql: -------------------------------------------------------------------------------- 1 | PREFIX rdf: 2 | PREFIX rdfs: 3 | 4 | SELECT ?X WHERE { ?X / } 5 | -------------------------------------------------------------------------------- /data/yago/sparql/query6.sparql: -------------------------------------------------------------------------------- 1 | PREFIX rdf: 2 | PREFIX rdfs: 3 | 4 | SELECT ?X WHERE { ?X ?Y . FILTER NOT EXISTS { ?X ?Z } } 5 | -------------------------------------------------------------------------------- /exec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$JAVA" == "" ]; then 4 | JAVA="java" 5 | fi 6 | 7 | HEAPSPACE=-Xmx8g 8 | if [ $(hostname) = "elvis" ]; then 9 | HEAPSPACE=-Xmx200g 10 | fi 11 | 12 | HEAPDUMP="-XX:+HeapDumpOnOutOfMemoryError" 13 | MAXDISKCACHE=-Dstorage.diskCache.bufferSize=8192 14 | OPT="$HEAPSPACE $HEAPDUMP" 15 | 16 | remove-color() { 17 | sed "s/[[:cntrl:]]\[[0-9;]\{0,4\}m//g" 18 | } 19 | 20 | CPDIR=exec/ 21 | CP=$(echo -e "$([ -e $CPDIR/classpath-additional.info ] && cat $CPDIR/classpath-additional.info):$(cat $CPDIR/classpath.info-$HOSTNAME)" | tr "\\n" ":") 22 | CP=$(eval echo $CP) 23 | 24 | if [ "$1" == "update-classpath" ]; then 25 | echo "update classpath" 26 | mvn dependency:build-classpath | remove-color | grep -v "\[INFO\]" | grep -v "\[WARN" > $CPDIR/classpath.info-$HOSTNAME 27 | elif [ "$1" == "show-classpath" ]; then 28 | echo $CP 29 | else 30 | if [ "$1" == "hotswap" ]; then 31 | OPT="$OPT -XXaltjvm=dcevm -javaagent:$HOME/software/java/addons/hotswap-agent.jar=autoHotswap=true $MAXDISKCACHE $HEAPSPACE" 32 | shift 1 33 | else 34 | OPT="$OPT" 35 | fi 36 | #echo "java -cp $(. cp.sh) $*" 37 | cls=$1 38 | shift 1 39 | $JAVA $OPT -cp $CP:target/classes $cls "${@}" 40 | fi 41 | -------------------------------------------------------------------------------- /module/bashlog-web/.gitignore: -------------------------------------------------------------------------------- 1 | /build/ 2 | /target/ 3 | -------------------------------------------------------------------------------- /module/bashlog-web/WebContent/META-INF/MANIFEST.MF: -------------------------------------------------------------------------------- 1 | Manifest-Version: 1.0 2 | Class-Path: 3 | 4 | -------------------------------------------------------------------------------- /module/bashlog-web/WebContent/WEB-INF/tags/page.tag: -------------------------------------------------------------------------------- 1 | <%@ tag description="Overall Page template" pageEncoding="UTF-8"%> 2 | <%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core" %> 3 | <%-- <%@ attribute name="page" required="false" type="java.lang.String" %> --%> 4 | 5 | 6 | 7 | 8 | 9 | Bashlog API 10 | 11 | 12 | 13 | 14 | 15 | 16 |
17 |
18 | Bash Datalog
19 | Answering Datalog Queries with Unix Shell Commands 20 |
21 |
22 |
23 | About 24 | Datalog mode 25 | SPARQL/OWL mode 26 | API 27 |
28 |
29 |
30 | 31 |
32 | 33 |
34 | 35 | 36 | -------------------------------------------------------------------------------- /module/bashlog-web/WebContent/WEB-INF/web.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | elites 4 | 5 | index.html 6 | index.htm 7 | index.jsp 8 | default.html 9 | default.htm 10 | default.jsp 11 | 12 | 13 | 14 | *.jsp 15 | UTF-8 16 | 17 | 18 | -------------------------------------------------------------------------------- /module/bashlog-web/WebContent/about.jsp: -------------------------------------------------------------------------------- 1 | <%@page contentType="text/html" pageEncoding="UTF-8"%> 2 | <%@taglib prefix="t" tagdir="/WEB-INF/tags" %> 3 | 4 | 5 | 6 | 7 |
8 | 9 |

10 | This project translates datalog programs to Unix shell scripts. It can be used to preprocess large tabular datasets. 11 |

12 | 13 | 14 |

15 |

Features

16 | 17 |
    18 |
  • translate a Datalog program to a Bash script (→ Datalog mode)
    • 19 |
  • translate a SPARQL query and an OWL ontology to a Bash script (→ SPARQL/OWL mode)
    • 20 |
  • it can be used from the command line (→ API)
    • 21 |
22 |

23 | 24 |

25 |

Prerequisites:

26 |
    27 |
  • bash (no support for other shells, e.g., sh, tcsh, ksh, zsh)
    • 28 |
  • POSIX commands cat, join, sort, comm, ... (e.g. from the GNU coreutils package)
    • 29 |
  • AWK (e.g., MAWK or GNU awk; install MAWK for better performance)
    • 30 |
31 |

32 | 33 |

34 |

Technical information

35 | 36 | 40 | 41 | 42 |

43 | 44 |
45 | 46 | -------------------------------------------------------------------------------- /module/bashlog-web/WebContent/api.jsp: -------------------------------------------------------------------------------- 1 | <%@page contentType="text/html" pageEncoding="UTF-8"%> 2 | <%@taglib prefix="t" tagdir="/WEB-INF/tags" %> 3 | 4 | 5 | 6 | 7 | 8 |
9 |
10 | 11 |
12 | 13 |

Sparql/OWL

14 | 15 | An API for transforming SPARQL queries and an OWL ontology to bash scripts.
16 | 17 | Please call it as follows: 18 | 19 | curl --data-urlencode owl@ontology.owl --data-urlencode sparql@query.sparql --data-urlencode nTriples=/path/to/kb.ntriples ${url}/sparql 20 | 21 | where ontology.owl is a file containing the OWL ontology 22 | and sparql.query is a file containing the SPARQL query 23 | 24 |
25 | 26 | You can save the script in a file by changing the command as follows: 27 | 28 | curl ... > query.sh 29 | 30 | Execute it with the command bash query.sh. 31 | 32 |
33 | 34 |
35 | 36 |

Datalog

37 | 38 | An API for transforming datalog to bash scripts. 39 | For the syntax of the datalog dialect, see the datalog page

40 | 41 | Please call it as follows: 42 | 43 |
44 | 45 | curl --data-binary @example.dlog ${url}/datalog\?query=predicate 46 | 47 | where example.dlog contains your datalog program, here an example: 48 | 49 | facts(S,P,O) :~ cat ~/facts.tsv 50 | main(X) :- facts(X, _, "person"). 51 | 52 | 53 |
54 | 55 | You can save the script in a file by changing the command as follows: 56 | 57 | curl ... > query.sh 58 | 59 | Execute it with the command bash query.sh. 60 | 61 |
62 | 63 |
64 |
65 | 66 | -------------------------------------------------------------------------------- /module/bashlog-web/WebContent/convert_sparql.jsp: -------------------------------------------------------------------------------- 1 | <%@page contentType="text/html" pageEncoding="UTF-8"%> 2 | <%@taglib prefix="t" tagdir="/WEB-INF/tags" %> 3 | 4 | 5 | 6 | 7 |
8 |
9 | 10 |
11 | 12 |
13 | 14 |

SPARQL query

15 | 16 | 17 |

OWL ontology

18 | 19 | 20 |

N-Triple input file

21 | 22 | 23 |
24 | 26 | 27 |
28 | 29 | 30 |
31 | 32 |
33 |

Bash script

34 | 35 |
36 | 37 | 38 |
39 | 40 | 41 |
42 |
43 | 44 |

How to try it:

45 |
    46 |
  1. 47 | Enter a SPARQL query in the first textbox. 48 |
    2. 49 |
  2. 50 | Optional: Enter an OWL ontology in the second textbox 51 |
    3. 52 |
  3. 53 | Enter the path to an N-Triples file into the third textbox. 54 |
    4. 55 |
  4. 56 | Click on the Convert to bash script button 57 |
    5. 58 |
  5. 59 | Copy the content of the "Bash script" textbox into a file named query.sh in the folder with the .tsv files 60 | (or click on Download script) 61 |
    6. 62 |
  6. 63 | Run it with bash query.sh 64 |
    7. 65 |
66 | Note: the script uses a folder tmp for temporary files and removes its contents afterwards 67 |

API

68 | You can also use bashlog from the command line, without a browser. For details, see API. 69 | 70 | 71 |

Examples:

72 | You can try the examples on this dataset (source). 73 | 74 |
    75 |
  • Find people that died in the city where they were born 76 | BASE <http://yago-knowledge.org/resource/> 77 | SELECT ?X WHERE { 78 | ?X <wasBornIn> ?Y. 79 | ?X <diedIn> ?Y. 80 | } 81 |
    • 82 | 83 |
  • Living people 84 | BASE <http://yago-knowledge.org/resource/> 85 | SELECT ?X WHERE { 86 | { ?X <wasBornIn> []. } 87 | UNION 88 | { ?X <wasBornOnDate> []. } 89 | 90 | MINUS 91 | { ?X <diedIn> []. } 92 | MINUS 93 | { ?X <diedOnDate> []. } 94 | } 95 |
    • 96 | 97 |
  • All people 98 | BASE <http://yago-knowledge.org/resource/> 99 | PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> 100 | SELECT ?X WHERE { 101 | ?X rdf:type/rdfs:subClassOf* <wordnet_person_100007846>. 102 | } 103 |
    • 104 | 105 |
  • Facts in the query 106 |

    SPARQL 107 | BASE <http://yago-knowledge.org/resource/> 108 | SELECT ?X WHERE { 109 | ?X <type> . 110 | } 111 |

    112 | 113 |

    OWL 114 | @prefix kb: <http://yago-knowledge.org/resource/> . 115 | 116 | kb:albert kb:type kb:person. 117 | kb:marie kb:type kb:person. 118 |

    119 |
    • 120 |
121 |
122 |
123 |
124 | 125 |
126 | 127 | 128 | 129 | -------------------------------------------------------------------------------- /module/bashlog-web/WebContent/links.jsp: -------------------------------------------------------------------------------- 1 | <%@ page language="java" contentType="text/html; charset=UTF-8" 2 | pageEncoding="UTF-8"%> 3 | 4 |
5 | 6 | first 7 |
8 | 9 |
10 | This project translates datalog programs to Unix shell scripts. It can be used to preprocess large tabular datasets. 11 | It has a datalog mode, 12 | a SPARQL/OWL mode, 13 | and an API. 14 | You can obtain the source code here. 15 | We describe how it works in the technical report. 16 |
-------------------------------------------------------------------------------- /module/bashlog-web/WebContent/reset.css: -------------------------------------------------------------------------------- 1 | /* http://meyerweb.com/eric/tools/css/reset/ 2 | v2.0 | 20110126 3 | License: none (public domain) 4 | */ 5 | 6 | html, body, div, span, applet, object, iframe, 7 | h1, h2, h3, h4, h5, h6, p, blockquote, pre, 8 | a, abbr, acronym, address, big, cite, code, 9 | del, dfn, em, img, ins, kbd, q, s, samp, 10 | small, strike, strong, sub, sup, tt, var, 11 | b, u, i, center, 12 | dl, dt, dd, ol, ul, li, 13 | fieldset, form, label, legend, 14 | table, caption, tbody, tfoot, thead, tr, th, td, 15 | article, aside, canvas, details, embed, 16 | figure, figcaption, footer, header, hgroup, 17 | menu, nav, output, ruby, section, summary, 18 | time, mark, audio, video { 19 | margin: 0; 20 | padding: 0; 21 | border: 0; 22 | font-size: 100%; 23 | font: inherit; 24 | vertical-align: baseline; 25 | } 26 | /* HTML5 display-role reset for older browsers */ 27 | article, aside, details, figcaption, figure, 28 | footer, header, hgroup, menu, nav, section { 29 | display: block; 30 | } 31 | body { 32 | line-height: 1; 33 | } 34 | ol, ul { 35 | list-style: none; 36 | } 37 | blockquote, q { 38 | quotes: none; 39 | } 40 | blockquote:before, blockquote:after, 41 | q:before, q:after { 42 | content: ''; 43 | content: none; 44 | } 45 | table { 46 | border-collapse: collapse; 47 | border-spacing: 0; 48 | } -------------------------------------------------------------------------------- /module/bashlog-web/WebContent/style.css: -------------------------------------------------------------------------------- 1 | 2 | header { 3 | position: fixed; 4 | left:0; 5 | right:0; 6 | top:0; 7 | padding: 5px; 8 | text-align: center; 9 | padding: 0px; 10 | } 11 | 12 | header .headline { 13 | margin:0px; 14 | background-color: #ccccff; 15 | border-color: #aaaaee; 16 | border-width: 0px 0px 2px 0px; 17 | border-style: solid; 18 | } 19 | 20 | .menu { 21 | margin: 8px; 22 | background-color: #eeeeff; 23 | margin:0; 24 | height: 30px; 25 | display: flex; 26 | justify-content: center; 27 | flex-direction: column; 28 | border-radius: 0px 0px 12px 12px; 29 | } 30 | 31 | a { 32 | text-decoration: none; 33 | } 34 | 35 | .menu a { 36 | padding: 10px 20px; 37 | position: relative; 38 | display: inline-block; 39 | } 40 | .menu a span::before { 41 | width: 5px; 42 | height: 5px; 43 | background: transparent; 44 | content: ""; 45 | position: absolute; 46 | left: 0; 47 | top: 0; 48 | border-top: 2px solid ; 49 | border-left: 2px solid ; 50 | opacity: 0; 51 | } 52 | .menu a span::after { 53 | width: 5px; 54 | height: 5px; 55 | background: transparent; 56 | content: ""; 57 | position: absolute; 58 | right: 0; 59 | bottom: 0; 60 | border-right: 2px solid ; 61 | border-bottom: 2px solid ; 62 | opacity: 0; 63 | } 64 | .menu a::before { 65 | width: 5px; 66 | height: 5px; 67 | background: transparent; 68 | content: ""; 69 | position: absolute; 70 | right: 0; 71 | top: 0; 72 | border-right: 2px solid ; 73 | border-top: 2px solid ; 74 | opacity: 0; 75 | } 76 | .menu a::after { 77 | width: 5px; 78 | height: 5px; 79 | background: transparent; 80 | content: ""; 81 | position: absolute; 82 | left: 0; 83 | bottom: 0; 84 | border-left: 2px solid; 85 | border-bottom: 2px solid; 86 | opacity: 0; 87 | } 88 | .menu a:hover { 89 | /*color: #000;*/ 90 | } 91 | .menu a:hover::before { 92 | opacity: 1; 93 | right: 5px; 94 | top: 5px; 95 | } 96 | .menu a:hover::after { 97 | opacity: 1; 98 | left: 5px; 99 | bottom: 5px; 100 | } 101 | .menu a:hover span::before { 102 | opacity: 1; 103 | left: 5px; 104 | top: 5px; 105 | } 106 | .menu a:hover span::after { 107 | opacity: 1; 108 | right: 5px; 109 | bottom: 5px; 110 | } 111 | 112 | 113 | main { 114 | padding: 80px 0px 0px; 115 | } 116 | 117 | @media ( min-width : 700px) { 118 | .nograil { 119 | margin-left: 40px; 120 | margin-right: 40px; 121 | } 122 | } 123 | 124 | 125 | /* HolyGrail: https://philipwalton.github.io/solved-by-flexbox/demos/holy-grail/ */ 126 | .grail, .grail-body { display: flex; flex-direction: column; } 127 | .grail-content { flex: 1; } 128 | .grail-nav { order: -1; } 129 | @media ( min-width : 700px) { 130 | .grail-body { flex-direction: row; flex: 1; } 131 | .grail-content { flex: 1; margin-left:40px; margin-right:40px; } 132 | } 133 | 134 | h1 { 135 | font-weight: bold; 136 | font-size: 20px; 137 | } 138 | 139 | h2 { 140 | font-weight: bold; 141 | font-size: 15px; 142 | } 143 | 144 | textarea { 145 | padding: 10px; 146 | width: 95%; 147 | height: 20vh; 148 | 149 | border-radius: 5px; 150 | border: 1px gray solid; 151 | } 152 | 153 | /*div { 154 | border: 1px dashed; 155 | }*/ 156 | 157 | code { 158 | font-family: monospace; 159 | white-space: pre-wrap; 160 | padding: 10px; 161 | margin: 5px; 162 | background-color: #eeeeee; 163 | display: block; 164 | 165 | border-radius: 5px; 166 | border: 1px gray solid; 167 | } 168 | 169 | code.inline { 170 | display: inline-block; 171 | padding: 3px 5px; 172 | margin: 0px 2px; 173 | } 174 | 175 | button { 176 | font-size: 15px; 177 | background-color: #ccccff; 178 | padding: 10px; 179 | border-color: #aaaaee; 180 | border-width: 2px; 181 | border-style: solid; 182 | border-radius: 12px; 183 | } 184 | 185 | ul li { 186 | margin: 5px 0 5px 0; 187 | } 188 | 189 | ul.examples li { 190 | margin: 20px 0 20px 0; 191 | } 192 | -------------------------------------------------------------------------------- /module/bashlog-web/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | bashlog-web 4 | bashlog-web 5 | 0.0.1-SNAPSHOT 6 | war 7 | 8 | 9 | 10 | maven-compiler-plugin 11 | 3.7.0 12 | 13 | 1.8 14 | 1.8 15 | 16 | 17 | 18 | maven-war-plugin 19 | 3.0.0 20 | 21 | WebContent 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | org.slf4j 31 | slf4j-api 32 | 1.7.25 33 | 34 | 44 | 45 | org.slf4j 46 | slf4j-simple 47 | 1.7.25 48 | test 49 | 50 | 51 | 52 | com.beust 53 | jcommander 54 | 1.72 55 | 56 | 57 | 58 | net.sourceforge.owlapi 59 | owlapi-distribution 60 | 5.1.4 61 | 62 | 63 | org.apache.commons 64 | commons-rdf-rdf4j 65 | 0.5.0 66 | 67 | 68 | 69 | javax.servlet 70 | javax.servlet-api 71 | 3.0.1 72 | provided 73 | 74 | 75 | jstl 76 | jstl 77 | 1.2 78 | 79 | 80 | -------------------------------------------------------------------------------- /module/bashlog-web/script/compare-bashlog-web.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | api=http://localhost:8080/bashlog-web/api 4 | 5 | mkdir -p queries/data 6 | ( 7 | cd queries 8 | data_dir=$(realpath data) 9 | 10 | ( 11 | cd $data_dir 12 | [ ! -e "sample.zip" ] && wget -N http://resources.mpi-inf.mpg.de/yago-naga/yago3.1/sample.zip && unzip sample.zip 13 | [ ! -e "sample-ntriples.zip" ] && wget -N http://resources.mpi-inf.mpg.de/yago-naga/yago3.1/sample-ntriples.zip && unzip sample-ntriples.zip 14 | ) 15 | 16 | result_dir=$(realpath result) 17 | mkdir -p $result_dir 18 | 19 | for j in *.datalog; do 20 | i=$(basename $j .datalog) 21 | echo "running $i" 22 | 23 | # generate bash code 24 | # datalog query 25 | curl -s --data-binary @./$i.datalog $api/datalog\?query\=main\&debug_algebra\&debug_datalog > $result_dir/$i.bash-datalog 26 | # sparql/owl query 27 | curl -s --data-urlencode owl@./$i.owl --data-urlencode sparql@./$i.sparql --data-urlencode nTriples=yago-sample.ntriples $api/sparql?debug_algebra\&debug_datalog > $result_dir/$i.bash-sparql 28 | 29 | # run bash code 30 | ( 31 | cd $data_dir 32 | bash $result_dir/$i.bash-datalog > $result_dir/$i.result-datalog 33 | bash $result_dir/$i.bash-sparql > $result_dir/$i.result-sparql 34 | ) 35 | 36 | # check result 37 | c1=$(wc -l < $result_dir/$i.result-datalog) 38 | c2=$(wc -l < $result_dir/$i.result-sparql) 39 | 40 | if [ "$c1" == "$c2" ]; then 41 | 42 | echo "ok: $(printf '%-20s' "$i") produced $c1 rows" 43 | else 44 | echo "" 45 | echo "count $i datalog: $c1" 46 | echo "count $i sparql : $c2" 47 | echo "" 48 | fi 49 | 50 | done 51 | ) 52 | 53 | -------------------------------------------------------------------------------- /module/bashlog-web/script/queries/.gitignore: -------------------------------------------------------------------------------- 1 | /data/ 2 | /result/ 3 | -------------------------------------------------------------------------------- /module/bashlog-web/script/queries/all-people.datalog: -------------------------------------------------------------------------------- 1 | facts(_, S, P, O) :~ cat *.tsv 2 | 3 | type(X, Y) :- facts(_, X, "rdf:type", Y). 4 | subclass(X, Y) :- facts(_, X, "rdfs:subClassOf", Y). 5 | type(X, Z) :- type(X, Y), subclass(Y, Z). 6 | 7 | main(X) :- type(X, ""). 8 | 9 | -------------------------------------------------------------------------------- /module/bashlog-web/script/queries/all-people.owl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasrebele/bashlog/906f1470c06a8ebfa12637ef31a8a81cbd1cf4f6/module/bashlog-web/script/queries/all-people.owl -------------------------------------------------------------------------------- /module/bashlog-web/script/queries/all-people.sparql: -------------------------------------------------------------------------------- 1 | BASE 2 | PREFIX rdf: 3 | SELECT ?X WHERE { 4 | ?X rdf:type/rdfs:subClassOf* . 5 | } 6 | 7 | 8 | -------------------------------------------------------------------------------- /module/bashlog-web/script/queries/facts-program.datalog: -------------------------------------------------------------------------------- 1 | type("albert", "person"). 2 | type("marie", "person"). 3 | main(X) :- type(X, "person"). 4 | -------------------------------------------------------------------------------- /module/bashlog-web/script/queries/facts-program.owl: -------------------------------------------------------------------------------- 1 | @prefix kb: . 2 | 3 | kb:albert kb:type kb:person. 4 | kb:marie kb:type kb:person. 5 | 6 | -------------------------------------------------------------------------------- /module/bashlog-web/script/queries/facts-program.sparql: -------------------------------------------------------------------------------- 1 | BASE 2 | SELECT ?X WHERE { 3 | ?X . 4 | } 5 | 6 | 7 | -------------------------------------------------------------------------------- /module/bashlog-web/script/queries/living-people.datalog: -------------------------------------------------------------------------------- 1 | facts(_, S, P, O) :~ cat *.tsv 2 | born(X) :- facts(_, X, "", Y). 3 | born(X) :- facts(_, X, "", Y). 4 | dead(X) :- facts(_, X, "", Y). 5 | dead(X) :- facts(_, X, "", Y). 6 | 7 | main(X) :- born(X), not dead(X). 8 | -------------------------------------------------------------------------------- /module/bashlog-web/script/queries/living-people.owl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasrebele/bashlog/906f1470c06a8ebfa12637ef31a8a81cbd1cf4f6/module/bashlog-web/script/queries/living-people.owl -------------------------------------------------------------------------------- /module/bashlog-web/script/queries/living-people.sparql: -------------------------------------------------------------------------------- 1 | BASE 2 | SELECT ?X WHERE { 3 | { ?X []. } 4 | UNION 5 | { ?X []. } 6 | 7 | MINUS 8 | { ?X []. } 9 | MINUS 10 | { ?X []. } 11 | } 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /module/bashlog-web/script/queries/same-city.datalog: -------------------------------------------------------------------------------- 1 | facts(_, S, P, O) :~ cat *.tsv 2 | main(X) :- 3 | facts(_, X, "", Y), 4 | facts(_, X, "", Y). 5 | -------------------------------------------------------------------------------- /module/bashlog-web/script/queries/same-city.sparql: -------------------------------------------------------------------------------- 1 | BASE 2 | SELECT ?X WHERE { 3 | ?X ?Y. 4 | ?X ?Y. 5 | } 6 | 7 | 8 | -------------------------------------------------------------------------------- /module/bashlog-web/src/main/java/bashlog: -------------------------------------------------------------------------------- 1 | ../../../../../src/main/java/bashlog -------------------------------------------------------------------------------- /module/bashlog-web/src/main/java/bashlogweb/Main.java: -------------------------------------------------------------------------------- 1 | package bashlogweb; 2 | 3 | import java.io.IOException; 4 | import java.net.URL; 5 | import java.util.Arrays; 6 | import java.util.Collections; 7 | import java.util.HashMap; 8 | import java.util.List; 9 | 10 | import javax.servlet.ServletException; 11 | import javax.servlet.annotation.WebServlet; 12 | import javax.servlet.http.HttpServlet; 13 | import javax.servlet.http.HttpServletRequest; 14 | import javax.servlet.http.HttpServletResponse; 15 | 16 | @WebServlet({ "/index.jsp", "/datalog", "/sparql" }) 17 | public class Main extends HttpServlet { 18 | 19 | private static final long serialVersionUID = -4646304382919974568L; 20 | 21 | @Override 22 | protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { 23 | doPost(req, resp); 24 | } 25 | 26 | @Override 27 | protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { 28 | URL uurl = new URL(req.getRequestURL().toString()); 29 | 30 | String bashlog = ""; 31 | String page = "about.jsp"; 32 | if (uurl.getPath().contains("datalog")) { 33 | String datalog = req.getParameter("datalog"); 34 | 35 | if (datalog != null) { 36 | bashlog = API.processDatalogQuery(datalog, req, resp); 37 | } else { 38 | datalog = "facts(S,P,O) :~ cat ~/facts.tsv\n" + // 39 | "main(X) :- facts(X, _, \"person\")."; 40 | } 41 | req.setAttribute("datalog", datalog); 42 | page = "convert_datalog.jsp"; 43 | 44 | } else if(uurl.getPath().contains("sparql")) { 45 | String owl = req.getParameter("owl"); 46 | String sparql = req.getParameter("sparql"); 47 | String nTriples = req.getParameter("nTriples"); 48 | 49 | if(sparql != null) { 50 | HashMap> params = new HashMap<>(); 51 | params.put("owl", Collections.singletonList(owl)); 52 | params.put("sparql", Collections.singletonList(sparql)); 53 | params.put("nTriples", Collections.singletonList(nTriples)); 54 | 55 | bashlog = API.processSparqlQuery(params, req, resp); 56 | } else { 57 | sparql = "BASE \n" + 58 | "SELECT ?x WHERE { ?X rdf:type }"; 59 | } 60 | if(nTriples == null) { 61 | nTriples = "/path/to/knowledge-base.ntriples"; 62 | } 63 | 64 | req.setAttribute("owl", owl); 65 | req.setAttribute("sparql", sparql); 66 | req.setAttribute("nTriples", nTriples); 67 | page = "convert_sparql.jsp"; 68 | } 69 | 70 | req.setAttribute("bashlog", bashlog); 71 | 72 | if (req.getParameter("download") != null) { 73 | req.getParameterMap().forEach((k, v) -> System.out.println(k + " " + Arrays.toString(v))); 74 | resp.setContentType("application/octet-stream"); 75 | resp.setHeader("Content-Disposition", "filename=\"query.sh\""); 76 | resp.getOutputStream().write(bashlog.getBytes()); 77 | return; 78 | } 79 | 80 | req.getRequestDispatcher(page).forward(req, resp); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /module/bashlog-web/src/main/java/common: -------------------------------------------------------------------------------- 1 | ../../../../../src/main/java/common -------------------------------------------------------------------------------- /module/bashlog-web/src/main/java/rdf: -------------------------------------------------------------------------------- 1 | ../../../../../src/main/java/rdf -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | org.yago-knowledge 5 | bashlog 6 | 0.0.1-SNAPSHOT 7 | 8 | 14 | 15 | 16 | UTF-8 17 | 1.8 18 | 1.4.0 19 | 20 | 21 | 22 | 23 | 24 | src/main/resources 25 | 26 | **/*.java 27 | 28 | 29 | 30 | 31 | 32 | maven-compiler-plugin 33 | 3.5.1 34 | 35 | ${java.version} 36 | ${java.version} 37 | UTF-8 38 | 39 | 40 | 41 | org.codehaus.mojo 42 | exec-maven-plugin 43 | 44 | bashlog.MainThomasWikidata 45 | 46 | 47 | 48 | maven-assembly-plugin 49 | 50 | 51 | 52 | bashlog.Cmd 53 | 54 | 55 | 56 | jar-with-dependencies 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | com.beust 66 | jcommander 67 | 1.72 68 | 69 | 70 | 71 | 72 | org.slf4j 73 | slf4j-api 74 | 1.7.25 75 | 76 | 77 | 82 | 83 | 84 | org.slf4j 85 | slf4j-simple 86 | 1.7.25 87 | 88 | 89 | net.sourceforge.owlapi 90 | owlapi-distribution 91 | 5.1.4 92 | 93 | 94 | org.apache.commons 95 | commons-rdf-rdf4j 96 | 0.5.0 97 | 98 | 99 | 100 | junit 101 | junit 102 | 4.13.1 103 | test 104 | 105 | 106 | org.apache.commons 107 | commons-compress 108 | 1.21 109 | test 110 | 111 | 112 | 113 | -------------------------------------------------------------------------------- /src/main/java/bashlog/Cmd.java: -------------------------------------------------------------------------------- 1 | package bashlog; 2 | 3 | import java.io.IOException; 4 | import java.util.Set; 5 | 6 | import com.beust.jcommander.JCommander; 7 | import com.beust.jcommander.Parameter; 8 | 9 | import common.parser.Program; 10 | 11 | /** Command line program to translate a bashlog datalog program to a bash script. */ 12 | public class Cmd { 13 | 14 | /** Command line arguments */ 15 | public static class Args { 16 | 17 | @Parameter(names = { "--help", "-h" }, description = "help", help=true, hidden=true) 18 | public boolean help; 19 | 20 | @Parameter(names = { "--plan" }, description = "print plan") 21 | public boolean debug; 22 | 23 | @Parameter(names = "--query-file", description = "a Bash Datalog query file, which contains datalog rules") 24 | private String queryFile; 25 | 26 | @Parameter(names = "--query-pred", description = "the predicate that should be evaluated") 27 | private String queryPredicate; 28 | } 29 | 30 | public static void main(String[] argv) throws IOException { 31 | // parse arguments 32 | Args args = new Args(); 33 | JCommander cmd = JCommander.newBuilder().addObject(args).build(); 34 | cmd.parse(argv); 35 | for (String str : cmd.getUnknownOptions()) { 36 | System.out.println("warning: unknown option " + str + ""); 37 | } 38 | if (argv.length == 0 || args.help) { 39 | cmd.usage(); 40 | return; 41 | } 42 | 43 | // translate/compile 44 | Set features = BashlogCompiler.BASHLOG_PARSER_FEATURES; 45 | Program p = Program.loadFile(args.queryFile, features); 46 | 47 | String queryPred = args.queryPredicate; 48 | queryPred = p.searchRelation(queryPred); 49 | if (queryPred == null || queryPred.trim().isEmpty()) { 50 | queryPred = p.rules().get(p.rules().size() - 1).head.getRelation(); 51 | } 52 | 53 | BashlogCompiler bc = BashlogCompiler.prepareQuery(p, queryPred); 54 | try { 55 | if(args.debug) { 56 | bc.enableDebug(); 57 | } 58 | String bash = bc.compile("", "", false); 59 | System.out.println(bash); 60 | if (args.debug) { 61 | System.out.println(bc.debugInfo()); 62 | } 63 | } catch (Exception e) { 64 | System.out.println(bc.debugInfo()); 65 | throw (e); 66 | } 67 | } 68 | 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/bashlog/plan/BashlogOptimizer.java: -------------------------------------------------------------------------------- 1 | package bashlog.plan; 2 | 3 | import java.util.Arrays; 4 | 5 | import common.Tools; 6 | import common.plan.node.*; 7 | import common.plan.optimizer.Optimizer; 8 | 9 | // TODO: "push up" sorting (only useful on topmost sort?) 10 | public class BashlogOptimizer implements Optimizer { 11 | 12 | public PlanNode apply(PlanNode node) { 13 | return node.transform((n) -> { 14 | if (n instanceof SortNode) { 15 | SortNode sn = (SortNode) n; 16 | int[] parentSortCols = sn.sortColumns(); 17 | int[] childSortCols = null; 18 | 19 | if (sn.getTable() instanceof RecursionNode) { 20 | childSortCols = Tools.sequence(sn.getTable().getArity()); 21 | } else if (sn.getTable() instanceof SortNode) { 22 | childSortCols = ((SortNode) sn.getTable()).sortColumns(); 23 | } 24 | 25 | if (childSortCols != null && parentSortCols.length <= childSortCols.length) { 26 | for(int i=0; i children() { 38 | return Arrays.asList(child); 39 | } 40 | 41 | @Override 42 | public int hashCode() { 43 | return child.hashCode() ^ Arrays.hashCode(columns); 44 | } 45 | 46 | @Override 47 | public boolean equals(Object obj) { 48 | return obj instanceof CombinedColumnNode && Objects.equals(child, ((CombinedColumnNode) obj).child) 49 | && Arrays.equals(columns, ((CombinedColumnNode) obj).columns); 50 | } 51 | 52 | @Override 53 | public PlanNode transform(TransformFn fn, List originalPath) { 54 | try { 55 | Tools.addLast(originalPath, this); 56 | return fn.apply(this, new CombinedColumnNode(child.transform(fn, originalPath), columns), originalPath); 57 | } finally { 58 | Tools.removeLast(originalPath); 59 | } 60 | } 61 | 62 | public PlanNode getTable() { 63 | return child; 64 | } 65 | 66 | public int[] getColumns() { 67 | return columns; 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/bashlog/plan/SortAntiJoinNode.java: -------------------------------------------------------------------------------- 1 | package bashlog.plan; 2 | 3 | import java.util.*; 4 | 5 | import common.Tools; 6 | import common.plan.node.PlanNode; 7 | 8 | /** Anti-Join two sorted inputs based on ONE column */ 9 | public class SortAntiJoinNode extends SortJoinNode { 10 | 11 | public SortAntiJoinNode(PlanNode left, PlanNode right, int[] leftJoinProjection) { 12 | this(left, right, leftJoinProjection, Tools.sequence(left.getArity() )); 13 | } 14 | 15 | public SortAntiJoinNode(PlanNode left, PlanNode right, int[] leftJoinProjection, int[] outputProjection) { 16 | super(left, right, leftJoinProjection, leftJoinProjection.length == 1 ? new int[] { 0 } : new int[] {}, outputProjection); 17 | if (leftJoinProjection.length > 1) throw new UnsupportedOperationException("sort join does not support sorting on more than one column"); 18 | } 19 | 20 | @Override 21 | public String operatorString() { 22 | return super.operatorString().replaceFirst("⋈_", "▷_"); 23 | } 24 | 25 | @Override 26 | public PlanNode transform(TransformFn fn, List originalPath) { 27 | try { 28 | return fn.apply(this, 29 | new SortAntiJoinNode(getLeft().transform(fn, originalPath), getRight().transform(fn, originalPath), getLeftProjection(), 30 | outputProjection), 31 | originalPath); 32 | } finally { 33 | Tools.removeLast(originalPath); 34 | } 35 | } 36 | 37 | @Override 38 | public boolean equals(Object obj) { 39 | return equals(obj, new HashMap<>()); 40 | } 41 | 42 | @Override 43 | public boolean equals(Object obj, Map assumedEqualities) { 44 | if (this == obj) return true; 45 | if (!(obj.getClass() == getClass())) { 46 | return false; 47 | } 48 | return super.equals(obj, assumedEqualities); 49 | } 50 | 51 | @Override 52 | public int hashCode() { 53 | return super.hashCode(); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/bashlog/plan/SortJoinNode.java: -------------------------------------------------------------------------------- 1 | package bashlog.plan; 2 | 3 | import java.util.*; 4 | 5 | import common.Tools; 6 | import common.plan.node.JoinNode; 7 | import common.plan.node.PlanNode; 8 | 9 | /** Join two sorted inputs based on ONE column */ 10 | public class SortJoinNode extends JoinNode { 11 | 12 | protected final int[] outputProjection; 13 | 14 | public SortJoinNode(PlanNode left, PlanNode right, int[] leftJoinProjection, int[] rightJoinProjection) { 15 | this(left, right, leftJoinProjection, rightJoinProjection, Tools.sequence(left.getArity() + right.getArity())); 16 | } 17 | 18 | public SortJoinNode(PlanNode left, PlanNode right, int[] leftJoinProjection, int[] rightJoinProjection, int[] outputProjection) { 19 | super(left, right, leftJoinProjection, rightJoinProjection); 20 | if (leftJoinProjection.length > 1) throw new UnsupportedOperationException("sort join does not support sorting on more than one column"); 21 | if (leftJoinProjection.length != rightJoinProjection.length) throw new UnsupportedOperationException("join requires one column on each child"); 22 | this.outputProjection = outputProjection; 23 | } 24 | 25 | @Override 26 | public String operatorString() { 27 | return super.operatorString().replaceFirst("_", "_sort_") + " out " + Arrays.toString(outputProjection); 28 | } 29 | 30 | @Override 31 | public PlanNode transform(TransformFn fn, List originalPath) { 32 | try { 33 | Tools.addLast(originalPath, this); 34 | return fn.apply(this, new SortJoinNode(getLeft().transform(fn, originalPath), getRight().transform(fn, originalPath), getLeftProjection(), 35 | getRightProjection(), outputProjection), originalPath); 36 | } finally { 37 | Tools.removeLast(originalPath); 38 | } 39 | } 40 | 41 | public int[] getOutputProjection() { 42 | return outputProjection; 43 | } 44 | 45 | @Override 46 | public boolean equals(Object obj) { 47 | return equals(obj, new HashMap<>()); 48 | } 49 | 50 | @Override 51 | public boolean equals(Object obj, Map assumedEqualities) { 52 | if (this == obj) return true; 53 | if (!(obj.getClass() == getClass())) { 54 | return false; 55 | } 56 | SortJoinNode node = (SortJoinNode) obj; 57 | return super.equals(obj, assumedEqualities) && Arrays.equals(outputProjection, node.outputProjection); 58 | } 59 | 60 | @Override 61 | public int hashCode() { 62 | return super.hashCode() ^ Arrays.hashCode(outputProjection); 63 | } 64 | 65 | @Override 66 | public int getArity() { 67 | return outputProjection.length; 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/bashlog/plan/SortNode.java: -------------------------------------------------------------------------------- 1 | package bashlog.plan; 2 | 3 | import java.util.*; 4 | 5 | import common.Tools; 6 | import common.plan.node.PlanNode; 7 | 8 | public class SortNode implements PlanNode { 9 | 10 | private final PlanNode child; 11 | 12 | private final int[] sortColumns; 13 | 14 | public SortNode(PlanNode child, int[] sortColumns) { 15 | this.child = child; 16 | //this.sortColumns = sortColumns == null ? Tools.sequence(child.getArity()) : sortColumns; 17 | this.sortColumns = sortColumns; 18 | } 19 | 20 | @Override 21 | public int getArity() { 22 | return child.getArity(); 23 | } 24 | 25 | public PlanNode getTable() { 26 | return child; 27 | } 28 | 29 | @Override 30 | public String toString() { 31 | return toPrettyString(); //operatorString() + "(" + child + ")"; 32 | } 33 | 34 | @Override 35 | public String operatorString() { 36 | return "sort_{" + Arrays.toString(sortColumns) + "}"; 37 | } 38 | 39 | @Override 40 | public List children() { 41 | return Arrays.asList(child); 42 | } 43 | 44 | public int[] sortColumns() { 45 | if (sortColumns == null) return Tools.sequence(getArity()); 46 | return sortColumns; 47 | } 48 | 49 | @Override 50 | public boolean equals(Object obj) { 51 | return equals(obj, Collections.emptyMap()); 52 | } 53 | 54 | public boolean equals(Object obj, Map assumedEqualities) { 55 | if (this == obj) return true; 56 | if (!(obj.getClass() == getClass())) { 57 | return false; 58 | } 59 | SortNode node = (SortNode) obj; 60 | return Arrays.equals(sortColumns, node.sortColumns) && child.equals(node.child, assumedEqualities); 61 | } 62 | 63 | @Override 64 | public int hashCode() { 65 | return Objects.hash(child, Arrays.hashCode(sortColumns)); 66 | } 67 | 68 | @Override 69 | public PlanNode transform(TransformFn fn, List originalPath) { 70 | try { 71 | Tools.addLast(originalPath, this); 72 | return fn.apply(this, new SortNode(child.transform(fn, originalPath), sortColumns), originalPath); 73 | } finally { 74 | Tools.removeLast(originalPath); 75 | } 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/bashlog/plan/TSVFileNode.java: -------------------------------------------------------------------------------- 1 | package bashlog.plan; 2 | 3 | import java.util.Collections; 4 | import java.util.List; 5 | 6 | import common.plan.node.PlanNode; 7 | 8 | public class TSVFileNode implements PlanNode { 9 | 10 | final String path; 11 | 12 | final int arity; 13 | 14 | public TSVFileNode(String path, int arity) { 15 | this.path = path; 16 | this.arity = arity; 17 | } 18 | 19 | @Override 20 | public int getArity() { 21 | return arity; 22 | } 23 | 24 | @Override 25 | public String operatorString() { 26 | return "file: " + path; 27 | } 28 | 29 | @Override 30 | public List children() { 31 | return Collections.emptyList(); 32 | } 33 | 34 | public String getPath() { 35 | return path; 36 | } 37 | 38 | @Override 39 | public boolean equals(Object obj) { 40 | if (!(obj.getClass() == getClass())) { 41 | return false; 42 | } 43 | TSVFileNode node = (TSVFileNode) obj; 44 | return this.path.equals(node.path) && this.arity == node.arity; 45 | } 46 | 47 | @Override 48 | public int hashCode() { 49 | return path.hashCode() ^ this.arity; 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/bashlog/translation/BashCmd.java: -------------------------------------------------------------------------------- 1 | package bashlog.translation; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | 6 | import bashlog.CompilerInternals; 7 | import bashlog.command.Bash; 8 | import common.parser.ParserReader; 9 | import common.plan.node.BashNode; 10 | import common.plan.node.PlanNode; 11 | 12 | /** Translate a builtin command, which is a bash command, to a bash command */ 13 | public class BashCmd implements BashTranslator { 14 | 15 | @Override 16 | public Bash translate(PlanNode planNode, CompilerInternals ci) { 17 | BashNode bn = (BashNode) planNode; 18 | List cmdParts = bn.getCommandParts(); 19 | Bash.Command bc = new Bash.Command(cmdParts.get(0)); 20 | 21 | List children = bn.children(); 22 | for (int i = 0; i < children.size(); i++) { 23 | bc.file(ci.compile(children.get(i))); 24 | bc.arg(cmdParts.get(i + 1)); 25 | } 26 | 27 | // instead of "<(cat file)", use file directly 28 | String cmd = bn.getCommand(); 29 | if (cmd.startsWith("cat ")) { 30 | ParserReader pr = new ParserReader(cmd); 31 | pr.expect("cat "); 32 | pr.skipWhitespace(); 33 | String file; 34 | if (pr.peek() == '\"' || pr.peek() == '\'') file = pr.readString(); 35 | else file = pr.readWhile((c, s) -> !Character.isWhitespace(c)); 36 | pr.skipWhitespace(); 37 | if (pr.peek() == '\0') { 38 | if (!file.startsWith("!")) { 39 | return new Bash.BashFile(file); 40 | } 41 | else { 42 | return ci.compile(children.get(0)); 43 | } 44 | } 45 | } 46 | 47 | return bc; 48 | } 49 | 50 | @Override 51 | public List> supports() { 52 | return Arrays.asList(BashNode.class); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/bashlog/translation/BashTranslator.java: -------------------------------------------------------------------------------- 1 | package bashlog.translation; 2 | 3 | 4 | import bashlog.command.Bash; 5 | 6 | /** Interface for bashlog translators ("type alias") */ 7 | public interface BashTranslator extends common.compiler.Translator { 8 | 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/bashlog/translation/CombineColumns.java: -------------------------------------------------------------------------------- 1 | package bashlog.translation; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | 6 | import bashlog.CompilerInternals; 7 | import bashlog.command.Bash; 8 | import bashlog.plan.CombinedColumnNode; 9 | import common.plan.node.PlanNode; 10 | 11 | /** Translate a combine column to an AWK command */ 12 | public class CombineColumns implements BashTranslator { 13 | 14 | @Override 15 | public Bash translate(PlanNode planNode, CompilerInternals bc) { 16 | CombinedColumnNode c = (CombinedColumnNode) planNode; 17 | Bash prev = bc.compile(c.getTable()); 18 | 19 | Bash.Pipe result = new Bash.Pipe(prev); 20 | Bash.Command cmd = result.cmd(AwkHelper.AWK); 21 | StringBuilder sb = new StringBuilder(); 22 | sb.append("{ print $0 FS "); 23 | for (int i = 0; i < c.getColumns().length; i++) { 24 | if (i > 0) { 25 | sb.append(" \"\\002\" "); 26 | } 27 | sb.append("$" + (c.getColumns()[i] + 1)); 28 | } 29 | sb.append("}'"); 30 | cmd.arg(sb.toString()); 31 | return result; 32 | 33 | } 34 | 35 | @Override 36 | public List> supports() { 37 | return Arrays.asList(CombinedColumnNode.class); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/bashlog/translation/Fact.java: -------------------------------------------------------------------------------- 1 | package bashlog.translation; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | import java.util.stream.Collectors; 6 | 7 | import bashlog.CompilerInternals; 8 | import bashlog.command.Bash; 9 | import common.plan.node.FactNode; 10 | import common.plan.node.PlanNode; 11 | 12 | /** Translate a fact node that represents a list of facts directly stored in the datalog program. */ 13 | public class Fact implements BashTranslator { 14 | 15 | @Override 16 | public Bash translate(PlanNode planNode, CompilerInternals bc) { 17 | FactNode j = (FactNode) planNode; 18 | 19 | Bash.CommandSequence result = new Bash.CommandSequence(); 20 | Bash.Command c = new Bash.Command("cat "); 21 | StringBuilder content = new StringBuilder(); 22 | for (Comparable[] fact : j.getFacts()) { 23 | content.append(Arrays.stream(fact).map(f -> f.toString()).collect(Collectors.joining("\t"))); 24 | content.append("\n"); 25 | } 26 | c.heredoc(new Bash.Heredoc("EOF", content.toString())); 27 | 28 | // wrap in a function 29 | Bash function = c.wrap("relation() {\n", "}"); 30 | result.add(function); 31 | result.add(new Bash.Command("relation")); 32 | return result; 33 | } 34 | 35 | @Override 36 | public List> supports() { 37 | return Arrays.asList(FactNode.class); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/bashlog/translation/FileInput.java: -------------------------------------------------------------------------------- 1 | package bashlog.translation; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | 6 | import bashlog.CompilerInternals; 7 | import bashlog.command.Bash; 8 | import bashlog.plan.TSVFileNode; 9 | import common.plan.node.PlanNode; 10 | 11 | /** Translate a file to a file */ 12 | public class FileInput implements BashTranslator { 13 | 14 | @Override 15 | public Bash translate(PlanNode planNode, CompilerInternals bc) { 16 | TSVFileNode file = (TSVFileNode) planNode; 17 | return new Bash.BashFile(file.getPath()); 18 | } 19 | 20 | @Override 21 | public List> supports() { 22 | return Arrays.asList(TSVFileNode.class); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/bashlog/translation/Join.java: -------------------------------------------------------------------------------- 1 | package bashlog.translation; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | 6 | import bashlog.CompilerInternals; 7 | import bashlog.command.Bash; 8 | import bashlog.plan.SortAntiJoinNode; 9 | import bashlog.plan.SortJoinNode; 10 | import common.plan.node.PlanNode; 11 | 12 | /** Translate a join to a join command. It supports a projection after the sort. Also treats antijoin. */ 13 | public class Join implements BashTranslator { 14 | 15 | @Override 16 | public Bash translate(PlanNode planNode, CompilerInternals bc) { 17 | SortJoinNode j = (SortJoinNode) planNode; 18 | 19 | Bash.Command result = new Bash.Command("join"); 20 | if (j instanceof SortAntiJoinNode) { 21 | result.arg(" -v 1 "); 22 | } 23 | result.arg("-t $'\\t'"); 24 | 25 | if (j.getLeftProjection().length > 0) { 26 | int colLeft = j.getLeftProjection()[0] + 1; 27 | result.arg("-1 " + colLeft); 28 | } 29 | if (j.getRightProjection().length > 0) { 30 | int colRight = j.getRightProjection()[0] + 1; 31 | result.arg("-2 " + colRight); 32 | } 33 | 34 | StringBuilder outCols = new StringBuilder(); 35 | for (int i = 0; i < j.getOutputProjection().length; i++) { 36 | if (i > 0) { 37 | outCols.append(","); 38 | } 39 | int dst = j.getOutputProjection()[i]; 40 | if (dst < j.getLeft().getArity()) { 41 | outCols.append("1." + (dst + 1)); 42 | } else { 43 | outCols.append("2." + (dst - j.getLeft().getArity() + 1)); 44 | } 45 | } 46 | result.arg("-o " + outCols); 47 | 48 | result.file(bc.compile(j.getLeft())); 49 | result.file(bc.compile(j.getRight())); 50 | return result; 51 | } 52 | 53 | @Override 54 | public List> supports() { 55 | return Arrays.asList(SortJoinNode.class, SortAntiJoinNode.class); 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/bashlog/translation/Materialization.java: -------------------------------------------------------------------------------- 1 | package bashlog.translation; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | 6 | import bashlog.CompilerInternals; 7 | import bashlog.command.Bash; 8 | import bashlog.command.Bash.BashFile; 9 | import common.plan.node.MaterializationNode; 10 | import common.plan.node.PlanNode; 11 | 12 | /** 13 | * Translate a materialization node. The command is just executed and its output stored in a file. 14 | */ 15 | public class Materialization implements BashTranslator { 16 | 17 | @Override 18 | public Bash translate(PlanNode planNode, CompilerInternals bc) { 19 | MaterializationNode m = (MaterializationNode) planNode; 20 | String matFile = "tmp/mat" + bc.getNextIndex(); 21 | bc.registerPlaceholder(m.getReuseNode(), matFile); 22 | Bash.CommandSequence result = new Bash.CommandSequence(); 23 | result.info(planNode, ""); 24 | 25 | Bash reused = bc.compile(m.getReusedPlan()); 26 | if (reused instanceof BashFile) { 27 | BashFile rf = (BashFile) reused; 28 | bc.registerPlaceholder(m.getReuseNode(), rf.getPath()); 29 | } else { 30 | if (bc.parallelMaterialization()) { 31 | String lockFile = matFile.replaceAll("tmp/", "tmp/lock_"); 32 | reused = reused.wrap("rm -f " + lockFile + "; mkfifo " + lockFile + "; ( ", // 33 | " > " + matFile + // 34 | "; unlock " + lockFile + // 35 | " ) & "); 36 | } else { 37 | reused = reused.wrap("", " > " + matFile); 38 | } 39 | result.add(reused); 40 | } 41 | 42 | if (!(m.getMainPlan() instanceof MaterializationNode)) { 43 | result.other("\n# plan"); 44 | } 45 | result.add(bc.compile(m.getMainPlan())); 46 | return result; 47 | 48 | } 49 | 50 | @Override 51 | public List> supports() { 52 | return Arrays.asList(MaterializationNode.class); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/bashlog/translation/MultiFilter.java: -------------------------------------------------------------------------------- 1 | package bashlog.translation; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | 6 | import bashlog.CompilerInternals; 7 | import bashlog.command.Bash; 8 | import common.plan.node.MultiFilterNode; 9 | import common.plan.node.PlanNode; 10 | 11 | /** 12 | * Translate multi filter plan to one AWK command. 13 | * A multi filter are (a union of) filters, projections on the same plan, that build one single plan. 14 | */ 15 | public class MultiFilter implements BashTranslator { 16 | 17 | @Override 18 | public Bash translate(PlanNode planNode, CompilerInternals bc) { 19 | MultiFilterNode m = (MultiFilterNode) planNode; 20 | Bash.Command cmd = new Bash.Command(AwkHelper.AWK); 21 | 22 | StringBuilder arg = new StringBuilder(); 23 | List remaining = AwkHelper.complexAwkLine(m.getFilter(), 0, null, arg); 24 | 25 | // process remaining filter nodes 26 | for (PlanNode c : remaining) { 27 | AwkHelper.simpleAwkLine(c, null, arg); 28 | } 29 | arg.append("' "); 30 | cmd.arg(arg.toString()); 31 | cmd.file(bc.compile(m.getTable())); 32 | 33 | return cmd; 34 | } 35 | 36 | @Override 37 | public List> supports() { 38 | return Arrays.asList(MultiFilterNode.class); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/bashlog/translation/MultiOutput.java: -------------------------------------------------------------------------------- 1 | package bashlog.translation; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | 6 | import bashlog.CompilerInternals; 7 | import bashlog.command.Bash; 8 | import common.plan.node.*; 9 | 10 | /** 11 | * Translates a multi output plan to one single AWK command. 12 | * A multi output executes several projections and filters (of several branches) that operate on one input plan. 13 | */ 14 | public class MultiOutput implements BashTranslator { 15 | 16 | @Override 17 | public Bash translate(PlanNode planNode, CompilerInternals bc) { 18 | Bash.CommandSequence result = new Bash.CommandSequence(); 19 | Bash.Command touch = result.cmd("touch"); 20 | Bash.Command cmd = result.cmd(AwkHelper.AWK); 21 | MultiOutputNode mo = (MultiOutputNode) planNode; 22 | 23 | StringBuilder arg = new StringBuilder(""); 24 | List plans = mo.reusedPlans(), nodes = mo.reuseNodes(); 25 | for (int i = 0; i < plans.size(); i++) { 26 | PlanNode plan = plans.get(i), node = nodes.get(i); 27 | 28 | String matFile = "tmp/mat" + bc.getNextIndex(); 29 | touch.file(matFile); 30 | bc.registerPlaceholder((PlaceholderNode) node, matFile); 31 | 32 | //TODO: if there are more conditions on one output file: 33 | // if (!complexAwkLine(Arrays.asList(plan), matFile, arg).isEmpty()) { ... } 34 | AwkHelper.multioutAwkLine(plan, i, matFile, arg); 35 | } 36 | cmd.arg(arg.toString()).arg("'"); 37 | cmd.file(bc.compile(mo.getLeaf())); 38 | cmd.arg("\n"); 39 | result.add(bc.compile(mo.getMainPlan())); 40 | return result; 41 | 42 | } 43 | 44 | @Override 45 | public List> supports() { 46 | return Arrays.asList(MultiOutputNode.class); 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/bashlog/translation/ProjectFilter.java: -------------------------------------------------------------------------------- 1 | package bashlog.translation; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | 6 | import bashlog.CompilerInternals; 7 | import bashlog.command.Bash; 8 | import common.plan.node.*; 9 | 10 | /** Translates projects and filters to an AWK command */ 11 | public class ProjectFilter implements BashTranslator { 12 | 13 | @Override 14 | public Bash translate(PlanNode planNode, CompilerInternals bc) { 15 | StringBuilder awk = new StringBuilder(); 16 | PlanNode inner = AwkHelper.simpleAwkLine(planNode, null, awk); 17 | 18 | StringBuilder advAwk = new StringBuilder(); 19 | if (AwkHelper.complexAwkLine(Arrays.asList(planNode), 0, null, advAwk).isEmpty()) { 20 | awk = advAwk; 21 | } 22 | 23 | return new Bash.Command(AwkHelper.AWK).arg(awk.toString()).arg("'").file(bc.compile(inner)); 24 | } 25 | 26 | @Override 27 | public List> supports() { 28 | return Arrays.asList(ProjectNode.class, ConstantEqualityFilterNode.class, VariableEqualityFilterNode.class); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/bashlog/translation/Recursion.java: -------------------------------------------------------------------------------- 1 | package bashlog.translation; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | 6 | import bashlog.CompilerInternals; 7 | import bashlog.command.Bash; 8 | import common.plan.node.PlanNode; 9 | import common.plan.node.RecursionNode; 10 | 11 | /** Translates a recursion node to a bash while loop */ 12 | public class Recursion implements BashTranslator { 13 | 14 | private Bash setMinusSorted(Bash prev, String filename) { 15 | Bash.Pipe result = prev.pipe(); 16 | result.cmd("comm")// 17 | .arg("-23").arg("-")// 18 | .file(filename); 19 | return result; 20 | } 21 | 22 | private Bash recursionSorted(CompilerInternals bc, RecursionNode rn, String fullFile, String deltaFile, String newDeltaFile) { 23 | Bash prev = bc.compile(rn.getRecursivePlan()); 24 | //setMinusInMemory(fullFile, sb); 25 | Bash delta = setMinusSorted(prev, fullFile); 26 | delta = delta.wrap("", " > " + newDeltaFile + ";"); 27 | 28 | Bash.CommandSequence result = new Bash.CommandSequence(); 29 | result.add(delta); 30 | result.info(rn, "continued"); 31 | result.cmd("mv").file(newDeltaFile).file(deltaFile).arg("; "); 32 | result.cmd("$sort")// 33 | .arg("-u").arg("--merge").arg("-o")// 34 | .file(fullFile).file(fullFile).file(deltaFile).arg("; "); 35 | 36 | return result; 37 | } 38 | 39 | @Override 40 | public Bash translate(PlanNode planNode, CompilerInternals bc) { 41 | RecursionNode rn = (RecursionNode) planNode; 42 | int idx = bc.getNextIndex(); 43 | String deltaFile = "tmp/delta" + idx; 44 | String newDeltaFile = "tmp/new" + idx; 45 | String fullFile = "tmp/full" + idx; 46 | bc.registerPlaceholder(rn.getDelta(), deltaFile); 47 | bc.registerPlaceholder(rn.getFull(), fullFile); 48 | 49 | Bash.CommandSequence result = new Bash.CommandSequence(); 50 | Bash b = bc.compile(rn.getExitPlan()); 51 | Bash.Pipe pipe = b.pipe(); 52 | Bash.Command cmd = pipe.cmd("tee"); 53 | cmd.file(fullFile); 54 | result.add(pipe.wrap("", " > " + deltaFile)); 55 | 56 | // "do while" loop in bash 57 | result.cmd("while \n"); 58 | 59 | result.add(recursionSorted(bc, rn, fullFile, deltaFile, newDeltaFile)); 60 | result.cmd("[ -s " + deltaFile + " ]; "); 61 | result.cmd("do continue; done\n"); 62 | result.cmd("rm").file(deltaFile).wrap("", "\n"); 63 | result.cmd("cat").file(fullFile); 64 | return result; 65 | 66 | } 67 | 68 | @Override 69 | public List> supports() { 70 | return Arrays.asList(RecursionNode.class); 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/bashlog/translation/Sort.java: -------------------------------------------------------------------------------- 1 | package bashlog.translation; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | 6 | import bashlog.CompilerInternals; 7 | import bashlog.command.Bash; 8 | import bashlog.plan.SortNode; 9 | import common.plan.node.PlanNode; 10 | 11 | /** Translates a sort node to a sort command */ 12 | public class Sort implements BashTranslator { 13 | 14 | @Override 15 | public Bash translate(PlanNode p, CompilerInternals bc) { 16 | SortNode s = (SortNode)p; 17 | int[] cols = s.sortColumns(); 18 | Bash prev = bc.compile(s.getTable()); 19 | Bash.Pipe result = new Bash.Pipe(prev); 20 | Bash.Command cmd = result.cmd("$sort").arg("-t $'\\t'"); 21 | 22 | boolean supportsUniq = cols == null; 23 | if (cols != null) { 24 | int used[] = new int[s.getTable().getArity()]; 25 | Arrays.fill(used, 0); 26 | for (int col : cols) { 27 | cmd.arg("-k " + (col + 1)); 28 | used[col] = 1; 29 | } 30 | if (Arrays.stream(used).allMatch(k -> k == 1)) { 31 | supportsUniq = true; 32 | } 33 | } 34 | if (supportsUniq) { 35 | cmd.arg("-u"); 36 | } 37 | cmd.file(prev); 38 | 39 | return cmd; 40 | } 41 | 42 | @Override 43 | public List> supports() { 44 | return Arrays.asList(SortNode.class); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/bashlog/translation/Union.java: -------------------------------------------------------------------------------- 1 | package bashlog.translation; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | 6 | import bashlog.CompilerInternals; 7 | import bashlog.command.Bash; 8 | import common.plan.node.PlanNode; 9 | import common.plan.node.UnionNode; 10 | 11 | /** Translates a union node to a sort command, which removes duplicates */ 12 | public class Union implements BashTranslator { 13 | 14 | @Override 15 | public Bash translate(PlanNode planNode, CompilerInternals bc) { 16 | if (planNode.children().size() == 0) { 17 | return new Bash.Command("echo").arg("-n"); 18 | } else { 19 | Bash.Command result = new Bash.Command("$sort").arg("-u"); 20 | for (PlanNode child : ((UnionNode) planNode).getChildren()) { 21 | result.file(bc.compile(child)); 22 | } 23 | return result; 24 | } 25 | } 26 | 27 | @Override 28 | public List> supports() { 29 | return Arrays.asList(UnionNode.class); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/common/CallStack.java: -------------------------------------------------------------------------------- 1 | package common; 2 | 3 | import java.util.Stack; 4 | 5 | /** 6 | Copyright 2016 Fabian M. Suchanek 7 | 8 | Licensed under the Apache License, Version 2.0 (the "License"); 9 | you may not use this file except in compliance with the License. 10 | You may obtain a copy of the License at 11 | 12 | http://www.apache.org/licenses/LICENSE-2.0 13 | 14 | Unless required by applicable law or agreed to in writing, software 15 | distributed under the License is distributed on an "AS IS" BASIS, 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | See the License for the specific language governing permissions and 18 | limitations under the License. 19 | 20 | 21 | This class represents the current position of a program, i.e. the stack of methods that 22 | have been called together with the line numbers.
23 | Example:
24 | 
 25 |    public class Blah {
 26 |      public m2() { 
 27 |        System.out.println(new CallStack());   // (1)
 28 |        System.out.println(CallStack.here());  // (2)
 29 |      }
 30 |   
 31 |      public m1() { 
 32 |        m2(); 
 33 |      }
 34 |   
 35 |      public static void main(String[] args) { 
 36 |        m1(); 
 37 |      }
 38 |    }
 39 |    -->
 40 |       Blah.main(12)->Blah.m1(8)->Blah.m2(2)  // Call Stack at (1)
 41 |       Blah.m2(3)                             // Method and line number at (2)
 42 |
43 | */ 44 | public class CallStack { 45 | 46 | /** Holds the call stack */ 47 | private Stack callstack = new Stack<>(); 48 | 49 | /** Constructs a call stack from the current program position (without the constructor call)*/ 50 | public CallStack() { 51 | try { 52 | throw new Exception(); 53 | } catch (Exception e) { 54 | StackTraceElement[] s = e.getStackTrace(); 55 | for (int i = s.length - 1; i != 0; i--) 56 | callstack.push(s[i]); 57 | } 58 | } 59 | 60 | /** Returns TRUE if the two call stacks have the same elements*/ 61 | @Override 62 | public boolean equals(Object o) { 63 | return (o instanceof CallStack && ((CallStack) o).callstack.equals(callstack)); 64 | } 65 | 66 | /** Returns a nice String for a Stacktraceelement*/ 67 | public static String toString(StackTraceElement e) { 68 | String cln = e.getClassName(); 69 | if (cln.lastIndexOf('.') != -1) cln = cln.substring(cln.lastIndexOf('.') + 1); 70 | return (cln + "." + e.getMethodName() + '(' + e.getLineNumber() + ')'); 71 | } 72 | 73 | /** Returns "method(line)->method(line)->..." */ 74 | @Override 75 | public String toString() { 76 | StringBuilder s = new StringBuilder(); 77 | for (int i = 0; i < callstack.size() - 1; i++) { 78 | s.append(toString(callstack.get(i))).append("->"); 79 | } 80 | s.append(toString(callstack.get(callstack.size() - 1))); 81 | return (s.toString()); 82 | } 83 | 84 | /** Gives the calling position as a StackTraceElement */ 85 | public StackTraceElement top() { 86 | return (callstack.peek()); 87 | } 88 | 89 | /** Gives the calling position */ 90 | public static StackTraceElement here() { 91 | CallStack p = new CallStack(); 92 | p.callstack.pop(); 93 | return (p.callstack.peek()); 94 | } 95 | 96 | /** Returns the callstack */ 97 | public Stack getCallstack() { 98 | return callstack; 99 | } 100 | 101 | /** Pops the top level of this callstack, returns this callstack */ 102 | public CallStack ret() { 103 | callstack.pop(); 104 | return (this); 105 | } 106 | 107 | /** Test routine */ 108 | public static void main(String[] args) { 109 | System.out.println(new CallStack()); 110 | System.out.println(here()); 111 | } 112 | 113 | } 114 | 115 | -------------------------------------------------------------------------------- /src/main/java/common/DatalogTools.java: -------------------------------------------------------------------------------- 1 | package common; 2 | 3 | import java.util.Collection; 4 | 5 | import common.parser.*; 6 | 7 | public class DatalogTools { 8 | 9 | /** Add input rules for classes and binary relations for predicates occurring in 'p'. 10 | * @param modeURI wrap in < ... > if true 11 | * */ 12 | public static Program inputRules3(String factPred, Program p, String rdfType, boolean modeURI) { 13 | return inputRules3(factPred, p.allRelations(), rdfType, modeURI); 14 | } 15 | 16 | /** Add input rules for classes and binary relations for predicates specified by 'relations' 17 | * @param modeURI wrap in < ... > if true 18 | * */ 19 | public static Program inputRules3(String factPred, Collection relations, String rdfType, boolean modeURI) { 20 | Program result = new Program(); 21 | 22 | Variable X = new Variable("X"); 23 | Variable Y = new Variable("Y"); 24 | 25 | Term rdfTypeConst = Constant.of(rdfType); 26 | for (String rel : relations) { 27 | int pos = rel.lastIndexOf("/"); 28 | if(pos < 0) continue; 29 | String arity = rel.substring(pos+1); 30 | String relation = rel.substring(0, pos); 31 | Term relConst = modeURI ? Constant.of(ensureURI(relation)) : Constant.of(relation); 32 | if ("1".equals(arity)) { 33 | // abc(X) :- facts(X, "rdf:type", "abc")."; 34 | Rule r = new Rule( 35 | new CompoundTerm(relation, X), // head 36 | new CompoundTerm(factPred, X, rdfTypeConst, relConst)); 37 | result.addRule(r); 38 | 39 | } else if ("2".equals(arity)) { 40 | // abc(X,Y) :- facts(X, "abc", Y). 41 | Rule r = new Rule( 42 | new CompoundTerm(relation, X, Y), // head 43 | new CompoundTerm(factPred, X, relConst, Y)); 44 | result.addRule(r); 45 | } 46 | } 47 | return result; 48 | } 49 | 50 | private static String ensureURI(String str) { 51 | return (str.startsWith("<") ? "" : "<") + str + (str.endsWith(">") ? "" : ">"); 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/common/Evaluator.java: -------------------------------------------------------------------------------- 1 | package common; 2 | 3 | import java.util.*; 4 | 5 | import common.parser.Program; 6 | 7 | public interface Evaluator { 8 | 9 | FactsSet evaluate(Program program, FactsSet facts, Set queryRelations) throws Exception; 10 | 11 | default void debug(Program program, FactsSet facts, Set queryRelations) throws Exception { 12 | System.err.println("warning: " + this.getClass() + " does not provide debug information"); 13 | } 14 | 15 | default Map getTiming() { 16 | return new HashMap<>(); 17 | } 18 | } -------------------------------------------------------------------------------- /src/main/java/common/FactsSet.java: -------------------------------------------------------------------------------- 1 | package common; 2 | 3 | import java.util.Set; 4 | import java.util.stream.Stream; 5 | 6 | public interface FactsSet { 7 | Set getRelations(); 8 | 9 | Stream[]> getByRelation(String relation); 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/common/SimpleFactsSet.java: -------------------------------------------------------------------------------- 1 | package common; 2 | 3 | 4 | import java.io.IOException; 5 | import java.nio.file.Path; 6 | import java.util.*; 7 | import java.util.stream.Collectors; 8 | import java.util.stream.Stream; 9 | 10 | import common.parser.*; 11 | 12 | public class SimpleFactsSet implements FactsSet { 13 | 14 | private Map[]>> facts = new HashMap<>(); 15 | 16 | public void add(String relation, Comparable... args) { 17 | facts.computeIfAbsent(relation, k -> new HashSet<>()).add(args); 18 | } 19 | 20 | public void add(CompoundTerm term) { 21 | Comparable[] args = new Comparable[term.args.length]; 22 | for (int i = 0; i < term.args.length; i++) { 23 | args[i] = ((Constant) term.args[i]).getValue(); 24 | } 25 | add(term.getRelation(), args); 26 | } 27 | 28 | public void loadFile(Path path) throws IOException { 29 | ParserReader pr = new ParserReader(Tools.getFileContent(path.toFile())); 30 | while (true) { 31 | pr.skipComments(); 32 | if (pr.peek() == null) return; 33 | CompoundTerm value = CompoundTerm.read(pr, Collections.emptyMap(), Parseable.ALL_FEATURES); 34 | if (value != null && pr.consume(".") != null) { 35 | add(value); 36 | } 37 | } 38 | } 39 | 40 | @Override 41 | public Set getRelations() { 42 | return facts.keySet(); 43 | } 44 | 45 | @Override 46 | public Stream[]> getByRelation(String relation) { 47 | //TODO: better datastructure 48 | return facts.getOrDefault(relation, Collections.emptySet()).stream(); 49 | } 50 | 51 | @Override 52 | public String toString() { 53 | return facts.entrySet().stream().flatMap(entry -> 54 | entry.getValue().stream().map(value -> 55 | entry.getKey() + "(" + Arrays.stream(value).map(Object::toString).collect(Collectors.joining(", ")) + ")" 56 | ) 57 | ).collect(Collectors.joining("\n")); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/common/TSVReader.java: -------------------------------------------------------------------------------- 1 | package common; 2 | 3 | import java.io.*; 4 | import java.util.*; 5 | 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | public class TSVReader implements Iterable>, Iterator>, Closeable { 10 | 11 | private static final Logger LOG = LoggerFactory.getLogger(TSVReader.class); 12 | 13 | File path; 14 | 15 | BufferedReader reader; 16 | 17 | List next = null; 18 | 19 | public TSVReader(File path) throws IOException { 20 | reader = new BufferedReader(new FileReader(path)); 21 | this.path = path; 22 | } 23 | 24 | @Override 25 | public void close() throws IOException { 26 | if (reader != null) { 27 | reader.close(); 28 | reader = null; 29 | } 30 | } 31 | 32 | @Override 33 | public Iterator> iterator() { 34 | return this; 35 | } 36 | 37 | private List peek() { 38 | if (reader == null) return null; 39 | if (next == null) { 40 | String line; 41 | try { 42 | line = reader.readLine(); 43 | if(line == null) { 44 | return null; 45 | } 46 | 47 | next = Arrays.asList(line.split("\t")); 48 | } catch (IOException e) { 49 | LOG.error("An error occured while processing {}", path.getAbsolutePath()); 50 | e.printStackTrace(); 51 | reader = null; 52 | } 53 | } 54 | return next; 55 | } 56 | 57 | @Override 58 | public boolean hasNext() { 59 | return peek() != null; 60 | } 61 | 62 | @Override 63 | public List next() { 64 | try { 65 | return peek(); 66 | } finally { 67 | next = null; 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/common/TSVWriter.java: -------------------------------------------------------------------------------- 1 | package common; 2 | 3 | import java.io.*; 4 | import java.util.Collection; 5 | 6 | public class TSVWriter { 7 | 8 | private BufferedWriter br; 9 | 10 | public TSVWriter(String path) throws IOException { 11 | br = new BufferedWriter(new FileWriter(path)); 12 | } 13 | 14 | public void close() throws IOException { 15 | br.close(); 16 | } 17 | 18 | public void write(Collection values) throws IOException { 19 | if (values == null || values.isEmpty()) 20 | throw new IllegalArgumentException(); 21 | StringBuilder sb = new StringBuilder(); 22 | int i = 1; 23 | for (String value : values) { 24 | if (i == values.size()) 25 | sb.append(value); 26 | else 27 | sb.append(value).append("\t"); 28 | i++; 29 | } 30 | sb.append("\n"); 31 | br.write(sb.toString()); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/common/VerifyNodeImplementation.java: -------------------------------------------------------------------------------- 1 | package common; 2 | 3 | import java.io.File; 4 | import java.net.URL; 5 | import java.nio.file.Files; 6 | import java.util.Enumeration; 7 | 8 | import common.plan.node.PlanNode; 9 | 10 | public class VerifyNodeImplementation { 11 | 12 | 13 | public static void main(String[] args) throws Exception { 14 | Enumeration roots = VerifyNodeImplementation.class.getClassLoader().getResources(""); 15 | while (roots.hasMoreElements()) { 16 | URL rootURL = roots.nextElement(); 17 | File root = new File(rootURL.getPath()); 18 | Files.walk(root.toPath()).forEach(path -> { 19 | File file = path.toFile(); 20 | if (file.getName().endsWith(".class") && !file.getName().contains("$")) { 21 | String fullClassName = file.getAbsolutePath().replace(root.getAbsolutePath() + "/", "").replace("/", "."); 22 | fullClassName = fullClassName.substring(0, fullClassName.lastIndexOf(".")); 23 | 24 | try { 25 | Class cls = Class.forName(fullClassName); 26 | if (!PlanNode.class.isAssignableFrom(cls)) return; 27 | 28 | checkMethod(fullClassName, "equals", Object.class); 29 | checkMethod(fullClassName, "hashCode"); 30 | 31 | } catch (ClassNotFoundException e) { 32 | System.err.println("Cannot check methods of " + fullClassName); 33 | } 34 | 35 | } 36 | }); 37 | } 38 | System.out.println("verification completed"); 39 | } 40 | 41 | private static void checkMethod(String fullClassName, String method, Class... params) throws ClassNotFoundException { 42 | Class cls = Class.forName(fullClassName); 43 | if (cls.isInterface()) return; 44 | 45 | try { 46 | cls.getDeclaredMethod(method, params); 47 | } catch (NoSuchMethodException e) { 48 | System.err.println("Please check whether " + fullClassName + " needs method '" + method + "'"); 49 | 50 | } catch (SecurityException e) { 51 | System.err.println("Cannot check methods of " + fullClassName); 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/common/compiler/AutoIndent.java: -------------------------------------------------------------------------------- 1 | package common.compiler; 2 | 3 | /** 4 | * StringBuilder replacement which allows indenting of code blocks. 5 | */ 6 | public class AutoIndent { 7 | 8 | private static final String TAB = " "; 9 | 10 | /** The string built so far */ 11 | private StringBuilder sb; 12 | 13 | /** The current indent */ 14 | private String indent; 15 | 16 | /** Default constructor */ 17 | public AutoIndent() { 18 | indent = ""; 19 | this.sb = new StringBuilder(); 20 | } 21 | 22 | /** Increase the indent */ 23 | public AutoIndent indent() { 24 | AutoIndent r = new AutoIndent(); 25 | r.sb = sb; 26 | r.indent = indent + TAB; 27 | return r; 28 | } 29 | 30 | /** Append string, respecting the current indentation level */ 31 | public AutoIndent append(String string) { 32 | sb.append(string.replaceAll("\n", "\n" + indent)); 33 | return this; 34 | } 35 | 36 | /** Append string representation of an Object. Convenience method */ 37 | public AutoIndent append(Object o) { 38 | if (o instanceof AutoIndent) { 39 | append(o.toString()); 40 | } 41 | else { 42 | append(o.toString()); 43 | } 44 | return this; 45 | } 46 | 47 | /** The string that we have generated so far */ 48 | public String generate() { 49 | return sb.toString(); 50 | } 51 | 52 | @Override 53 | public String toString() { 54 | return generate(); 55 | } 56 | 57 | /** Append string as it is */ 58 | public void appendRaw(String string) { 59 | sb.append(string); 60 | } 61 | } -------------------------------------------------------------------------------- /src/main/java/common/compiler/CompilerInternals.java: -------------------------------------------------------------------------------- 1 | package common.compiler; 2 | 3 | import java.util.Map; 4 | 5 | import common.plan.node.PlaceholderNode; 6 | import common.plan.node.PlanNode; 7 | 8 | public class CompilerInternals<_Translator> { 9 | 10 | protected final Map, _Translator> translators; 11 | 12 | protected Map placeholderToParent; 13 | 14 | public CompilerInternals(Map, _Translator> translators, PlanNode fullPlan) { 15 | this.translators = translators; 16 | placeholderToParent = PlaceholderNode.placeholderToParentMap(fullPlan); 17 | } 18 | 19 | public PlanNode getParent(PlaceholderNode pn) { 20 | return (PlanNode) placeholderToParent.get(pn); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/common/compiler/Translator.java: -------------------------------------------------------------------------------- 1 | package common.compiler; 2 | 3 | import java.util.List; 4 | 5 | import common.plan.node.PlanNode; 6 | 7 | public interface Translator { 8 | 9 | /** 10 | * Translate planNode to bash snippet 11 | * @param planNode 12 | * @param bc helps to translate children plans, and to obtain temporary file numbers 13 | * @return 14 | */ 15 | public Output translate(PlanNode planNode, Internals bc); 16 | 17 | /** Which node types are supported by this translator */ 18 | public List> supports(); 19 | } -------------------------------------------------------------------------------- /src/main/java/common/parser/Atom.java: -------------------------------------------------------------------------------- 1 | package common.parser; 2 | 3 | public class Atom extends Term implements Parseable { 4 | 5 | String name; 6 | 7 | @Override 8 | public String toString() { 9 | return "atom:" + name; 10 | } 11 | 12 | @Override 13 | public int compareTo(Object o) { 14 | if (o instanceof Atom) { 15 | return name.compareTo(((Atom) o).name); 16 | } else { 17 | return toString().compareTo(o.toString()); //TODO: inefficient 18 | } 19 | } 20 | 21 | @Override 22 | public boolean equals(Object obj) { 23 | return obj instanceof Atom && name.equals(((Atom) obj).name); 24 | } 25 | 26 | @Override 27 | public int hashCode() { 28 | return name.hashCode(); 29 | } 30 | 31 | public String getName() { 32 | return name; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/common/parser/BashRule.java: -------------------------------------------------------------------------------- 1 | package common.parser; 2 | 3 | import java.util.*; 4 | 5 | /** 6 | * A rule of the form rel(X,Y) :~ cat abc.txt !other/2 7 | * Where other/2 is another predicate. 8 | * Relations need to be prefixed by " !" 9 | */ 10 | public class BashRule extends Rule { 11 | 12 | /** 13 | * Constructor 14 | * @param command bash command 15 | * @param relations used in bash command, in the order they appear in the command 16 | * @param commandParts list of strings that surround the "!abc" 17 | */ 18 | public BashRule(CompoundTerm head, String command, List relations, List commandParts) { 19 | super(head, new ArrayList<>()); 20 | this.relations = relations; 21 | this.command = command; 22 | this.commandParts = commandParts; 23 | } 24 | 25 | public String command; 26 | 27 | public List commandParts; 28 | 29 | public List relations; 30 | 31 | public static BashRule read(ParserReader pr, Set supportedFeatures, CompoundTerm head) { 32 | 33 | pr.skipComments(); 34 | String command = pr.readLine().trim(); 35 | 36 | List relations = new ArrayList<>(), commandParts = new ArrayList<>(); 37 | int start = -1, prev = 0; 38 | while ((start = command.indexOf(" !", start + 1)) > 0) { 39 | commandParts.add(command.substring(prev, start + 1)); 40 | 41 | ParserReader prName = new ParserReader(command, start + 2); 42 | CharSequence name = prName.readName(); 43 | 44 | if(name == null) { 45 | prName.error("name expected", ""); 46 | } 47 | 48 | if (prName.consume("/") != null) { 49 | name = name + "/" + prName.readInteger(); 50 | } 51 | 52 | relations.add(name.toString()); 53 | 54 | start = prName.pos(); 55 | prev = start; 56 | } 57 | commandParts.add(command.substring(prev)); 58 | 59 | return new BashRule(head, command, relations, commandParts); 60 | } 61 | 62 | @Override 63 | public String toString() { 64 | return String.valueOf(head) + " :~ " + command; 65 | } 66 | 67 | public static void main(String[] args) { 68 | ParserReader pr = new ParserReader("cat !abc xyz !def 123"); 69 | BashRule br = read(pr, new HashSet<>(), new CompoundTerm("abc")); 70 | System.out.println(br.relations); 71 | System.out.println(br.commandParts); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/common/parser/CompoundTerm.java: -------------------------------------------------------------------------------- 1 | package common.parser; 2 | 3 | import java.util.*; 4 | import java.util.stream.Stream; 5 | 6 | public class CompoundTerm extends Term implements Parseable { 7 | 8 | public final String name; 9 | 10 | public Term[] args; 11 | 12 | public final boolean negated; 13 | 14 | public CompoundTerm(String name) { 15 | this(name, false, new Term[]{}); 16 | } 17 | 18 | public CompoundTerm(String name, boolean negated) { 19 | this(name, negated, new Term[]{}); 20 | } 21 | 22 | public CompoundTerm(String name, Term... args) { 23 | this(name, false, args); 24 | } 25 | 26 | public CompoundTerm(String name, boolean negated, Term... args) { 27 | this.name = name; 28 | this.negated = negated; 29 | this.args = args; 30 | } 31 | 32 | 33 | public static CompoundTerm read(ParserReader pr, Map varMap, Set supportedFeatures) { 34 | pr.debug(); 35 | pr.skipComments(); 36 | boolean negated = (pr.consume("not") != null); 37 | pr.skipComments(); 38 | CharSequence name = pr.readName(); 39 | if (name == null) return null; 40 | return read(name.toString(), negated, pr, varMap, supportedFeatures); 41 | } 42 | 43 | public static CompoundTerm read(String name, boolean negated, ParserReader pr, Map varMap, Set supportedFeatures) { 44 | CompoundTerm a = new CompoundTerm(name, negated); 45 | pr.debug(); 46 | Term v; 47 | pr.skipComments(); 48 | List args = new ArrayList<>(); 49 | if (pr.expect("(") != null) { 50 | while ((v = Term.read(pr, varMap, supportedFeatures)) != null) { 51 | args.add(v); 52 | pr.skipComments(); 53 | if (pr.peek() == ')' || pr.expect(",") == null) { 54 | break; 55 | } 56 | } 57 | pr.expect(")"); 58 | } else { 59 | return null; 60 | } 61 | a.args = args.toArray(new Term[] {}); 62 | return a; 63 | } 64 | 65 | @Override 66 | public boolean equals(Object obj) { 67 | return obj instanceof CompoundTerm && name.equals(((CompoundTerm) obj).name) && Arrays.equals(args, ((CompoundTerm) obj).args); 68 | } 69 | 70 | @Override 71 | public String toString() { 72 | return ( negated ? "not " : "") + (name + Arrays.toString(args).replace('[', '(').replace(']', ')')); 73 | } 74 | 75 | @Override 76 | public int hashCode() { 77 | return name.hashCode() ^ Arrays.hashCode(args); 78 | } 79 | 80 | public String getRelation() { 81 | return name + "/" + args.length; 82 | } 83 | 84 | @Override 85 | public Stream getVariables() { 86 | return Arrays.stream(args).flatMap(Term::getVariables); 87 | } 88 | 89 | public static void main(String[] args) { 90 | ParserReader pr; 91 | CompoundTerm a = CompoundTerm.read(pr = new ParserReader("abc(\"a\")"), new HashMap<>(), Parseable.ALL_FEATURES); 92 | System.out.println(a); 93 | System.out.println("remaining: " + pr.peekLine()); 94 | a = CompoundTerm.read(pr = new ParserReader("abc(a)"), new HashMap<>(), Parseable.ALL_FEATURES); 95 | System.out.println(a); 96 | System.out.println("remaining: " + pr.peekLine()); 97 | a = CompoundTerm.read(pr = new ParserReader("abc(A)"), new HashMap<>(), Parseable.ALL_FEATURES); 98 | System.out.println(a); 99 | System.out.println("remaining: " + pr.peekLine()); 100 | //System.out.println(YCompoundTerm.read("abc(\"a\", b)", new int[] { 0 }, new HashMap<>(), null)); 101 | } 102 | 103 | @Override 104 | public int compareTo(Object o) { 105 | throw new UnsupportedOperationException(); 106 | } 107 | 108 | public static int parseRelationArity(String relation) { 109 | return Integer.parseInt(relation.substring(relation.lastIndexOf('/') + 1)); 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /src/main/java/common/parser/Constant.java: -------------------------------------------------------------------------------- 1 | package common.parser; 2 | 3 | public class Constant extends Term implements Parseable { 4 | 5 | private T value; 6 | 7 | public Constant(T value) { 8 | this.value = value; 9 | } 10 | 11 | @Override 12 | public String toString() { 13 | return "\"" + value + "\""; 14 | } 15 | 16 | @Override 17 | public boolean equals(Object obj) { 18 | return obj instanceof Constant && value.equals(((Constant) obj).value); 19 | } 20 | 21 | @Override 22 | public int hashCode() { 23 | return value.hashCode(); 24 | } 25 | 26 | @Override 27 | public int compareTo(Object o) { 28 | if (o instanceof Constant) { 29 | return value.compareTo(((Constant) o).value); 30 | } else { 31 | return toString().compareTo(o.toString()); //TODO: inefficient 32 | } 33 | } 34 | 35 | public T getValue() { 36 | return value; 37 | } 38 | 39 | public static Term of(String str) { 40 | return new Constant<>(str); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/common/parser/ParseException.java: -------------------------------------------------------------------------------- 1 | package common.parser; 2 | 3 | 4 | public class ParseException extends RuntimeException { 5 | 6 | private static final long serialVersionUID = 4072097330506643228L; 7 | 8 | public ParseException(String message) { 9 | super(message); 10 | } 11 | 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/common/parser/Parseable.java: -------------------------------------------------------------------------------- 1 | package common.parser; 2 | 3 | import java.util.*; 4 | 5 | public interface Parseable { 6 | 7 | public static final String ATOMS = "atoms"; 8 | 9 | public static final Set ALL_FEATURES = new HashSet<>(Arrays.asList(ATOMS)); 10 | 11 | 12 | 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/common/parser/Term.java: -------------------------------------------------------------------------------- 1 | package common.parser; 2 | 3 | import java.util.Map; 4 | import java.util.Set; 5 | import java.util.stream.Stream; 6 | 7 | public abstract class Term implements Parseable, Comparable { 8 | 9 | public static Term read(ParserReader pr, Map varMap, Set supportedFeatures) { 10 | pr.debug(); 11 | pr.skipComments(); 12 | if (pr.peek() == null) return null; 13 | 14 | if (pr.peek() == '_' || Character.isUpperCase(pr.peek())) { 15 | return Variable.read(pr, varMap, supportedFeatures); 16 | } 17 | 18 | if (pr.peek() == '[') { 19 | return TermList.read(pr, varMap, supportedFeatures); 20 | } 21 | 22 | if (pr.peek() == '"' || pr.peek() == '\'') { 23 | return new Constant<>(pr.readString()); 24 | } 25 | 26 | if (Character.isDigit(pr.peek())) { 27 | return new Constant>((Comparable) pr.readNumber()); 28 | } 29 | 30 | if (Character.isLowerCase(pr.peek())) { 31 | if(!supportedFeatures.contains(Parseable.ATOMS)) { 32 | pr.error("variables need to start with an uppercase character", null); 33 | } 34 | Atom a = new Atom(); 35 | a.name = pr.readName().toString(); 36 | 37 | pr.skipComments(); 38 | if (pr.peek() == '(') { 39 | return CompoundTerm.read(a.name, false, pr, varMap, supportedFeatures); 40 | } 41 | 42 | return a; 43 | } 44 | 45 | return null; 46 | } 47 | 48 | public Stream getVariables() { 49 | return Stream.empty(); 50 | } 51 | 52 | @Override 53 | public abstract int hashCode(); 54 | 55 | @Override 56 | public abstract boolean equals(Object obj); 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/common/parser/TermList.java: -------------------------------------------------------------------------------- 1 | package common.parser; 2 | 3 | import java.util.*; 4 | import java.util.stream.Stream; 5 | 6 | public class TermList extends Term { 7 | 8 | private List terms = new ArrayList<>(); 9 | 10 | public TermList() { 11 | 12 | } 13 | 14 | public TermList(Term... terms) { 15 | this.terms = Arrays.asList(terms); 16 | } 17 | 18 | public static TermList read(ParserReader pr, Map varMap, Set supportedFeatures) { 19 | if (pr.consume("[") != null) { 20 | TermList tl = new TermList(); 21 | while (pr.peek() != ']') { 22 | Term t = Term.read(pr, varMap, supportedFeatures); 23 | if (t == null) return null; 24 | tl.terms.add(t); 25 | // TODO: better parsing of lists 26 | pr.consume(","); 27 | } 28 | pr.read(); 29 | return tl; 30 | } 31 | return null; 32 | } 33 | 34 | @Override 35 | public String toString() { 36 | return terms.toString(); 37 | } 38 | 39 | public static void main(String[] args) { 40 | Term tl = read(new ParserReader("[a,b]"), new HashMap<>(), Parseable.ALL_FEATURES); 41 | System.out.println(tl); 42 | } 43 | 44 | @Override 45 | public Stream getVariables() { 46 | return terms.stream().flatMap(Term::getVariables); 47 | } 48 | 49 | @Override 50 | public int compareTo(Object o) { 51 | return toString().compareTo(o.toString()); //TODO: inefficient 52 | } 53 | 54 | @Override 55 | public int hashCode() { 56 | return terms.hashCode(); 57 | } 58 | 59 | @Override 60 | public boolean equals(Object obj) { 61 | return obj instanceof TermList && Objects.equals(terms, ((TermList) obj).terms); 62 | } 63 | 64 | public List terms() { 65 | return terms; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/common/parser/Variable.java: -------------------------------------------------------------------------------- 1 | package common.parser; 2 | 3 | import java.util.Map; 4 | import java.util.Set; 5 | import java.util.stream.Stream; 6 | 7 | public class Variable extends Term implements Parseable { 8 | 9 | public final String name; 10 | 11 | public Variable(String name) { 12 | this.name = name; 13 | } 14 | 15 | @Override 16 | public String toString() { 17 | return name; 18 | } 19 | 20 | public static Variable read(ParserReader pr, Map varMap, Set supportedFeatures) { 21 | pr.debug(); 22 | pr.skipComments(); 23 | if (pr.peek() == null) return null; 24 | String name = pr.readName().toString(); 25 | if ("_".equals(name)) { 26 | name = "Var_" + varMap.size(); 27 | } 28 | if (name != null) { 29 | return varMap.computeIfAbsent(name, Variable::new); 30 | } 31 | return null; 32 | } 33 | 34 | @Override 35 | public Stream getVariables() { 36 | return Stream.of(this); 37 | } 38 | 39 | @Override 40 | public boolean equals(Object obj) { 41 | return obj instanceof Variable && name.equals(((Variable) obj).name); 42 | } 43 | 44 | @Override 45 | public int hashCode() { 46 | return name.hashCode(); 47 | } 48 | 49 | @Override 50 | public int compareTo(Object o) { 51 | if (o instanceof Variable) { 52 | return name.compareTo(((Variable) o).name); 53 | } else { 54 | return toString().compareTo(o.toString()); //TODO: inefficient 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/common/plan/node/AntiJoinNode.java: -------------------------------------------------------------------------------- 1 | package common.plan.node; 2 | 3 | import java.util.*; 4 | import java.util.stream.Collectors; 5 | import java.util.stream.IntStream; 6 | 7 | import common.Tools; 8 | 9 | /** 10 | * Returns tuple in left such that there exists no tuple in right with the left projection of left is the same as the right projection of right 11 | */ 12 | public class AntiJoinNode implements PlanNode { 13 | 14 | private final PlanNode left; 15 | 16 | private final PlanNode right; 17 | 18 | private final int[] leftProjection; 19 | 20 | AntiJoinNode(PlanNode left, PlanNode right, int[] leftProjection) { 21 | if (leftProjection.length != right.getArity()) { 22 | throw new IllegalArgumentException("The left projection and the right plan for anti join should have the same size"); 23 | } 24 | if (Arrays.stream(leftProjection).anyMatch(i -> i >= left.getArity())) { 25 | throw new IllegalArgumentException("Invalid left projection: trying to project a non-existing field"); 26 | } 27 | 28 | this.left = left; 29 | this.right = right; 30 | this.leftProjection = leftProjection; 31 | } 32 | 33 | public PlanNode getLeft() { 34 | return left; 35 | } 36 | 37 | public PlanNode getRight() { 38 | return right; 39 | } 40 | 41 | public int[] getLeftProjection() { 42 | return leftProjection; 43 | } 44 | 45 | @Override 46 | public int getArity() { 47 | return left.getArity(); 48 | } 49 | 50 | @Override 51 | public String toString() { 52 | return left.toString() + operatorString() + right.toString(); 53 | } 54 | 55 | @Override 56 | public String operatorString() { 57 | return "▷_{" + IntStream.range(0, leftProjection.length).mapToObj(i -> leftProjection[i] + "=" + i).collect(Collectors.joining(", ")) + "}"; 58 | } 59 | 60 | @Override 61 | public List children() { 62 | return Arrays.asList(left, right); 63 | } 64 | 65 | @Override 66 | public boolean equals(Object obj) { 67 | return equals(obj, Collections.emptyMap()); 68 | } 69 | 70 | @Override 71 | public boolean equals(Object obj, Map assumedEqualities) { 72 | if (this == obj) return true; 73 | if (!(obj.getClass() == getClass())) { 74 | return false; 75 | } 76 | AntiJoinNode node = (AntiJoinNode) obj; 77 | return Arrays.equals(leftProjection, node.leftProjection) && left.equals(node.left, assumedEqualities) 78 | && right.equals(node.right, assumedEqualities); 79 | } 80 | 81 | @Override 82 | public int hashCode() { 83 | return left.hashCode() ^ right.hashCode() ^ Arrays.hashCode(leftProjection); 84 | } 85 | 86 | @Override 87 | public PlanNode transform(TransformFn fn, List originalPath) { 88 | try { 89 | Tools.addLast(originalPath, this); 90 | PlanNode newLeft = left.transform(fn, originalPath); 91 | PlanNode newRight = right.transform(fn, originalPath); 92 | 93 | PlanNode newNode = left.equals(newLeft) && right.equals(newRight) ? this : newLeft.antiJoin(newRight, leftProjection); 94 | return fn.apply(this, newNode, originalPath); 95 | } finally { 96 | Tools.removeLast(originalPath); 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/main/java/common/plan/node/BashNode.java: -------------------------------------------------------------------------------- 1 | package common.plan.node; 2 | 3 | import java.util.*; 4 | import java.util.stream.Collectors; 5 | 6 | import common.Tools; 7 | 8 | public class BashNode implements PlanNode { 9 | 10 | private final int arity; 11 | 12 | private final String command; 13 | 14 | private final List commandParts; 15 | 16 | private final List children; 17 | 18 | public BashNode(String command, List commandParts, Collection children, int arity) { 19 | this.command = command; 20 | this.commandParts = commandParts; 21 | this.children = new ArrayList<>(children); 22 | this.arity = arity; 23 | } 24 | 25 | @Override 26 | public int getArity() { 27 | return arity; 28 | } 29 | 30 | public String getCommand() { 31 | return command; 32 | } 33 | 34 | public List getCommandParts() { 35 | return commandParts; 36 | } 37 | 38 | @Override 39 | public String operatorString() { 40 | return "bash: " + commandParts; 41 | } 42 | 43 | @Override 44 | public String toString() { 45 | return operatorString(); 46 | } 47 | 48 | @Override 49 | public List children() { 50 | return children; 51 | } 52 | 53 | @Override 54 | public boolean equals(Object obj) { 55 | return equals(obj, Collections.emptyMap()); 56 | } 57 | 58 | @Override 59 | public boolean equals(Object obj, Map assumedEqualities) { 60 | if (this == obj) return true; 61 | if (obj.getClass() != this.getClass()) { 62 | return false; 63 | } 64 | 65 | BashNode node = (BashNode) obj; 66 | if (this.arity != node.arity || !this.commandParts.equals(node.commandParts) || children.size() != node.children.size()) return false; 67 | for (int i = 0; i < children.size(); i++) { 68 | if (!children.get(i).equals(node.children.get(i), assumedEqualities)) { 69 | return false; 70 | } 71 | } 72 | return true; 73 | } 74 | 75 | @Override 76 | public int hashCode() { 77 | return arity + Objects.hash(commandParts, children); 78 | } 79 | 80 | @Override 81 | public PlanNode transform(TransformFn fn, List originalPath) { 82 | try { 83 | Tools.addLast(originalPath, this); 84 | List newChildren = this.children.stream().map(pn -> pn.transform(fn, originalPath)).collect(Collectors.toList()); 85 | PlanNode newNode = this; 86 | if (!this.children.equals(newChildren)) { 87 | newNode = new BashNode(this.command, this.commandParts, newChildren, this.arity); 88 | } 89 | return fn.apply(this, newNode, originalPath); 90 | } finally { 91 | Tools.removeLast(originalPath); 92 | } 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /src/main/java/common/plan/node/BuiltinNode.java: -------------------------------------------------------------------------------- 1 | package common.plan.node; 2 | 3 | import java.util.Collections; 4 | import java.util.List; 5 | 6 | import common.parser.CompoundTerm; 7 | import common.parser.Constant; 8 | 9 | public class BuiltinNode implements PlanNode { 10 | 11 | public final CompoundTerm compoundTerm; 12 | 13 | public BuiltinNode(CompoundTerm t) { 14 | this.compoundTerm = t; 15 | } 16 | 17 | @Override 18 | public int getArity() { 19 | return (int) compoundTerm.getVariables().count(); 20 | } 21 | 22 | @Override 23 | public String toString() { 24 | return operatorString(); 25 | } 26 | 27 | @Override 28 | public String operatorString() { 29 | if ("bash_command".equals(compoundTerm.name)) { 30 | return "$ " + ((Constant) compoundTerm.args[0]).getValue(); 31 | } 32 | return "builtin:" + compoundTerm; 33 | } 34 | 35 | @Override 36 | public List children() { 37 | return Collections.emptyList(); 38 | } 39 | 40 | @Override 41 | public boolean equals(Object obj) { 42 | return obj instanceof BuiltinNode && compoundTerm.equals(((BuiltinNode) obj).compoundTerm); 43 | } 44 | 45 | @Override 46 | public int hashCode() { 47 | return compoundTerm.hashCode(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/common/plan/node/ConstantEqualityFilterNode.java: -------------------------------------------------------------------------------- 1 | package common.plan.node; 2 | 3 | import java.util.*; 4 | 5 | import common.Tools; 6 | 7 | public class ConstantEqualityFilterNode implements EqualityFilterNode { 8 | 9 | private final PlanNode table; 10 | 11 | private final int field; 12 | 13 | private final Comparable value; 14 | 15 | ConstantEqualityFilterNode(PlanNode table, int field, Comparable value) { 16 | if (field >= table.getArity()) { 17 | throw new IllegalArgumentException("The table has arity " + table.getArity() + " and the field has id " + field); 18 | } 19 | this.table = table; 20 | this.field = field; 21 | this.value = value; 22 | } 23 | 24 | public PlanNode getTable() { 25 | return table; 26 | } 27 | 28 | public int getField() { 29 | return field; 30 | } 31 | 32 | public Comparable getValue() { 33 | return value; 34 | } 35 | 36 | @Override 37 | public int getArity() { 38 | return table.getArity(); 39 | } 40 | 41 | @Override 42 | public String toString() { 43 | return operatorString() + "(" + table + ")"; 44 | } 45 | 46 | @Override 47 | public String operatorString() { 48 | return "σ_{" + field + " = \"" + value.toString() + "\"}"; 49 | } 50 | 51 | @Override 52 | public List children() { 53 | return Collections.singletonList(table); 54 | } 55 | 56 | @Override 57 | public boolean equals(Object obj) { 58 | return equals(obj, Collections.emptyMap()); 59 | } 60 | 61 | @Override 62 | public boolean equals(Object obj, Map assumedEqualities) { 63 | if (this == obj) return true; 64 | if (!(obj.getClass() == getClass())) { 65 | return false; 66 | } 67 | ConstantEqualityFilterNode node = (ConstantEqualityFilterNode) obj; 68 | return field == node.field && value.equals(node.value) && table.equals(node.table, assumedEqualities); 69 | } 70 | 71 | @Override 72 | public int hashCode() { 73 | return "selection".hashCode() ^ table.hashCode() ^ value.hashCode(); 74 | } 75 | 76 | @Override 77 | public PlanNode transform(TransformFn fn, List originalPath) { 78 | try { 79 | Tools.addLast(originalPath, this); 80 | PlanNode newTable = table.transform(fn, originalPath); 81 | PlanNode newNode = newTable.equals(table) ? this : newTable.equalityFilter(field, value); 82 | return fn.apply(this, newNode, originalPath); 83 | } finally { 84 | Tools.removeLast(originalPath); 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/common/plan/node/EqualityFilterNode.java: -------------------------------------------------------------------------------- 1 | package common.plan.node; 2 | 3 | /** Filter for equality of two columns, or one column and a constant. */ 4 | public interface EqualityFilterNode extends PlanNode { 5 | PlanNode getTable(); 6 | } 7 | -------------------------------------------------------------------------------- /src/main/java/common/plan/node/FactNode.java: -------------------------------------------------------------------------------- 1 | package common.plan.node; 2 | 3 | import java.util.*; 4 | 5 | public class FactNode implements PlanNode { 6 | 7 | private final List[]> facts; 8 | 9 | public FactNode(String... fields) { 10 | this((Comparable[]) fields); 11 | } 12 | 13 | public FactNode(Comparable[] fields) { 14 | facts = new ArrayList<>(); 15 | facts.add(fields); 16 | } 17 | 18 | public FactNode(List list) { 19 | facts = new ArrayList<>(); 20 | for (FactNode fn : list) { 21 | facts.addAll(fn.facts); 22 | } 23 | // TODO: check arity 24 | } 25 | 26 | public List[]> getFacts() { 27 | return facts; 28 | } 29 | 30 | @Override 31 | public int getArity() { 32 | return facts.get(0).length; 33 | } 34 | 35 | @Override 36 | public String operatorString() { 37 | if (facts.size() == 1) { 38 | return "fact " + Arrays.toString(facts.get(0)); 39 | } 40 | return "facts (" + facts.size() + "x)"; 41 | } 42 | 43 | @Override 44 | public List children() { 45 | return Collections.emptyList(); 46 | } 47 | 48 | @Override 49 | public int hashCode() { 50 | return facts.stream().mapToInt(Object::hashCode).reduce(0, (a, b) -> a ^ b); 51 | } 52 | 53 | @Override 54 | public boolean equals(Object obj) { 55 | return equals(obj, Collections.emptyMap()); 56 | } 57 | 58 | @Override 59 | public boolean equals(Object obj, Map assumedEqualities) { 60 | if(this == obj) return true; 61 | if (obj.getClass() != this.getClass()) { 62 | return false; 63 | } 64 | 65 | FactNode node = (FactNode) obj; 66 | return facts.equals(node.facts); 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/common/plan/node/MaterializationNode.java: -------------------------------------------------------------------------------- 1 | package common.plan.node; 2 | 3 | import java.util.*; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import common.Tools; 9 | 10 | public class MaterializationNode implements PlanNode { 11 | 12 | private static final Logger LOG = LoggerFactory.getLogger(MaterializationNode.class); 13 | 14 | private final PlanNode mainPlan; 15 | 16 | private final PlanNode reusedPlan; 17 | 18 | private final PlaceholderNode reuseNode; 19 | 20 | /** see {@link #getReuseCount()} */ 21 | private final int reuseCount; 22 | 23 | public static class Builder { 24 | 25 | PlaceholderNode placeholder; 26 | 27 | public Builder(int arity) { 28 | placeholder = new PlaceholderNode("reuse_mat", arity);//new ReuseNode(null, arity); 29 | } 30 | 31 | public PlanNode getReuseNode() { 32 | return placeholder; 33 | } 34 | 35 | public MaterializationNode build(PlanNode mainPlan, PlanNode reusedPlan, int reuseCount) { 36 | MaterializationNode result = new MaterializationNode(mainPlan, reusedPlan, placeholder, reuseCount); 37 | return result; 38 | } 39 | 40 | } 41 | 42 | /** Use builder if possible */ 43 | protected MaterializationNode(PlanNode mainPlan, PlanNode reusedPlan, PlaceholderNode reuseNode, int reuseCount) { 44 | // first initialize reuse node! 45 | 46 | if (!(reuseNode instanceof PlaceholderNode)) { 47 | throw new IllegalArgumentException(); 48 | } 49 | this.reuseNode = reuseNode; 50 | this.reusedPlan = reusedPlan; 51 | if (!mainPlan.contains(reuseNode)) { 52 | System.err.println("exception while constructing " + this.operatorString()); 53 | System.err.println("main plan:\n" + mainPlan.toPrettyString()); 54 | throw new IllegalArgumentException( 55 | "incorrect reuse node specified:" + reuseNode.toPrettyString() + "\nfor reuse plan\n" + reusedPlan.toPrettyString()); 56 | } 57 | this.mainPlan = mainPlan.replace(reuseNode, this.reuseNode); 58 | this.reuseCount = reuseCount; 59 | } 60 | 61 | public PlanNode getReusedPlan() { 62 | return reusedPlan; 63 | } 64 | 65 | public PlanNode getMainPlan() { 66 | return mainPlan; 67 | } 68 | 69 | public PlaceholderNode getReuseNode() { 70 | return reuseNode; 71 | } 72 | 73 | /** How often the plan is reused. Infinity is denoted by -1 */ 74 | public int getReuseCount() { 75 | return reuseCount; 76 | } 77 | 78 | @Override 79 | public int getArity() { 80 | return mainPlan == null ? 0 : mainPlan.getArity(); 81 | } 82 | 83 | @Override 84 | public String operatorString() { 85 | return "mat_" + hash(); 86 | } 87 | 88 | @Override 89 | public List children() { 90 | return Arrays.asList(reusedPlan, mainPlan); 91 | } 92 | 93 | @Override 94 | public List placeholders() { 95 | return Arrays.asList(reuseNode); 96 | } 97 | 98 | @Override 99 | public PlanNode transform(TransformFn fn, List originalPath) { 100 | try { 101 | Tools.addLast(originalPath, this); 102 | PlanNode newMainPlan = mainPlan.transform(fn, originalPath); 103 | PlanNode newReusedPlan = reusedPlan.transform(fn, originalPath); 104 | PlanNode newNode = mainPlan.equals(newMainPlan) && reusedPlan.equals(newReusedPlan) ? this 105 | : new MaterializationNode(newMainPlan, newReusedPlan, reuseNode, reuseCount); 106 | return fn.apply(this, newNode, originalPath); 107 | } finally { 108 | Tools.removeLast(originalPath); 109 | } 110 | } 111 | 112 | @Override 113 | public boolean equals(Object obj) { 114 | return equals(obj, Collections.emptyMap()); 115 | } 116 | 117 | @Override 118 | public boolean equals(Object obj, Map assumedEqualities) { 119 | if (this == obj) return true; 120 | if (!(obj.getClass() == getClass())) { 121 | return false; 122 | } 123 | MaterializationNode node = (MaterializationNode) obj; 124 | assumedEqualities = Tools.with(assumedEqualities, reuseNode, node.reuseNode); 125 | return mainPlan.equals(node.mainPlan, assumedEqualities) && reusedPlan.equals(node.reusedPlan, assumedEqualities); 126 | } 127 | 128 | @Override 129 | public int hashCode() { 130 | return Objects.hash(mainPlan, reusedPlan); 131 | } 132 | 133 | @Override 134 | public String toString() { 135 | return operatorString(); 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /src/main/java/common/plan/node/MultiFilterNode.java: -------------------------------------------------------------------------------- 1 | package common.plan.node; 2 | 3 | import java.util.*; 4 | import java.util.stream.Collectors; 5 | import java.util.stream.Stream; 6 | 7 | import common.Tools; 8 | 9 | public class MultiFilterNode implements PlanNode { 10 | 11 | private final PlanNode table; 12 | 13 | protected final Set children; 14 | 15 | protected final int arity; 16 | 17 | protected final String operatorString; 18 | 19 | private final PlaceholderNode placeholder; 20 | 21 | public MultiFilterNode(Set children, PlanNode table, int arity) { 22 | this.arity = arity; 23 | this.table = table; 24 | 25 | placeholder = new PlaceholderNode("inner_plan", table.getArity()); 26 | 27 | this.children = children.stream().map(c -> c.replace(table, placeholder)).collect(Collectors.toSet()); 28 | operatorString = this.children.stream().map(PlanNode::toString).collect(Collectors.joining(", ")); 29 | children.stream().filter(n -> !n.contains(table)).forEach(child -> { 30 | throw new IllegalArgumentException("child doesn't contain inner plan: " + child.toPrettyString()); 31 | }); 32 | } 33 | 34 | public MultiFilterNode(Set children, PlanNode table, PlaceholderNode placeholder, int arity) { 35 | this.children = children; 36 | this.table = table; 37 | this.placeholder = placeholder; 38 | this.arity = arity; 39 | operatorString = this.children.stream().map(PlanNode::toString).collect(Collectors.joining(", ")); 40 | } 41 | 42 | public PlanNode getTable() { 43 | return table; 44 | } 45 | 46 | public Set getFilter() { 47 | return children; 48 | } 49 | 50 | // TODO: toString / operatorString 51 | public String operatorString() { 52 | return "multi_filter_" + hash(); 53 | } 54 | 55 | @Override 56 | public int getArity() { 57 | return arity; 58 | } 59 | 60 | @Override 61 | public List children() { 62 | return Arrays.asList(table); 63 | } 64 | 65 | @Override 66 | public List childrenForPrettyString() { 67 | return Stream.concat(children.stream(), Stream.of(table)).collect(Collectors.toList()); 68 | } 69 | 70 | @Override 71 | public List placeholders() { 72 | return Arrays.asList(placeholder); 73 | } 74 | 75 | @Override 76 | public boolean equals(Object obj) { 77 | return equals(obj, Collections.emptyMap()); 78 | } 79 | 80 | @Override 81 | public boolean equals(Object obj, Map assumedEqualities) { 82 | if (this == obj) return true; 83 | if (!(obj.getClass() == getClass())) { 84 | return false; 85 | } 86 | MultiFilterNode node = (MultiFilterNode) obj; 87 | Map newAssumedEqualities = Tools.with(assumedEqualities, placeholder, node.placeholder); 88 | return table.equals(node.table, newAssumedEqualities) && children.size() == node.children.size() 89 | && children.stream().allMatch(child -> node.children.stream().anyMatch(other -> other.equals(child, newAssumedEqualities))); 90 | } 91 | 92 | @Override 93 | public int hashCode() { 94 | return children.hashCode(); 95 | } 96 | 97 | public PlanNode transform(TransformFn fn, List originalPath) { 98 | try { 99 | Tools.addLast(originalPath, this); 100 | PlanNode newTable = table.transform(fn, originalPath); 101 | PlanNode newNode = newTable.equals(table) ? this : new MultiFilterNode(children, newTable, placeholder, arity); 102 | return fn.apply(this, newNode, originalPath); 103 | } finally { 104 | Tools.removeLast(originalPath); 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/main/java/common/plan/node/PlaceholderNode.java: -------------------------------------------------------------------------------- 1 | package common.plan.node; 2 | 3 | import java.util.*; 4 | import java.util.stream.Collectors; 5 | 6 | public class PlaceholderNode implements PlanNode { 7 | 8 | protected String operatorString; 9 | 10 | protected final int arity; 11 | 12 | public PlaceholderNode(String operatorString, int arity) { 13 | this.operatorString = operatorString; 14 | this.arity = arity; 15 | } 16 | 17 | @Override 18 | public String toString() { 19 | return operatorString; 20 | } 21 | 22 | @Override 23 | public boolean equals(Object obj) { 24 | return equals(obj, Collections.emptyMap()); 25 | } 26 | 27 | @Override 28 | public boolean equals(Object obj, Map assumedEqualities) { 29 | if (this == obj) return true; 30 | return assumedEqualities.getOrDefault(this, this) == obj; 31 | } 32 | 33 | @Override 34 | public int hashCode() { 35 | return System.identityHashCode(this); 36 | } 37 | 38 | @Override 39 | public int getArity() { 40 | return arity; 41 | } 42 | 43 | @Override 44 | public List children() { 45 | return Collections.emptyList(); 46 | } 47 | 48 | @Override 49 | public String operatorString() { 50 | return operatorString; 51 | } 52 | 53 | /** Walk through plan, and create a map for obtaining the parents of placeholder nodes */ 54 | public static Map placeholderToParentMap(PlanNode plan) { 55 | Map map = new HashMap<>(); 56 | if (plan != null) { 57 | plan.transform(n -> { 58 | for (PlaceholderNode pn : n.placeholders()) { 59 | map.put(pn, n); 60 | } 61 | return n; 62 | }); 63 | } 64 | return map; 65 | } 66 | 67 | /** Walk through the plan and collect outer nodes that have a place holder in the plan given by argument 'ofPlan' 68 | * @param placeholderToParent */ 69 | public static Set outerParents(PlanNode ofPlan, Map placeholderToParent) { 70 | HashMap nodeToContained = new HashMap<>(); 71 | 72 | ofPlan.transform(pn -> { 73 | nodeToContained.put(pn, true); 74 | 75 | if (pn instanceof PlaceholderNode) { 76 | nodeToContained.putIfAbsent(placeholderToParent.get(pn), false); 77 | } 78 | return pn; 79 | }); 80 | 81 | return nodeToContained.entrySet().stream().filter(e -> !e.getValue()).map(e -> e.getKey()).collect(Collectors.toSet()); 82 | } 83 | 84 | /** Get all placeholders that occur in a plan */ 85 | public static Set searchInPlan(PlanNode plan) { 86 | Set placeholders = new HashSet<>(); 87 | plan.transform(x -> { 88 | if (x instanceof PlaceholderNode) { 89 | placeholders.add((PlaceholderNode) x); 90 | } 91 | return x; 92 | }); 93 | return placeholders; 94 | } 95 | 96 | } -------------------------------------------------------------------------------- /src/main/java/common/plan/node/ProjectNode.java: -------------------------------------------------------------------------------- 1 | package common.plan.node; 2 | 3 | import java.util.*; 4 | import java.util.function.IntFunction; 5 | import java.util.stream.Collectors; 6 | import java.util.stream.IntStream; 7 | 8 | import common.Tools; 9 | 10 | /** Restrict a plan node to certain columns, and/or introduce new columns (based on existing columns or constants). */ 11 | public class ProjectNode implements PlanNode { 12 | 13 | private final PlanNode table; 14 | 15 | private final int[] projection; 16 | 17 | private final Comparable[] constants; 18 | 19 | /** 20 | * @param projection position in the new tuple => position in the old tuple. If <0 the value is null (i.e. empty fields) 21 | * @param constants position in the new tuple => constant to set 22 | */ 23 | ProjectNode(PlanNode table, int[] projection, Comparable[] constants) { 24 | if (Arrays.stream(projection).anyMatch(i -> i >= table.getArity())) { 25 | throw new IllegalArgumentException("Invalid projection: trying to project a non-existing field: " + Arrays.toString(projection) + "\n" + table 26 | + ", arity " + table.getArity()); 27 | } 28 | if (IntStream.range(0, projection.length).anyMatch(i -> projection[i] < 0 && (constants.length <= i || constants[i] == null))) { 29 | throw new IllegalArgumentException("Invalid projection: trying to project a non-existing field"); 30 | } 31 | 32 | this.table = table; 33 | this.projection = projection; 34 | this.constants = IntStream.range(0, projection.length).mapToObj(i -> i < constants.length ? constants[i] : null) 35 | .toArray((IntFunction[]>) Comparable[]::new); 36 | } 37 | 38 | /** Table the projection applies to */ 39 | public PlanNode getTable() { 40 | return table; 41 | } 42 | 43 | /** Get projection columns */ 44 | public int[] getProjection() { 45 | return projection; 46 | } 47 | 48 | /** Get projection constants, (used for columns with a constant value) */ 49 | public Comparable[] getConstants() { 50 | return constants; 51 | } 52 | 53 | /** Get constant for column i */ 54 | public Optional> getConstant(int i) { 55 | return Tools.get(constants, i); 56 | } 57 | 58 | @Override 59 | public int getArity() { 60 | return Math.max(projection.length, constants.length); 61 | } 62 | 63 | @Override 64 | public String toString() { 65 | return operatorString() + "(" + table.toString() + ")"; 66 | } 67 | 68 | @Override 69 | public String operatorString() { 70 | String params = IntStream.range(0, projection.length).mapToObj(i -> { 71 | if (projection[i] >= 0) { 72 | if (i == projection[i]) { 73 | return "" + i; 74 | } else { 75 | return Integer.toString(i) + " = " + projection[i]; 76 | } 77 | } else { 78 | return Integer.toString(i) + " = " + getConstant(i).map(Object::toString).orElse("null"); 79 | } 80 | }).filter(p -> !p.isEmpty()).collect(Collectors.joining(", ")); 81 | return "π_{" + params + "}"; 82 | } 83 | 84 | @Override 85 | public List children() { 86 | return Collections.singletonList(table); 87 | } 88 | 89 | @Override 90 | public boolean equals(Object obj) { 91 | return equals(obj, Collections.emptyMap()); 92 | } 93 | 94 | @Override 95 | public boolean equals(Object obj, Map assumedEqualities) { 96 | if (this == obj) return true; 97 | if (!(obj.getClass() == getClass())) { 98 | return false; 99 | } 100 | ProjectNode node = (ProjectNode) obj; 101 | return Arrays.equals(projection, node.projection) && Arrays.equals(constants, node.constants) && table.equals(node.table, assumedEqualities); 102 | } 103 | 104 | @Override 105 | public int hashCode() { 106 | return "projection".hashCode() ^ table.hashCode() ^ Arrays.hashCode(projection) ^ Arrays.hashCode(constants); 107 | } 108 | 109 | @Override 110 | public PlanNode transform(TransformFn fn, List originalPath) { 111 | try { 112 | Tools.addLast(originalPath, this); 113 | PlanNode newTable = table.transform(fn, originalPath); 114 | PlanNode newNode = newTable.equals(table) ? this : newTable.project(projection, constants); 115 | return fn.apply(this, newNode, originalPath); 116 | } finally { 117 | Tools.removeLast(originalPath); 118 | } 119 | } 120 | 121 | /** Whether this projection uses any constants */ 122 | public boolean hasConstants() { 123 | return constants.length != 0 && Arrays.stream(constants).anyMatch(Objects::nonNull); 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /src/main/java/common/plan/node/UnionNode.java: -------------------------------------------------------------------------------- 1 | package common.plan.node; 2 | 3 | import java.util.*; 4 | import java.util.stream.Collectors; 5 | import java.util.stream.Stream; 6 | 7 | import common.Tools; 8 | 9 | public class UnionNode implements PlanNode { 10 | 11 | protected final Set children; 12 | 13 | protected final int arity; 14 | 15 | private UnionNode(Set children, int arity) { 16 | this.children = children; 17 | this.arity = arity; 18 | } 19 | 20 | UnionNode(int arity) { 21 | this(Collections.emptySet(), arity); 22 | } 23 | 24 | public UnionNode(Set children) { 25 | if (children.size() == 0) { 26 | throw new IllegalArgumentException("Please set the arity of this empty union"); 27 | } 28 | 29 | int arity = -1; 30 | for (PlanNode child : children) { 31 | if (arity == -1) { 32 | arity = child.getArity(); 33 | } else if (child.getArity() != arity) { 34 | throw new IllegalArgumentException("All terms of the union should have the same arity"); 35 | } 36 | } 37 | this.arity = arity; 38 | this.children = children; 39 | } 40 | 41 | public Collection getChildren() { 42 | return children; 43 | } 44 | 45 | @Override 46 | public int getArity() { 47 | return arity; 48 | } 49 | 50 | @Override 51 | public boolean isEmpty() { 52 | return children.isEmpty(); 53 | } 54 | 55 | @Override 56 | public String toString() { 57 | return children.stream().map(Object::toString).collect(Collectors.joining(" " + operatorString() + " ")); 58 | } 59 | 60 | @Override 61 | public String operatorString() { 62 | return "∪"; 63 | } 64 | 65 | @Override 66 | public Set children() { 67 | return Collections.unmodifiableSet(children); 68 | } 69 | 70 | @Override 71 | public boolean equals(Object obj) { 72 | return equals(obj, Collections.emptyMap()); 73 | } 74 | 75 | @Override 76 | public boolean equals(Object obj, Map assumedEqualities) { 77 | if (this == obj) return true; 78 | if (!(obj.getClass() == getClass())) { 79 | return false; 80 | } 81 | UnionNode node = (UnionNode) obj; 82 | return this.arity == node.getArity() && children.size() == node.children.size() 83 | && children.stream().allMatch(child -> node.children.stream().anyMatch(other -> child.equals(other, assumedEqualities))); 84 | } 85 | 86 | @Override 87 | public int hashCode() { 88 | return children.hashCode(); 89 | } 90 | 91 | public PlanNode transform(TransformFn fn, List originalPath) { 92 | try { 93 | Tools.addLast(originalPath, this); 94 | Set newChildren = children.stream()// 95 | .map(child -> child.transform(fn, originalPath))// 96 | .flatMap(child -> (Stream) ((child instanceof UnionNode) ? ((UnionNode) child).children().stream() : Stream.of(child)))// 97 | .collect(Collectors.toSet()); 98 | PlanNode newNode = newChildren.equals(children) ? this : PlanNode.empty(this.getArity()).union(newChildren); 99 | return fn.apply(this, newNode, originalPath); 100 | } finally { 101 | Tools.removeLast(originalPath); 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/common/plan/node/VariableEqualityFilterNode.java: -------------------------------------------------------------------------------- 1 | package common.plan.node; 2 | 3 | import java.util.*; 4 | 5 | import common.Tools; 6 | 7 | public class VariableEqualityFilterNode implements EqualityFilterNode { 8 | 9 | private final PlanNode table; 10 | 11 | private final int field1; 12 | 13 | private final int field2; 14 | 15 | VariableEqualityFilterNode(PlanNode table, int field1, int field2) { 16 | if (field1 >= table.getArity() || field2 >= table.getArity()) { 17 | throw new IllegalArgumentException("The table has arity " + table.getArity() + " and the fields have ids " + field1 + " and " + field2); 18 | } 19 | this.table = table; 20 | this.field1 = Math.min(field1, field2); 21 | this.field2 = Math.max(field1, field2); 22 | } 23 | 24 | public PlanNode getTable() { 25 | return table; 26 | } 27 | 28 | public int getField1() { 29 | return field1; 30 | } 31 | 32 | public int getField2() { 33 | return field2; 34 | } 35 | 36 | @Override 37 | public int getArity() { 38 | return table.getArity(); 39 | } 40 | 41 | @Override 42 | public String toString() { 43 | return operatorString() + "(" + table.toString() + ")"; 44 | } 45 | 46 | @Override 47 | public String operatorString() { 48 | return "σ_{" + field1 + " = " + field2 + "}"; 49 | } 50 | 51 | @Override 52 | public List children() { 53 | return Collections.singletonList(table); 54 | } 55 | 56 | @Override 57 | public boolean equals(Object obj) { 58 | return equals(obj, Collections.emptyMap()); 59 | } 60 | 61 | @Override 62 | public boolean equals(Object obj, Map assumedEqualities) { 63 | if (this == obj) return true; 64 | if (!(obj.getClass() == getClass())) { 65 | return false; 66 | } 67 | VariableEqualityFilterNode node = (VariableEqualityFilterNode) obj; 68 | return field1 == node.field1 && field2 == node.field2 && table.equals(node.table, assumedEqualities); 69 | } 70 | 71 | @Override 72 | public int hashCode() { 73 | return "selection".hashCode() ^ table.hashCode() ^ (field1 + field2); 74 | } 75 | 76 | @Override 77 | public PlanNode transform(TransformFn fn, List originalPath) { 78 | try { 79 | Tools.addLast(originalPath, this); 80 | PlanNode newTable = table.transform(fn, originalPath); 81 | PlanNode newNode = newTable.equals(table) ? this : newTable.equalityFilter(field1, field2); 82 | return fn.apply(this, newNode, originalPath); 83 | } finally { 84 | Tools.removeLast(originalPath); 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/common/plan/optimizer/CatToFile.java: -------------------------------------------------------------------------------- 1 | package common.plan.optimizer; 2 | 3 | import bashlog.plan.TSVFileNode; 4 | import common.parser.ParserReader; 5 | import common.plan.node.BashNode; 6 | import common.plan.node.PlanNode; 7 | 8 | public class CatToFile implements Optimizer { 9 | 10 | @Override 11 | public PlanNode apply(PlanNode t) { 12 | return t.transform(pn -> { 13 | if (pn instanceof BashNode) { 14 | String cmd = ((BashNode) pn).getCommand(); 15 | // TODO: support multiple file names 16 | if (cmd.trim().startsWith("cat ")) { 17 | ParserReader pr = new ParserReader(cmd); 18 | pr.expect("cat "); 19 | pr.skipWhitespace(); 20 | String file; 21 | if (pr.peek() == '\"' || pr.peek() == '\'') file = pr.readString(); 22 | else file = pr.readWhile((c, s) -> !Character.isWhitespace(c)); 23 | pr.skipWhitespace(); 24 | if (pr.peek() == '\0') { 25 | if (!file.startsWith("!")) { 26 | return new TSVFileNode(file, pn.getArity()); 27 | } 28 | } 29 | } 30 | } 31 | 32 | return pn; 33 | }); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/common/plan/optimizer/CombineFacts.java: -------------------------------------------------------------------------------- 1 | package common.plan.optimizer; 2 | 3 | import java.util.List; 4 | import java.util.Set; 5 | import java.util.stream.Collectors; 6 | 7 | import common.plan.node.*; 8 | 9 | /** 10 | * Combine a union of selections/projections on one subplan to a single node 11 | */ 12 | public class CombineFacts implements Optimizer { 13 | 14 | @Override 15 | public PlanNode apply(PlanNode t) { 16 | return t.transform((old, node, parent) -> { 17 | if (node.getClass() == UnionNode.class) { 18 | UnionNode u = (UnionNode) node; 19 | 20 | // collect facts 21 | List facts = u.children().stream().filter(c -> c instanceof FactNode).map(c -> (FactNode) c).collect(Collectors.toList()); 22 | 23 | if (facts.size() > 1) { 24 | FactNode fn = new FactNode(facts); 25 | Set others = u.children().stream().filter(c -> !(c instanceof FactNode)).collect(Collectors.toSet()); 26 | return fn.union(others); 27 | } 28 | 29 | return u; 30 | } 31 | 32 | return node; 33 | }); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/common/plan/optimizer/CombineFilter.java: -------------------------------------------------------------------------------- 1 | package common.plan.optimizer; 2 | 3 | import java.util.*; 4 | 5 | import common.plan.node.*; 6 | 7 | /** 8 | * Combine a union of selections/projections on one subplan to a single node 9 | */ 10 | public class CombineFilter implements Optimizer { 11 | 12 | private boolean condenseNonUnionFilter = false; 13 | 14 | public CombineFilter(boolean condenseNonUnionFilter) { 15 | this.condenseNonUnionFilter = condenseNonUnionFilter; 16 | } 17 | 18 | @Override 19 | public PlanNode apply(PlanNode t) { 20 | return t.transform((old, node, parent) -> { 21 | if (node.getClass() == UnionNode.class) { 22 | UnionNode u = (UnionNode) node; 23 | 24 | // collect plans within projections/selections 25 | Map> innerToFiltered = new HashMap<>(); 26 | for (PlanNode c : u.children()) { 27 | innerToFiltered.computeIfAbsent(getInnerTable(c), k -> new HashSet<>()).add(c); 28 | } 29 | 30 | // replace by a multi filter node 31 | return innerToFiltered.entrySet().stream().map(e -> { 32 | boolean isMultiFilter = e.getValue().size() > 1; 33 | if (condenseNonUnionFilter && !isMultiFilter && e.getValue().size() == 1) { 34 | int depth = getFilterDepth(e.getValue().iterator().next()); 35 | isMultiFilter = depth > 1; 36 | } 37 | return isMultiFilter ? new MultiFilterNode(e.getValue(), e.getKey(), u.getArity()) : e.getValue().iterator().next(); 38 | }).reduce(PlanNode.empty(node.getArity()), PlanNode::union); 39 | } 40 | 41 | return node; 42 | }); 43 | } 44 | 45 | /** Descend into project and filter */ 46 | public static PlanNode getInnerTable(PlanNode p) { 47 | if (p instanceof ProjectNode || p instanceof VariableEqualityFilterNode || p instanceof ConstantEqualityFilterNode) { 48 | return getInnerTable(p.children().iterator().next()); 49 | } 50 | return p; 51 | } 52 | 53 | /** Descend into project and filter */ 54 | public static int getFilterDepth(PlanNode parent) { 55 | if (parent instanceof ProjectNode || parent instanceof VariableEqualityFilterNode || parent instanceof ConstantEqualityFilterNode) { 56 | return 1 + getFilterDepth(parent.children().iterator().next()); 57 | } 58 | return 0; 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/common/plan/optimizer/Optimizer.java: -------------------------------------------------------------------------------- 1 | package common.plan.optimizer; 2 | 3 | import java.util.Iterator; 4 | import java.util.List; 5 | import java.util.function.Function; 6 | 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import common.plan.node.PlanNode; 11 | 12 | /** A plan node optimizer transforms one plan node to another one */ 13 | public interface Optimizer extends Function { 14 | 15 | static PlanNode applyOptimizer(PlanNode root, List> stages) { 16 | for (List stage : stages) { 17 | for (Optimizer o : stage) { 18 | root = o.apply(root); 19 | } 20 | } 21 | return root; 22 | } 23 | 24 | static PlanNode applyOptimizer(PlanNode root, List stageNames, List> stages, StringBuilder debugBuilder) { 25 | Iterator it = stageNames.iterator(); 26 | PlanValidator check = new PlanValidator(debugBuilder); 27 | for (List stage : stages) { 28 | debugBuilder.append("\n\n").append(it.hasNext() ? it.next() : "").append("\n"); 29 | for (Optimizer o : stage) { 30 | PlanNode prevPlan = root; 31 | try { 32 | root = o.apply(root); 33 | 34 | debugBuilder.append("applied ").append(o.getClass()).append(" \n"); 35 | debugBuilder.append(root.toPrettyString()).append("\n"); 36 | 37 | check.apply(root); 38 | } catch (Exception e) { 39 | System.err.println("some problems while applying " + o + " to plan"); 40 | System.err.println(prevPlan.toPrettyString()); 41 | 42 | LogHolder.LOG.error(e.getMessage()); 43 | debugBuilder.append("WARNING: ").append(e.getMessage()); 44 | //throw e; 45 | } 46 | } 47 | } 48 | return root; 49 | } 50 | } 51 | 52 | final class LogHolder { 53 | 54 | static final Logger LOG = LoggerFactory.getLogger(Optimizer.class); 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/common/plan/optimizer/PlanValidator.java: -------------------------------------------------------------------------------- 1 | package common.plan.optimizer; 2 | 3 | import java.util.*; 4 | import java.util.Map.Entry; 5 | import java.util.stream.Collectors; 6 | 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import common.plan.node.*; 11 | 12 | /** 13 | * Checks a plan for errors 14 | * - union nodes having a union node as child 15 | * - plans that contain placeholders, but no parents 16 | */ 17 | public class PlanValidator implements Optimizer { 18 | 19 | private static final Logger LOG = LoggerFactory.getLogger(PlanValidator.class); 20 | 21 | private CharSequence debug = null; 22 | 23 | private static final String save = "mat_1777"; 24 | 25 | Set saved = new HashSet<>(); 26 | 27 | public PlanValidator(CharSequence debugInfo) { 28 | this.debug = debugInfo; 29 | } 30 | 31 | @Override 32 | public PlanNode apply(PlanNode node) { 33 | Set allPlaceholders = new HashSet<>(); 34 | Map> placeholderToParent = new HashMap<>(); 35 | 36 | Set allNodes = new HashSet<>(); 37 | 38 | node.transform((orig, n, origParent) -> { 39 | if (save != null) { 40 | if (n.operatorString().contains(save)) { 41 | saved.add(n); 42 | if (saved.size() > 1) { 43 | System.out.println("here"); 44 | } 45 | } 46 | } 47 | if (n instanceof UnionNode) { 48 | check((UnionNode) n); 49 | } 50 | else if (n instanceof PlaceholderNode) { 51 | check((PlaceholderNode) n); 52 | } 53 | 54 | allNodes.add(n); 55 | n.placeholders().forEach(p -> placeholderToParent.computeIfAbsent(p, k -> new HashSet<>(1)).add(n)); 56 | if (n instanceof PlaceholderNode) allPlaceholders.add((PlaceholderNode) n); 57 | return n; 58 | }); 59 | 60 | allPlaceholders.removeAll(placeholderToParent.keySet()); 61 | if (allPlaceholders.size() > 0) { 62 | if (debug != null) { 63 | LOG.error("{}", debug); 64 | } 65 | allPlaceholders.stream() // 66 | .forEach(n -> LOG.error("parent of placeholder {} not found", n.operatorString())); 67 | throw new IllegalStateException("some orphaned placeholders, check log messages. " + (debug != null ? "debug length: " + debug.length() : "")); 68 | } 69 | 70 | Set>> cuckoos = placeholderToParent.entrySet().stream()// 71 | .filter(e -> e.getValue().size() > 1) 72 | .collect(Collectors.toSet()); 73 | 74 | if (cuckoos.size() > 0) { 75 | LOG.error("warning: some placeholders used multiple times: "); 76 | cuckoos.forEach(e -> LOG.error(" {} in {}", e.getKey(), e.getValue())); 77 | throw new IllegalStateException(); 78 | } 79 | 80 | return node; 81 | } 82 | 83 | private void check(UnionNode n) { 84 | for (PlanNode c : n.children()) { 85 | if (c instanceof UnionNode) { 86 | LOG.error("union may not have a union as child"); 87 | } 88 | } 89 | } 90 | 91 | private void check(PlaceholderNode n) { 92 | if (n.getArity() < 0) { 93 | LOG.error("placeholder has no arity: {}", n); 94 | } 95 | } 96 | 97 | } 98 | -------------------------------------------------------------------------------- /src/main/java/experiments/MainThomas.java: -------------------------------------------------------------------------------- 1 | package experiments; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.nio.file.Files; 6 | 7 | import bashlog.BashlogCompiler; 8 | import common.parser.Program; 9 | import experiments.lubm.BashlogLUBM; 10 | import sparqlog.SparqlogCompiler; 11 | 12 | public class MainThomas { 13 | 14 | public static void main(String[] args) throws IOException { 15 | /*System.out.println("press key"); 16 | System.in.read();*/ 17 | profile(); 18 | } 19 | 20 | public static void profile() throws IOException { 21 | // reach 22 | /*Program p = new Program().loadFile("experiments/edbt2017/reach/bashlog/query-com-orkut.ungraph.txt.gz.txt"); 23 | BashlogCompiler bc = BashlogCompiler.prepareQuery(p, "reach/1"); 24 | System.out.println(bc.compile());*/ 25 | 26 | // lubm 27 | //System.in.read(); 28 | 29 | String scriptDir = "experiments/edbt2017/lubm/bashlog/"; 30 | new File(scriptDir).mkdirs(); 31 | Program p = BashlogLUBM.lubmProgram3("~/extern/data/bashlog/lubm/$1/", "data/lubm"); 32 | String relation = BashlogLUBM.queries[1]; 33 | //relation = "Student/1"; 34 | //relation = "takesCourse/2"; 35 | //relation = BashlogLUBM.queries[2]; 36 | //for (int i = 1; i <= 14; i++) { 37 | for (int i = 11; i <= 11; i++) { 38 | relation = BashlogLUBM.queries[i - 1]; 39 | p.rulesForRelation(relation).forEach(r -> System.out.println(r)); 40 | BashlogCompiler bc = BashlogCompiler.prepareQuery(p, relation); 41 | 42 | try { 43 | bc.enableDebug(); 44 | String script = bc.compile(); 45 | Files.write(new File("/home/tr/tmp/bashlog/new_query" + i).toPath(), script.getBytes()); 46 | System.out.println(bc.debugInfo()); 47 | System.out.println(script); 48 | } catch (Exception e) { 49 | System.err.println("problem with lubm query " + i); 50 | e.printStackTrace(); 51 | } 52 | 53 | //SqllogCompiler sc = new SqllogCompiler(true, true); 54 | SparqlogCompiler sc = new SparqlogCompiler(); 55 | Program sqlProg = new Program(); 56 | p.rules().forEach(r -> { 57 | /*if (r instanceof BashRule) { 58 | // ignore this rule 59 | } else {*/ 60 | sqlProg.addRule(r); 61 | //} 62 | }); 63 | System.out.println(sc.compile(p, relation)); 64 | } 65 | 66 | //-------------------------------------------------------------------------------- 67 | 68 | /*String code = "rel(X,Y) :~ cat !old/2 \n" + // 69 | "old(X,Y) :~ cat abc.txt \n"; 70 | p = Program.read(new ParserReader(code)); 71 | BashlogCompiler bc = BashlogCompiler.prepareQuery(p, "rel"); 72 | String bash = bc.compile(); 73 | System.out.println(bash); 74 | System.out.println(bc.debugInfo());*/ 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/experiments/MainThomasOWL.java: -------------------------------------------------------------------------------- 1 | package experiments; 2 | 3 | import java.io.File; 4 | import java.util.Collections; 5 | 6 | import org.semanticweb.owlapi.apibinding.OWLManager; 7 | import org.semanticweb.owlapi.model.IRI; 8 | import org.semanticweb.owlapi.model.OWLOntologyManager; 9 | 10 | import bashlog.BashlogCompiler; 11 | import common.parser.ParserReader; 12 | import common.parser.Program; 13 | import rdf.OntologyConverter; 14 | import rdf.RDFSpecificTuplesSerializer; 15 | 16 | public class MainThomasOWL { 17 | 18 | private static String lubmScript(String lubmDir) { 19 | StringBuilder sb = new StringBuilder(); 20 | File dir = new File(lubmDir); 21 | for (File f : dir.listFiles()) { 22 | int cols = 1; 23 | if (Character.isLowerCase(f.getName().charAt(0))) { 24 | cols = 2; 25 | } 26 | sb.append(f.getName()).append("("); 27 | for (int i = 0; i < cols; i++) { 28 | if (i > 0) sb.append(", "); 29 | sb.append((char) ('X' + i)); 30 | } 31 | sb.append(") :~ cat ").append(lubmDir).append(f.getName()).append("\n"); 32 | } 33 | 34 | return sb.toString(); 35 | } 36 | 37 | public static Program lubmProgram(String lubmDir, String queryDir) throws Exception { 38 | Program lubmProgram = Program.merge(lubmTBox(), Program.loadFile(queryDir + "/queries.txt")); 39 | String script = lubmScript(lubmDir); 40 | lubmProgram.addRules(Program.read(new ParserReader(script))); 41 | return lubmProgram; 42 | } 43 | 44 | private static Program lubmTBox() throws Exception { 45 | OWLOntologyManager ontologyManager = OWLManager.createOWLOntologyManager(); 46 | IRI ontologyIRI = IRI.create("http://swat.cse.lehigh.edu/onto/univ-bench.owl"); 47 | ontologyManager.loadOntology(ontologyIRI); 48 | OntologyConverter converter = new OntologyConverter(new RDFSpecificTuplesSerializer( 49 | Collections.singletonMap("http://swat.cse.lehigh.edu/onto/univ-bench.owl#", "") 50 | )); 51 | return converter.convert(ontologyManager.getOntology(ontologyIRI)); 52 | } 53 | 54 | public static void main(String[] args) throws Exception { 55 | Program p = lubmProgram("data/lubm/1/", "data/lubm"); 56 | System.out.println(p); 57 | 58 | long start = System.currentTimeMillis(); 59 | System.out.println(BashlogCompiler.compileQuery(p, "query1/1")); 60 | System.out.println("end: " + (System.currentTimeMillis() - start) / 1000); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/experiments/MainThomasWikidata.java: -------------------------------------------------------------------------------- 1 | package experiments; 2 | 3 | import java.io.IOException; 4 | 5 | import bashlog.BashlogCompiler; 6 | import common.parser.Program; 7 | 8 | public class MainThomasWikidata { 9 | 10 | public static Program wikidataProgram(String queryDir) throws IOException { 11 | return Program.merge( 12 | Program.loadFile(queryDir + "/full_tbox.txt"), 13 | Program.loadFile(queryDir + "/full_queries.txt") 14 | ); 15 | } 16 | 17 | public static void main(String[] args) throws Exception { 18 | Program p = wikidataProgram("data/wikidata"); 19 | 20 | long start = System.currentTimeMillis(); 21 | BashlogCompiler bc = BashlogCompiler.prepareQuery(p, "query2/1"); 22 | System.out.println(bc.debugInfo()); 23 | System.out.println(bc.compile()); 24 | System.out.println("end: " + (System.currentTimeMillis() - start) / 1000); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/experiments/MainThomasYago.java: -------------------------------------------------------------------------------- 1 | package experiments; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.nio.file.Files; 6 | import java.nio.file.Paths; 7 | 8 | import bashlog.BashlogCompiler; 9 | import common.parser.Program; 10 | 11 | public class MainThomasYago { 12 | 13 | public static Program yagoProgram(String queryDir) throws IOException { 14 | return Program.merge( 15 | Program.loadFile(queryDir + "/full_tbox.txt"), 16 | Program.loadFile(queryDir + "/full_queries.txt") 17 | ); 18 | } 19 | 20 | public static void main(String[] args) throws IOException { 21 | 22 | String scriptDir = "experiments/edbt2017/yago/bashlog/"; 23 | String sqlDir = "experiments/edbt2017/yago/sql/"; 24 | new File(scriptDir).mkdirs(); 25 | new File(sqlDir).mkdirs(); 26 | 27 | for (int i = 5; i <= 5; i++) { 28 | Program p = yagoProgram("data/yago"); 29 | String relation = "query" + i + "/1"; 30 | try { 31 | String script = BashlogCompiler.compileQuery(p, relation); 32 | 33 | Program sqlProg = new Program(); 34 | p.rules().forEach(r -> { 35 | if (r.body.size() == 1 && "bash_command".equals(r.body.get(0).name)) { 36 | // ignore this rule 37 | } else { 38 | sqlProg.addRule(r); 39 | } 40 | }); 41 | 42 | Files.write(Paths.get(scriptDir + "query" + i + ".sh"), script.getBytes()); 43 | /*String sql = new SqllogCompiler().compile(sqlProg, new HashSet<>(Arrays.asList("allFacts/4")), relation); 44 | Files.write(Paths.get(sqlDir + "query" + (i + 1) + ".sql"), sql.getBytes());*/ 45 | } catch (Exception e) { 46 | throw new RuntimeException("in query " + i, e); 47 | } 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/experiments/lubm/generator/DamlWriter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * by Yuanbo Guo 3 | * Semantic Web and Agent Technology Lab, CSE Department, Lehigh University, USA 4 | * Copyright (C) 2004 5 | * 6 | * This program is free software; you can redistribute it and/or modify it under 7 | * the terms of the GNU General Public License as published by the Free Software 8 | * Foundation; either version 2 of the License, or (at your option) any later 9 | * version. 10 | * 11 | * This program is distributed in the hope that it will be useful, but WITHOUT 12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | * FOR A PARTICULAR PURPOSE.See the GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License along with 16 | * this program; if not, write to the Free Software Foundation, Inc., 59 Temple 17 | * Place, Suite 330, Boston, MA 02111-1307 USA 18 | */ 19 | 20 | package experiments.lubm.generator; 21 | 22 | public class DamlWriter 23 | extends RdfWriter { 24 | /** abbreviation of DAML+OIL namesapce */ 25 | private static final String T_DAML_NS = "daml"; 26 | /** prefix of DAML+OIL namespace */ 27 | private static final String T_DAML_PREFIX = T_DAML_NS + ":"; 28 | 29 | /** 30 | * Constructor. 31 | * @param generator The generator object. 32 | */ 33 | public DamlWriter(Generator generator) { 34 | super(generator); 35 | } 36 | 37 | /** 38 | * Writes the header part, including namespace declarations and imports statements. 39 | */ 40 | void writeHeader() { 41 | String s; 42 | s = "xmlns:" + T_RDF_NS + 43 | "=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\""; 44 | out.println(s); 45 | s = "xmlns:" + T_RDFS_NS + "=\"http://www.w3.org/2000/01/rdf-schema#\""; 46 | out.println(s); 47 | s = "xmlns:" + T_DAML_NS + "=\"http://www.daml.org/2001/03/daml+oil#\""; 48 | out.println(s); 49 | s = "xmlns:" + T_ONTO_NS + "=\"" + generator.ontology + "#\">"; 50 | out.println(s); 51 | s = "<" + T_RDF_PREFIX + "Description " + T_RDF_ABOUT + "=\"\">"; 52 | out.println(s); 53 | s = "<" + T_DAML_PREFIX + "imports " + T_RDF_RES + "=\"" + 54 | generator.ontology + "\" />"; 55 | out.println(s); 56 | s = ""; 57 | out.println(s); 58 | } 59 | } -------------------------------------------------------------------------------- /src/main/java/experiments/lubm/generator/DatalogWriter.java: -------------------------------------------------------------------------------- 1 | package experiments.lubm.generator; 2 | 3 | import java.io.*; 4 | 5 | public class DatalogWriter implements Writer { 6 | 7 | final static String PATH = "/home/tr/extern/data/bashlog/lubm/"; 8 | 9 | String path; 10 | 11 | String currentInstance = null; 12 | 13 | PrintWriter writer; 14 | 15 | public DatalogWriter(String path) throws FileNotFoundException { 16 | new File(path).mkdirs(); 17 | this.path = path; 18 | writer = new PrintWriter(path + "all.datalog"); 19 | } 20 | 21 | @Override 22 | public void start() { 23 | new File(path).mkdirs(); 24 | } 25 | 26 | @Override 27 | public void end() { 28 | writer.close(); 29 | } 30 | 31 | @Override 32 | public void startFile(String fileName) { 33 | } 34 | 35 | @Override 36 | public void endFile() { 37 | } 38 | 39 | 40 | @Override 41 | public void startSection(int classType, String id) { 42 | currentInstance = id; 43 | writer.append(Generator.CLASS_TOKEN[classType]).append("(\"").append(id).append("\").").println(); 44 | } 45 | 46 | @Override 47 | public void startAboutSection(int classType, String id) { 48 | currentInstance = id; 49 | writer.append(Generator.CLASS_TOKEN[classType]).append("(\"").append(id).append("\").").println(); 50 | } 51 | 52 | @Override 53 | public void endSection(int classType) { 54 | currentInstance = null; 55 | } 56 | 57 | @Override 58 | public void addProperty(int property, String value, boolean isResource) { 59 | writer// 60 | .append(Generator.PROP_TOKEN[property]).append("(\"").append(currentInstance).append("\", \"").append(value).append("\").").println(); 61 | } 62 | 63 | @Override 64 | public void addProperty(int property, int valueClass, String valueId) { 65 | writer// 66 | .append(Generator.PROP_TOKEN[property]).append("(\"").append(currentInstance).append("\", \"").append(valueId).append("\").").println(); 67 | } 68 | 69 | public static void generate(int univNum, int startIndex, int seed, String path) throws FileNotFoundException { 70 | new Generator().start(univNum, startIndex, seed, new DatalogWriter(path), "dummy"); 71 | } 72 | 73 | public static void main(String[] args) throws FileNotFoundException { 74 | if (args.length == 0) args = new String[] { "1" }; 75 | int univNum = Integer.parseInt(args[0]); 76 | int startIndex = 0; 77 | int seed = 0; 78 | generate(univNum, 0, 0, PATH + "/" + univNum + "/"); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/experiments/lubm/generator/NtlitWriter.java: -------------------------------------------------------------------------------- 1 | package experiments.lubm.generator; 2 | 3 | import java.io.*; 4 | 5 | /** Write LUBM triples for RDFSlice */ 6 | public class NtlitWriter implements Writer { 7 | 8 | final static String PATH = "/home/tr/extern/data/bashlog/lubm/"; 9 | 10 | String path; 11 | 12 | String currentInstance = null; 13 | 14 | PrintWriter writer; 15 | 16 | public NtlitWriter(String path) throws FileNotFoundException { 17 | new File(path).getAbsoluteFile().getParentFile().mkdirs(); 18 | this.path = path; 19 | writer = new PrintWriter(path); 20 | } 21 | 22 | @Override 23 | public void start() { 24 | new File(path).mkdirs(); 25 | } 26 | 27 | @Override 28 | public void end() { 29 | writer.close(); 30 | } 31 | 32 | @Override 33 | public void startFile(String fileName) { 34 | } 35 | 36 | @Override 37 | public void endFile() { 38 | } 39 | 40 | 41 | @Override 42 | public void startSection(int classType, String id) { 43 | currentInstance = id; 44 | writer.append("<").append(id).append("> rdf:type ub:").append(Generator.CLASS_TOKEN[classType]).append(" .").println(); 45 | } 46 | 47 | @Override 48 | public void startAboutSection(int classType, String id) { 49 | startSection(classType, id); 50 | } 51 | 52 | @Override 53 | public void endSection(int classType) { 54 | currentInstance = null; 55 | } 56 | 57 | @Override 58 | public void addProperty(int property, String value, boolean isResource) { 59 | writer.append("<").append(currentInstance).append("> ub:") // 60 | .append(Generator.PROP_TOKEN[property]).append(" "); 61 | if (isResource) { 62 | writer.append("<").append(value).append("> .").println(); 63 | } else { 64 | writer.append("lit:").append(value).append(" .").println(); 65 | } 66 | } 67 | 68 | @Override 69 | public void addProperty(int property, int valueClass, String valueId) { 70 | writer.append("<").append(currentInstance).append("> ub:") // 71 | .append(Generator.PROP_TOKEN[property]).append(" "); 72 | writer.append("<").append(valueId).append("> .").println(); 73 | } 74 | 75 | public static void generate(int univNum, int startIndex, int seed, String path) throws FileNotFoundException { 76 | new Generator().start(univNum, startIndex, seed, new NtlitWriter(path), "dummy"); 77 | } 78 | 79 | public static void main(String[] args) throws FileNotFoundException { 80 | if (args.length == 0) args = new String[] { "1" }; 81 | int univNum = Integer.parseInt(args[0]); 82 | String path = PATH + "/" + univNum + "/" + "all.ntlit"; 83 | if (args.length >= 2) { 84 | path = args[1]; 85 | } 86 | generate(univNum, 0, 0, path); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/experiments/lubm/generator/OwlWriter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * by Yuanbo Guo 3 | * Semantic Web and Agent Technology Lab, CSE Department, Lehigh University, USA 4 | * Copyright (C) 2004 5 | * 6 | * This program is free software; you can redistribute it and/or modify it under 7 | * the terms of the GNU General Public License as published by the Free Software 8 | * Foundation; either version 2 of the License, or (at your option) any later 9 | * version. 10 | * 11 | * This program is distributed in the hope that it will be useful, but WITHOUT 12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | * FOR A PARTICULAR PURPOSE.See the GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License along with 16 | * this program; if not, write to the Free Software Foundation, Inc., 59 Temple 17 | * Place, Suite 330, Boston, MA 02111-1307 USA 18 | */ 19 | 20 | package experiments.lubm.generator; 21 | 22 | public class OwlWriter 23 | extends RdfWriter { 24 | /** abbreviation of OWL namespace */ 25 | private static final String T_OWL_NS = "owl"; 26 | /** prefix of the OWL namespace */ 27 | private static final String T_OWL_PREFIX = T_OWL_NS + ":"; 28 | 29 | /** 30 | * Constructor. 31 | * @param generator The generator object. 32 | */ 33 | public OwlWriter(Generator generator) { 34 | super(generator); 35 | } 36 | 37 | /** 38 | * Writes the header, including namespace declarations and ontology header. 39 | */ 40 | void writeHeader() { 41 | String s; 42 | s = "xmlns:" + T_RDF_NS + 43 | "=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\""; 44 | out.println(s); 45 | s = "xmlns:" + T_RDFS_NS + "=\"http://www.w3.org/2000/01/rdf-schema#\""; 46 | out.println(s); 47 | s = "xmlns:" + T_OWL_NS + "=\"http://www.w3.org/2002/07/owl#\""; 48 | out.println(s); 49 | s = "xmlns:" + T_ONTO_NS + "=\"" + generator.ontology + "#\">"; 50 | out.println(s); 51 | out.println("\n"); 52 | s = "<" + T_OWL_PREFIX + "Ontology " + T_RDF_ABOUT + "=\"\">"; 53 | out.println(s); 54 | s = "<" + T_OWL_PREFIX + "imports " + T_RDF_RES + "=\"" + 55 | generator.ontology + "\" />"; 56 | out.println(s); 57 | s = ""; 58 | out.println(s); 59 | } 60 | } -------------------------------------------------------------------------------- /src/main/java/experiments/lubm/generator/Tsv2Writer.java: -------------------------------------------------------------------------------- 1 | package experiments.lubm.generator; 2 | 3 | import java.io.*; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | @Deprecated 8 | public class Tsv2Writer implements Writer { 9 | 10 | final static String PATH = "/home/tr/extern/data/bashlog/lubm/"; 11 | 12 | String path; 13 | 14 | String currentInstance = null; 15 | 16 | Map fileToWriter = new HashMap<>(); 17 | 18 | public Tsv2Writer(String path) { 19 | new File(path).mkdirs(); 20 | this.path = path; 21 | } 22 | 23 | @Override 24 | public void start() { 25 | new File(path).mkdirs(); 26 | } 27 | 28 | @Override 29 | public void end() { 30 | fileToWriter.forEach((f, pw) -> pw.close()); 31 | } 32 | 33 | @Override 34 | public void startFile(String fileName) { 35 | } 36 | 37 | @Override 38 | public void endFile() { 39 | } 40 | 41 | public PrintWriter writer(String filename) { 42 | return fileToWriter.computeIfAbsent(filename, k -> { 43 | try { 44 | return new PrintWriter(path + k); 45 | } catch (FileNotFoundException e) { 46 | e.printStackTrace(); 47 | } 48 | return null; 49 | }); 50 | } 51 | 52 | @Override 53 | public void startSection(int classType, String id) { 54 | currentInstance = id; 55 | writer(Generator.CLASS_TOKEN[classType]).println(id); 56 | } 57 | 58 | @Override 59 | public void startAboutSection(int classType, String id) { 60 | currentInstance = id; 61 | writer(Generator.CLASS_TOKEN[classType]).println(id); 62 | } 63 | 64 | @Override 65 | public void endSection(int classType) { 66 | currentInstance = null; 67 | } 68 | 69 | @Override 70 | public void addProperty(int property, String value, boolean isResource) { 71 | writer(Generator.PROP_TOKEN[property])// 72 | .append(currentInstance).append("\t").append(value).println(); 73 | } 74 | 75 | @Override 76 | public void addProperty(int property, int valueClass, String valueId) { 77 | writer(Generator.PROP_TOKEN[property])// 78 | .append(currentInstance).append("\t").append(valueId).println(); 79 | } 80 | 81 | public static void generate(int univNum, int startIndex, int seed, String path) { 82 | new Generator().start(univNum, startIndex, seed, new Tsv2Writer(path), "dummy"); 83 | } 84 | 85 | public static void main(String[] args) { 86 | int univNum = 1; 87 | int startIndex = 0; 88 | int seed = 0; 89 | generate(1, 0, 0, PATH + "/" + univNum + "/"); 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/experiments/lubm/generator/Tsv3Writer.java: -------------------------------------------------------------------------------- 1 | package experiments.lubm.generator; 2 | 3 | import java.io.*; 4 | 5 | public class Tsv3Writer implements Writer { 6 | 7 | final static String PATH = "/home/tr/extern/data/bashlog/lubm/"; 8 | 9 | String path; 10 | 11 | String currentInstance = null; 12 | 13 | PrintWriter writer; 14 | 15 | public Tsv3Writer(String path) throws FileNotFoundException { 16 | new File(path).mkdirs(); 17 | this.path = path; 18 | writer = new PrintWriter(path + "all"); 19 | } 20 | 21 | @Override 22 | public void start() { 23 | new File(path).mkdirs(); 24 | } 25 | 26 | @Override 27 | public void end() { 28 | writer.close(); 29 | } 30 | 31 | @Override 32 | public void startFile(String fileName) { 33 | } 34 | 35 | @Override 36 | public void endFile() { 37 | } 38 | 39 | 40 | @Override 41 | public void startSection(int classType, String id) { 42 | currentInstance = id; 43 | writer.append(id).append("\trdf:type\t").append(Generator.CLASS_TOKEN[classType]).println(); 44 | } 45 | 46 | @Override 47 | public void startAboutSection(int classType, String id) { 48 | currentInstance = id; 49 | writer.append(id).append("\trdf:type\t").append(Generator.CLASS_TOKEN[classType]).println(); 50 | } 51 | 52 | @Override 53 | public void endSection(int classType) { 54 | currentInstance = null; 55 | } 56 | 57 | @Override 58 | public void addProperty(int property, String value, boolean isResource) { 59 | writer// 60 | .append(currentInstance).append("\t").append(Generator.PROP_TOKEN[property]).append("\t").append(value).println(); 61 | } 62 | 63 | @Override 64 | public void addProperty(int property, int valueClass, String valueId) { 65 | writer// 66 | .append(currentInstance).append("\t").append(Generator.PROP_TOKEN[property]).append("\t").append(valueId).println(); 67 | } 68 | 69 | public static void generate(int univNum, int startIndex, int seed, String path) throws FileNotFoundException { 70 | new Generator().start(univNum, startIndex, seed, new Tsv3Writer(path), "dummy"); 71 | } 72 | 73 | public static void main(String[] args) throws FileNotFoundException { 74 | if (args.length == 0) args = new String[] { "1" }; 75 | int univNum = Integer.parseInt(args[0]); 76 | int startIndex = 0; 77 | int seed = 0; 78 | generate(univNum, 0, 0, PATH + "/" + univNum + "/"); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/experiments/lubm/generator/TtlWriter.java: -------------------------------------------------------------------------------- 1 | package experiments.lubm.generator; 2 | 3 | import java.io.*; 4 | 5 | public class TtlWriter implements Writer { 6 | 7 | final static String PATH = "/home/tr/extern/data/bashlog/lubm/"; 8 | 9 | String path; 10 | 11 | String currentInstance = null; 12 | 13 | PrintWriter writer; 14 | 15 | public TtlWriter(String path) throws FileNotFoundException { 16 | new File(path).getAbsoluteFile().getParentFile().mkdirs(); 17 | this.path = path; 18 | writer = new PrintWriter(path); 19 | writer.write("@base .\n"); 20 | writer.write("@prefix rdf: .\n"); 21 | writer.write("@prefix ub: .\n"); 22 | } 23 | 24 | @Override 25 | public void start() { 26 | new File(path).mkdirs(); 27 | } 28 | 29 | @Override 30 | public void end() { 31 | writer.close(); 32 | } 33 | 34 | @Override 35 | public void startFile(String fileName) { 36 | } 37 | 38 | @Override 39 | public void endFile() { 40 | } 41 | 42 | 43 | @Override 44 | public void startSection(int classType, String id) { 45 | currentInstance = id; 46 | writer.append("<").append(id).append(">\trdf:type\tub:").append(Generator.CLASS_TOKEN[classType]).append("\t.").println(); 47 | } 48 | 49 | @Override 50 | public void startAboutSection(int classType, String id) { 51 | startSection(classType, id); 52 | } 53 | 54 | @Override 55 | public void endSection(int classType) { 56 | currentInstance = null; 57 | } 58 | 59 | @Override 60 | public void addProperty(int property, String value, boolean isResource) { 61 | writer.append("<").append(currentInstance).append(">\tub:") // 62 | .append(Generator.PROP_TOKEN[property]).append("\t"); 63 | if (isResource) { 64 | writer.append("<").append(value).append(">\t.").println(); 65 | } else { 66 | writer.append("\"").append(value).append("\"\t.").println(); 67 | } 68 | } 69 | 70 | @Override 71 | public void addProperty(int property, int valueClass, String valueId) { 72 | writer.append("<").append(currentInstance).append(">\tub:") // 73 | .append(Generator.PROP_TOKEN[property]).append("\t"); 74 | //writer.append("\"").append(valueId).append("\".").println(); 75 | writer.append("<").append(valueId).append(">.").println(); 76 | } 77 | 78 | public static void generate(int univNum, int startIndex, int seed, String path) throws FileNotFoundException { 79 | new Generator().start(univNum, startIndex, seed, new TtlWriter(path), "dummy"); 80 | } 81 | 82 | public static void main(String[] args) throws FileNotFoundException { 83 | if (args.length == 0) args = new String[] { "1" }; 84 | int univNum = Integer.parseInt(args[0]); 85 | String path = PATH + "/" + univNum + "/" + "all.ttl"; 86 | if (args.length >= 2) { 87 | path = args[1]; 88 | } 89 | generate(univNum, 0, 0, path); 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/experiments/lubm/generator/Writer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * by Yuanbo Guo 3 | * Semantic Web and Agent Technology Lab, CSE Department, Lehigh University, USA 4 | * Copyright (C) 2004 5 | * 6 | * This program is free software; you can redistribute it and/or modify it under 7 | * the terms of the GNU General Public License as published by the Free Software 8 | * Foundation; either version 2 of the License, or (at your option) any later 9 | * version. 10 | * 11 | * This program is distributed in the hope that it will be useful, but WITHOUT 12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | * FOR A PARTICULAR PURPOSE.See the GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License along with 16 | * this program; if not, write to the Free Software Foundation, Inc., 59 Temple 17 | * Place, Suite 330, Boston, MA 02111-1307 USA 18 | */ 19 | 20 | package experiments.lubm.generator; 21 | 22 | public interface Writer { 23 | /** 24 | * Called when starting data generation. 25 | */ 26 | public void start(); 27 | 28 | /** 29 | * Called when finish data generation. 30 | */ 31 | public void end(); 32 | 33 | /** 34 | * Starts file writing. 35 | * @param fileName File name. 36 | */ 37 | public void startFile(String fileName); 38 | 39 | /** 40 | * Finishes the current file. 41 | */ 42 | public void endFile(); 43 | 44 | /** 45 | * Starts a section for the specified instance. 46 | * @param classType Type of the instance. 47 | * @param id Id of the instance. 48 | */ 49 | public void startSection(int classType, String id); 50 | 51 | /** 52 | * Starts a section for the specified instance identified by an rdf:about attribute. 53 | * @param classType Type of the instance. 54 | * @param id Id of the instance. 55 | */ 56 | public void startAboutSection(int classType, String id); 57 | 58 | /** 59 | * Finishes the current section. 60 | * @param classType Type of the current instance. 61 | */ 62 | public void endSection(int classType); 63 | 64 | /** 65 | * Adds the specified property statement for the current element. 66 | * @param property Type of the property. 67 | * @param value Property value. 68 | * @param isResource Indicates if the property value is an rdf resource (True), 69 | * or it is literal (False). 70 | */ 71 | public void addProperty(int property, String value, boolean isResource); 72 | 73 | /** 74 | * Adds a property statement for the current element whose value is an individual. 75 | * @param property Type of the property. 76 | * @param valueClass Type of the individual. 77 | * @param valueId Id of the individual. 78 | */ 79 | public void addProperty(int property, int valueClass, String valueId); 80 | } -------------------------------------------------------------------------------- /src/main/java/experiments/lubm/readme.txt: -------------------------------------------------------------------------------- 1 | #################################################### 2 | Univ-Bench Artificial Data Generator (UBA) 3 | Version 1.7 4 | The Semantic Web and Agent Technologies (SWAT) Lab 5 | CSE Department, Lehigh University 6 | #################################################### 7 | 8 | ================== 9 | USAGES 10 | ================== 11 | 12 | command: 13 | edu.lehigh.swat.bench.uba.Generator 14 | [-univ ] 15 | [-index ] 16 | [-seed ] 17 | [-daml] 18 | -onto 19 | 20 | options: 21 | -univ number of universities to generate; 1 by default 22 | -index starting index of the universities; 0 by default 23 | -seed seed used for random data generation; 0 by default 24 | -daml generate DAML+OIL data; OWL data by default 25 | -onto url of the univ-bench ontology 26 | 27 | - The package's path should be on CLASSPATH. 28 | 29 | ================== 30 | Contact 31 | ================== 32 | 33 | Yuanbo Guo yug2@lehigh.edu 34 | 35 | For more information about the benchmark, visit its homepage http://www.lehigh.edu/~yug2/Research/SemanticWeb/LUBM/LUBM.htm. 36 | -------------------------------------------------------------------------------- /src/main/java/rdf/MainOwl.java: -------------------------------------------------------------------------------- 1 | package rdf; 2 | 3 | import java.nio.file.Files; 4 | import java.nio.file.Paths; 5 | import java.util.Collections; 6 | 7 | import org.semanticweb.owlapi.apibinding.OWLManager; 8 | import org.semanticweb.owlapi.model.IRI; 9 | import org.semanticweb.owlapi.model.OWLOntologyManager; 10 | 11 | import common.parser.Program; 12 | 13 | public class MainOwl { 14 | public static void main(String[] args) throws Exception { 15 | RDFTupleSerializer serializer = new RDFSpecificTuplesSerializer(Collections.singletonMap( 16 | "http://swat.cse.lehigh.edu/onto/univ-bench.owl#", "" 17 | )); 18 | 19 | OWLOntologyManager ontologyManager = OWLManager.createOWLOntologyManager(); 20 | IRI ontologyIRI = IRI.create("http://swat.cse.lehigh.edu/onto/univ-bench.owl"); 21 | ontologyManager.loadOntologyFromOntologyDocument(ontologyIRI); 22 | 23 | OntologyConverter owlConverter = new OntologyConverter(serializer); 24 | Program ontologyProgram = owlConverter.convert(ontologyManager.getOntology(ontologyIRI)); 25 | 26 | SPARQLConverter sparqlConverter = new SPARQLConverter(serializer); 27 | int i = 1; 28 | for(String query : new String(Files.readAllBytes(Paths.get( "data/lubm/queries.sparql"))).split("\n\n")) { 29 | Program program = sparqlConverter.convert(query, "query" + (i++)); 30 | if(i == 8) { 31 | System.out.println(query); 32 | System.out.println(program); 33 | return; 34 | } 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/rdf/RDFSpecificTuplesSerializer.java: -------------------------------------------------------------------------------- 1 | package rdf; 2 | 3 | import java.util.Map; 4 | 5 | import org.apache.commons.rdf.api.*; 6 | 7 | import common.parser.*; 8 | 9 | /** 10 | * Serializes RDF triples in tuples like predicate(subject, object) or type(subject) 11 | */ 12 | public class RDFSpecificTuplesSerializer implements RDFTupleSerializer { 13 | 14 | 15 | private final Map prefixes; 16 | private final String rdfType; 17 | 18 | public RDFSpecificTuplesSerializer(Map prefixes) { 19 | this.prefixes = prefixes; 20 | rdfType = shortened("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"); 21 | 22 | } 23 | 24 | @Override 25 | public CompoundTerm convertTriple(Term subject, Term predicate, Term object) { 26 | if(predicate instanceof Constant) { 27 | Object predicateValue = ((Constant) predicate).getValue(); 28 | if(rdfType.equals(predicateValue)) { 29 | if(object instanceof Constant) { 30 | Object value = ((Constant) object).getValue(); 31 | if(value instanceof String) { 32 | return new CompoundTerm((String) value, subject); 33 | } 34 | } 35 | throw new IllegalArgumentException("The range of rdf:type is owl:Class and should be bounded: <" + subject + ", " + predicate + ", " + object + ">"); 36 | } else if(predicateValue instanceof String) { 37 | return new CompoundTerm((String) predicateValue, subject, object); 38 | } 39 | } 40 | throw new UnsupportedOperationException("The predicate of the triple should be a string constant: " + subject + " " + predicate + " " + object); 41 | } 42 | 43 | public Term convertTerm(RDFTerm term) { 44 | if (term instanceof Literal) { 45 | return new Constant<>(((Literal) term).getLexicalForm()); 46 | } else if (term instanceof BlankNode) { 47 | return new Constant<>(((BlankNode) term).uniqueReference()); 48 | } else if (term instanceof IRI) { 49 | return new Constant<>(shortened(((IRI) term).getIRIString())); 50 | } else { 51 | throw new IllegalArgumentException("Not supported RDF term: " + term); 52 | } 53 | } 54 | 55 | private String shortened(String IRI) { 56 | for(Map.Entry prefix : prefixes.entrySet()) { 57 | if(IRI.startsWith(prefix.getKey())) { 58 | return prefix.getValue() + IRI.substring(prefix.getKey().length()); 59 | } 60 | } 61 | return "<" + IRI + ">"; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/rdf/RDFTripleTupleSerializer.java: -------------------------------------------------------------------------------- 1 | package rdf; 2 | 3 | import java.util.Collections; 4 | import java.util.Map; 5 | 6 | import org.apache.commons.rdf.api.*; 7 | 8 | import common.parser.*; 9 | 10 | /** 11 | * Serializes RDF triples in tuples like tripleTupleName(subject, predicate, object) with optional IRI shortening 12 | */ 13 | public class RDFTripleTupleSerializer implements RDFTupleSerializer { 14 | 15 | private final String tripleTupleName; 16 | private final Map prefixes; 17 | 18 | public RDFTripleTupleSerializer(String tripleTupleName, Map prefixes) { 19 | this.tripleTupleName = tripleTupleName; 20 | this.prefixes = prefixes; 21 | } 22 | 23 | public RDFTripleTupleSerializer(String tripleTupleName) { 24 | this(tripleTupleName, Collections.emptyMap()); 25 | } 26 | 27 | public CompoundTerm convertTriple(Term subject, Term predicate, Term object) { 28 | return new CompoundTerm(tripleTupleName, subject, predicate, object); 29 | } 30 | 31 | public Term convertTerm(RDFTerm term) { 32 | if (term instanceof Literal) { 33 | return new Constant<>(((Literal) term).getLexicalForm()); 34 | } else if (term instanceof BlankNode) { 35 | return new Constant<>(((BlankNode) term).uniqueReference()); 36 | } else if (term instanceof IRI) { 37 | return new Constant<>(shortened(((IRI) term).getIRIString())); 38 | } else { 39 | throw new IllegalArgumentException("Not supported RDF term: " + term); 40 | } 41 | } 42 | 43 | private String shortened(String IRI) { 44 | for(Map.Entry prefix : prefixes.entrySet()) { 45 | if(IRI.startsWith(prefix.getKey())) { 46 | return prefix.getValue() + IRI.substring(prefix.getKey().length()); 47 | } 48 | } 49 | return "<" + IRI + ">"; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/rdf/RDFTupleSerializer.java: -------------------------------------------------------------------------------- 1 | package rdf; 2 | 3 | import org.apache.commons.rdf.api.*; 4 | 5 | import common.parser.CompoundTerm; 6 | import common.parser.Term; 7 | 8 | public interface RDFTupleSerializer { 9 | CompoundTerm convertTriple(Term subject, Term predicate, Term object); 10 | 11 | default CompoundTerm convertTriple(Term subject, IRI predicate, Term object) { 12 | return convertTriple(subject, convertTerm(predicate), object); 13 | } 14 | 15 | default CompoundTerm convertTriple(Term subject, IRI predicate, RDFTerm object) { 16 | return convertTriple(subject, predicate, convertTerm(object)); 17 | } 18 | 19 | default CompoundTerm convertTriple(BlankNodeOrIRI subject, IRI predicate, Term object) { 20 | return convertTriple(convertTerm(subject), predicate, object); 21 | } 22 | 23 | default CompoundTerm convertTriple(BlankNodeOrIRI subject, IRI predicate, RDFTerm object) { 24 | return convertTriple(convertTerm(subject), predicate, convertTerm(object)); 25 | } 26 | 27 | Term convertTerm(RDFTerm term); 28 | } 29 | -------------------------------------------------------------------------------- /src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 36 | 37 | -------------------------------------------------------------------------------- /src/test/java/bashlog/BashlogIntegrationTests.java: -------------------------------------------------------------------------------- 1 | package bashlog; 2 | 3 | public class BashlogIntegrationTests extends common.IntegrationTests { 4 | 5 | public BashlogIntegrationTests() { 6 | super(new BashlogEvaluator(".", "/tmp/bashlog-tests/", true)); 7 | } 8 | } -------------------------------------------------------------------------------- /src/test/java/bashlog/BashlogLUBMTest.java: -------------------------------------------------------------------------------- 1 | package bashlog; 2 | 3 | import org.junit.Rule; 4 | import org.junit.rules.Timeout; 5 | 6 | import common.Evaluator; 7 | 8 | public class BashlogLUBMTest extends common.LUBMTest { 9 | 10 | @Override 11 | public Evaluator evaluator() { 12 | return new BashlogEvaluator(".", "/tmp/bashlog-test/", false); 13 | } 14 | 15 | @Rule 16 | public Timeout globalTimeout = Timeout.seconds(20); // limit execution time for a test */ 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/test/java/bashlog/FileTest.java: -------------------------------------------------------------------------------- 1 | package bashlog; 2 | 3 | import java.io.File; 4 | import java.nio.file.Path; 5 | import java.util.*; 6 | 7 | import org.junit.Test; 8 | import org.junit.runner.RunWith; 9 | import org.junit.runners.Parameterized; 10 | import org.junit.runners.Parameterized.Parameters; 11 | 12 | import common.*; 13 | import common.parser.Program; 14 | 15 | @RunWith(Parameterized.class) 16 | public class FileTest { 17 | 18 | @Parameters(name = "{0}") 19 | public static Collection getFiles() { 20 | Collection params = new ArrayList(); 21 | int i = 0; 22 | for (File directory : new File("src/test/resources").listFiles()) { 23 | Object[] arr = new Object[] { directory, i++ }; 24 | params.add(arr); 25 | } 26 | return params; 27 | } 28 | 29 | private File directory; 30 | 31 | private Integer index; 32 | 33 | public FileTest(File directory, Integer index) { 34 | this.directory = directory; 35 | this.index = index; 36 | } 37 | 38 | @Test 39 | public void evaluateProgramInTestDir() throws Exception { 40 | // load test file 41 | 42 | Path programFile = directory.toPath().resolve("main.txt"); 43 | Set features = BashlogCompiler.BASHLOG_PARSER_FEATURES; 44 | Program p = Program.loadFile(programFile.toString(), features); 45 | 46 | String queryPred = "main"; 47 | queryPred = p.searchRelation(queryPred); 48 | if (queryPred == null || queryPred.trim().isEmpty()) { 49 | queryPred = p.rules().get(p.rules().size() - 1).head.getRelation(); 50 | } 51 | 52 | SimpleFactsSet facts = new SimpleFactsSet(); 53 | FactsSet evaluate = new BashlogEvaluator(directory.toString(), "/tmp/bashlog-tests/").evaluate(p, facts, new HashSet<>(Arrays.asList(queryPred))); 54 | 55 | try (Check check = new Check()) { 56 | File expectedFile = directory.toPath().resolve("expected.txt").toFile(); 57 | try (TSVReader expected = new TSVReader(expectedFile)) { 58 | for (List line : expected) { 59 | check.onceList(line); 60 | } 61 | } 62 | 63 | evaluate.getByRelation(queryPred).forEach(check::apply); 64 | } 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/test/java/common/Check.java: -------------------------------------------------------------------------------- 1 | package common; 2 | 3 | import java.util.*; 4 | import java.util.function.BiConsumer; 5 | import java.util.stream.Collectors; 6 | 7 | import org.junit.Assert; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | public class Check implements AutoCloseable { 12 | 13 | public final static Logger log = LoggerFactory.getLogger(Check.class); 14 | 15 | Map, Integer> expected = new HashMap<>(), expectedParts = new HashMap<>(); 16 | 17 | Map, Integer> unexpected = new HashMap<>(); 18 | 19 | boolean debug = true; 20 | 21 | boolean ignoreUnexpected = false; 22 | 23 | boolean ignoreTooOften = false; 24 | 25 | public void ignoreUnexpected() { 26 | this.ignoreUnexpected = true; 27 | } 28 | 29 | public void ignoreTooOften() { 30 | this.ignoreTooOften = true; 31 | } 32 | 33 | public boolean apply(Object... vals) { 34 | return apply(Arrays.stream(vals).collect(Collectors.toList())); 35 | } 36 | 37 | public boolean apply(List vals) { 38 | log.debug("got " + vals); 39 | Integer count = expected.computeIfPresent(vals, (l, c) -> --c); 40 | if (count == null) { 41 | next_exp: for (List os : expectedParts.keySet()) { 42 | for (int i = 0; i < os.size(); i++) { 43 | if (os.get(i) != null) { 44 | if (!Objects.equals(os.get(i), vals.get(i))) { 45 | continue next_exp; 46 | } 47 | } 48 | } 49 | count = expectedParts.computeIfPresent(os, (l, c) -> --c); 50 | } 51 | } 52 | if (count == null) { 53 | unexpected.merge(vals, 1, (a, b) -> a + b); 54 | } 55 | 56 | return true; 57 | } 58 | 59 | @Override 60 | public void close() throws Exception { 61 | log.debug("closing worker, unexpected " + unexpected.size()); 62 | boolean fail[] = { false }; 63 | int[] correct = { 0 }; 64 | 65 | List reasons = new ArrayList<>(); 66 | 67 | try { 68 | int count[] = { 0 }; 69 | BiConsumer, Integer> f = (l, c) -> { 70 | if (c > 0) { 71 | String reason = "missing (" + c + "x): " + l; 72 | reasons.add(reason); 73 | log.error(reason); 74 | fail[0] = true; 75 | if (count[0]++ > 10) return; 76 | } 77 | if (c < 0 && !ignoreTooOften) { 78 | String reason = "too often (" + c + "x): " + l; 79 | reasons.add(reason); 80 | log.error(reason); 81 | fail[0] = true; 82 | if (count[0]++ > 10) return; 83 | } 84 | if (c == 0) { 85 | correct[0]++; 86 | } 87 | }; 88 | 89 | expected.forEach(f); 90 | expectedParts.forEach(f); 91 | if (!ignoreUnexpected) { 92 | unexpected.forEach((l, c) -> { 93 | String reason = "unexpected (" + c + "x): " + l; 94 | reasons.add(reason); 95 | log.error(reason); 96 | fail[0] = true; 97 | if (count[0]++ > 10) return; 98 | }); 99 | } 100 | } finally { 101 | if (fail[0]) { 102 | log.info("found at least " + correct[0] + " correct entries"); 103 | List firstReasons = reasons.subList(0, Math.min(3, reasons.size())); 104 | String message = "unexpected output, example problems:\n"; 105 | message += firstReasons.stream().collect(Collectors.joining("\n")); 106 | message += "\ncheck console output for complete list"; 107 | Assert.fail(message); 108 | } 109 | } 110 | } 111 | 112 | public void onceList(List objects) { 113 | timesList(1, objects); 114 | } 115 | 116 | public void once(Object... objects) { 117 | times(1, objects); 118 | } 119 | 120 | @SuppressWarnings("unchecked") 121 | public void timesList(int times, List objects) { 122 | for (Object o : objects) { 123 | if (o == null) { 124 | expectedParts.put((List) objects, times); 125 | return; 126 | } 127 | } 128 | expected.put((List) objects, times); 129 | } 130 | 131 | public void times(int times, Object... objects) { 132 | timesList(times, Arrays.asList(objects)); 133 | } 134 | 135 | 136 | } 137 | -------------------------------------------------------------------------------- /src/test/java/common/DownloadTools.java: -------------------------------------------------------------------------------- 1 | package common; 2 | 3 | import java.io.*; 4 | import java.net.URL; 5 | import java.nio.channels.Channels; 6 | import java.nio.channels.ReadableByteChannel; 7 | import java.util.*; 8 | import java.util.zip.ZipEntry; 9 | import java.util.zip.ZipInputStream; 10 | 11 | import org.apache.commons.compress.archivers.tar.TarArchiveEntry; 12 | import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; 13 | import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; 14 | import org.slf4j.Logger; 15 | import org.slf4j.LoggerFactory; 16 | 17 | public class DownloadTools { 18 | 19 | public final static Logger log = LoggerFactory.getLogger(DownloadTools.class); 20 | 21 | public static void download(String url, String path) throws IOException { 22 | log.info("downloading " + url); 23 | URL website = new URL(url); 24 | ReadableByteChannel rbc = Channels.newChannel(website.openStream()); 25 | if (path.endsWith("/")) { 26 | new File(path).mkdirs(); 27 | } 28 | if (new File(path).isDirectory()) { 29 | path += website.getFile().replaceAll(".*/", ""); 30 | } 31 | try (FileOutputStream fos = new FileOutputStream(path)) { 32 | fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); 33 | } 34 | } 35 | 36 | // from http://java-tweets.blogspot.fr/2012/07/untar-targz-file-with-apache-commons.html 37 | static void untar(String archive, String dstPath, Collection list) throws IOException { 38 | final int BUFFER = 65536; 39 | // TODO Auto-generated method stub 40 | FileInputStream fin = new FileInputStream(archive); 41 | BufferedInputStream in = new BufferedInputStream(fin); 42 | GzipCompressorInputStream gzIn = new GzipCompressorInputStream(in); 43 | TarArchiveInputStream tarIn = new TarArchiveInputStream(gzIn); 44 | 45 | TarArchiveEntry entry = null; 46 | 47 | /** Read the tar entries using the getNextEntry method **/ 48 | while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) { 49 | if (list != null && !list.contains(entry.getName())) { 50 | continue; 51 | } 52 | 53 | /** If the entry is a directory, create the directory. **/ 54 | if (entry.isDirectory()) { 55 | File f = new File(dstPath + entry.getName()); 56 | f.mkdirs(); 57 | } else { 58 | int count; 59 | byte data[] = new byte[BUFFER]; 60 | File f = new File(dstPath + entry.getName()); 61 | f.getParentFile().mkdirs(); 62 | 63 | FileOutputStream fos = new FileOutputStream(dstPath + entry.getName()); 64 | BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER); 65 | while ((count = tarIn.read(data, 0, BUFFER)) != -1) { 66 | dest.write(data, 0, count); 67 | } 68 | dest.close(); 69 | } 70 | } 71 | 72 | /** Close the input stream **/ 73 | 74 | tarIn.close(); 75 | } 76 | 77 | // from https://www.mkyong.com/java/how-to-decompress-files-from-a-zip-file 78 | public static void unzip(String zipFile, String outputFolder) { 79 | byte[] buffer = new byte[1024]; 80 | try { 81 | //create output directory is not exists 82 | File folder = new File(outputFolder); 83 | if (!folder.exists()) { 84 | folder.mkdir(); 85 | } 86 | 87 | //get the zip file content 88 | ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFile)); 89 | ZipEntry ze = zis.getNextEntry(); 90 | while (ze != null) { 91 | String fileName = ze.getName(); 92 | File newFile = new File(outputFolder + File.separator + fileName); 93 | 94 | //create all non exists folders 95 | //else you will hit FileNotFoundException for compressed folder 96 | new File(newFile.getParent()).mkdirs(); 97 | 98 | FileOutputStream fos = new FileOutputStream(newFile); 99 | int len; 100 | while ((len = zis.read(buffer)) > 0) { 101 | fos.write(buffer, 0, len); 102 | } 103 | 104 | fos.close(); 105 | ze = zis.getNextEntry(); 106 | } 107 | 108 | zis.closeEntry(); 109 | zis.close(); 110 | } catch (IOException ex) { 111 | ex.printStackTrace(); 112 | } 113 | } 114 | 115 | public static int[] toIntArray(List intList) { 116 | return intList.stream().mapToInt(Integer::intValue).toArray(); 117 | } 118 | 119 | public static int[] array(int length, int initialValue) { 120 | int[] array = new int[length]; 121 | Arrays.fill(array, initialValue); 122 | return array; 123 | } 124 | 125 | } 126 | -------------------------------------------------------------------------------- /src/test/java/common/parser/ProgramTest.java: -------------------------------------------------------------------------------- 1 | package common.parser; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | public class ProgramTest { 7 | 8 | @Test 9 | public void testHasAncestor() { 10 | Program program = Program.read(new ParserReader( 11 | "a(X,Z) :- b(X,Y), in(Y,Z). b(X,Z) :- b(X,Y), in(Y,Z). b(X,Z) :- a(X,Y), in(Y,Z)." 12 | )); 13 | Assert.assertFalse(program.hasAncestor("in/2", "a/2")); 14 | Assert.assertTrue(program.hasAncestor("a/2", "b/2")); 15 | Assert.assertTrue(program.hasAncestor("b/2", "b/2")); 16 | Assert.assertTrue(program.hasAncestor("b/2", "a/2")); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/test/java/sqllog/IntegrationTests.java: -------------------------------------------------------------------------------- 1 | package sqllog; 2 | 3 | import java.util.Collections; 4 | 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | import common.parser.ParserReader; 9 | import common.parser.Program; 10 | 11 | public class IntegrationTests { 12 | 13 | @Test 14 | public void testSimple() { 15 | Program program = Program.read(new ParserReader( 16 | "sibling(X,Y) :- parent(X,Z), parent(Y,Z). bad(X,X) :- parent(X,X). bobParent(X) :- parent(\"bob\", X)." 17 | )); 18 | 19 | Assert.assertEquals( 20 | "SELECT T1.C0, T2.C0 FROM parent AS T1, parent AS T2 WHERE T1.C1 = T2.C1", 21 | (new SqllogCompiler()).compile(program, Collections.singleton("parent/2"), "sibling/2") 22 | ); 23 | Assert.assertEquals( 24 | "SELECT T1.C1 FROM parent AS T1 WHERE T1.C0 = 'bob'", 25 | (new SqllogCompiler()).compile(program, Collections.singleton("parent/2"), "bobParent/1") 26 | ); 27 | } 28 | 29 | @Test 30 | public void testLinearClosure() { 31 | Program program = Program.read(new ParserReader( 32 | "ancestor(X,Y) :- parent(X,Y). ancestor(X,Z) :- ancestor(X,Y), parent(Y,Z)." 33 | )); 34 | 35 | Assert.assertEquals( 36 | "WITH RECURSIVE T1(C0, C1) AS ((SELECT T2.C0, T2.C1 FROM parent AS T2) UNION ALL (SELECT T3.C0, T4.C1 FROM T1 AS T3, parent AS T4 WHERE T3.C1 = T4.C0)) SELECT T5.C0, T5.C1 FROM T1 AS T5", 37 | (new SqllogCompiler()).compile(program, Collections.singleton("parent/2"), "ancestor/2") 38 | ); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/test/resources/test1-extern/expected.txt: -------------------------------------------------------------------------------- 1 | A B 2 | B C 3 | A C 4 | -------------------------------------------------------------------------------- /src/test/resources/test1-extern/input-data.txt: -------------------------------------------------------------------------------- 1 | A B 2 | B C 3 | -------------------------------------------------------------------------------- /src/test/resources/test1-extern/main.txt: -------------------------------------------------------------------------------- 1 | input(X,Y) :~ cat input-data.txt 2 | 3 | p1(X,Y) :- input(X,Y). 4 | p2(X,Y) :- input(X,Y). 5 | 6 | p2(X,Y) :- p2(X,Z), p2(Z,Y). 7 | 8 | result(X,Y) :- p1(X,Y). 9 | result(X,Y) :- p2(X,Y). 10 | -------------------------------------------------------------------------------- /src/test/resources/test1-intern/expected.txt: -------------------------------------------------------------------------------- 1 | A B 2 | B C 3 | A C 4 | -------------------------------------------------------------------------------- /src/test/resources/test1-intern/main.txt: -------------------------------------------------------------------------------- 1 | input("A", "B"). 2 | input("B", "C"). 3 | 4 | p1(X,Y) :- input(X,Y). 5 | p2(X,Y) :- input(X,Y). 6 | 7 | p2(X,Y) :- p2(X,Z), p2(Z,Y). 8 | 9 | result(X,Y) :- p1(X,Y). 10 | result(X,Y) :- p2(X,Y). 11 | --------------------------------------------------------------------------------