├── snap ├── Version.saved ├── snap-core │ ├── alg.cpp │ ├── stdafx.h │ ├── graph.txt │ ├── doc │ │ ├── util.h.txt │ │ ├── gviz.h.txt │ │ ├── flow.h.txt │ │ ├── gbase.h.txt │ │ ├── mmnet.h.txt │ │ ├── bignet.h.txt │ │ ├── bfsdfs.h.txt │ │ └── Snap.h.txt │ ├── numpy.h │ ├── coreper.h │ ├── flow.h │ ├── Makefile │ ├── numpy.cpp │ ├── casc.h │ ├── triad.cpp │ ├── gviz.cpp │ ├── gbase.cpp │ ├── Snap.cpp │ ├── gviz.h │ ├── Snap.h │ └── anf.cpp ├── glib-core │ ├── stdafx.h │ ├── doc │ │ ├── bd.h.txt │ │ ├── ss.h.txt │ │ └── ssmp.h.txt │ └── wch.cpp ├── Makefile ├── Credits.txt └── Makefile.config ├── src ├── trident │ ├── utils │ │ └── parallel.cpp │ ├── java │ │ ├── karmaresearch │ │ │ └── trident │ │ │ │ └── Trident.class │ │ └── native │ │ │ └── karmaresearch_trident_Trident.h │ ├── iterators │ │ └── compositetermitr.cpp │ ├── ml │ │ ├── pairwiselearner.cpp │ │ ├── graddebug.cpp │ │ └── tester.cpp │ ├── kb │ │ └── partial.cpp │ ├── binarytables │ │ └── newrowtableinserter.cpp │ └── tree │ │ └── treeitr.cpp └── snap │ ├── undirected.cpp │ └── analytics.cpp ├── scripts └── docker │ ├── Dockerfile │ └── update_and_make.sh ├── win64 ├── trident │ ├── trident.vcxproj.user │ └── trident.vcxproj.filters ├── trident-core │ └── trident-core.vcxproj.user └── trident-sparql │ └── trident-sparql.vcxproj.user ├── tests ├── test_json.cpp ├── test_httpserver.cpp ├── test_inv_dictionary_tree.cpp ├── test_ppr.cpp ├── ontotranse2trident.py ├── test_parallel_for.cpp ├── test_lz4io.cpp ├── test_graph.cpp ├── test_insert_perm.cpp ├── test_insert4.cpp ├── test_readdictfiles.cpp ├── test_mergesort.cpp ├── test_httpclient.cpp ├── test_insertdictionary.cpp ├── cmpoutput.py ├── test_insertlarge.cpp ├── test_index.cpp ├── test_transe3.cpp ├── test_sorting.cpp ├── test_insert_tree.cpp ├── test_hashnumbers.cpp ├── test_map.cpp ├── test_lru.cpp ├── test_stxxl.cpp ├── test_io.cpp ├── test_insert8.cpp ├── test_intermediatetripledata.cpp ├── test_tree.cpp ├── test_number_flags.cpp └── test_treeitr.cpp ├── .gitignore ├── include ├── snap │ ├── readers.h │ └── tedge.h ├── trident │ ├── ml │ │ ├── hole.h │ │ ├── transe.h │ │ ├── feedback.h │ │ ├── pairwiselearner.h │ │ ├── graddebug.h │ │ ├── distmultester.h │ │ ├── transetester.h │ │ └── transebinarytester.h │ ├── sparql │ │ ├── sparql.h │ │ ├── plan.h │ │ └── resultprinter.h │ ├── iterators │ │ ├── tupleiterators.h │ │ └── emptyitr.h │ ├── kb │ │ ├── memoryopt.h │ │ └── statistics.h │ ├── tree │ │ ├── treeitr.h │ │ └── flattreeitr.h │ ├── utils │ │ └── fft.h │ ├── binarytables │ │ ├── newrowtableinserter.h │ │ ├── newtable.h │ │ ├── rowtableinserter.h │ │ ├── newclustertableinserter.h │ │ ├── columntableinserter.h │ │ └── newcolumntableinserter.h │ ├── tests │ │ └── timings.h │ ├── server │ │ └── server.h │ └── model │ │ ├── term.h │ │ └── tuple.h └── python │ └── trident.h ├── rdf3x ├── include │ ├── infra │ │ ├── Config.hpp │ │ ├── osdep │ │ │ ├── Thread.hpp │ │ │ ├── MemoryMappedFile.hpp │ │ │ ├── Mutex.hpp │ │ │ ├── Event.hpp │ │ │ ├── GrowableMappedFile.hpp │ │ │ ├── Timestamp.hpp │ │ │ └── Latch.hpp │ │ └── util │ │ │ └── Type.hpp │ ├── rts │ │ ├── operator │ │ │ ├── ValuesScan.hpp │ │ │ ├── EmptyScan.hpp │ │ │ ├── SingletonScan.hpp │ │ │ ├── NestedLoopJoin.hpp │ │ │ ├── DuplLimit.hpp │ │ │ ├── GroupBy.hpp │ │ │ ├── AggrFunctions.hpp │ │ │ ├── CartProd.hpp │ │ │ ├── Filter.hpp │ │ │ ├── NestedLoopFilter.hpp │ │ │ ├── MergeUnion.hpp │ │ │ ├── Union.hpp │ │ │ ├── HashGroupify.hpp │ │ │ └── Assignment.hpp │ │ └── runtime │ │ │ └── QueryDict.hpp │ └── cts │ │ └── codegen │ │ └── CodeGen.hpp ├── makefile └── src │ ├── rts │ ├── LocalMakefile │ ├── Operator.cpp │ ├── ValuesScan.cpp │ ├── Runtime.cpp │ ├── EmptyScan.cpp │ ├── Assignment.cpp │ └── SingletonScan.cpp │ └── infra │ ├── Mutex.cpp │ └── Pool.cpp ├── webinterface ├── yasgui │ └── license.txt └── index.html └── rapidjson └── include └── rapidjson └── internal └── swap.h /snap/Version.saved: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /snap/snap-core/alg.cpp: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snap/glib-core/stdafx.h: -------------------------------------------------------------------------------- 1 | // empty StdAfx.h for non-Visual-C++ compilers 2 | -------------------------------------------------------------------------------- /snap/snap-core/stdafx.h: -------------------------------------------------------------------------------- 1 | // empty StdAfx.h for non-Visual-C++ compilers 2 | -------------------------------------------------------------------------------- /src/trident/utils/parallel.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int32_t ParallelTasks::nthreads = -1; 4 | -------------------------------------------------------------------------------- /scripts/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM karmaresearch/trident:latest 2 | WORKDIR /app/trident 3 | RUN ["/app/trident/scripts/docker/update_and_make.sh"] 4 | -------------------------------------------------------------------------------- /src/trident/java/karmaresearch/trident/Trident.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karmaresearch/trident/HEAD/src/trident/java/karmaresearch/trident/Trident.class -------------------------------------------------------------------------------- /win64/trident/trident.vcxproj.user: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /tests/test_json.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int main(int argc, const char** argv) { 4 | std::string in = "{ \"a\": { \"c\": \"b\"}}"; 5 | JSON value; 6 | JSON::read(in, value); 7 | } 8 | -------------------------------------------------------------------------------- /win64/trident-core/trident-core.vcxproj.user: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /win64/trident-sparql/trident-sparql.vcxproj.user: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /snap/snap-core/graph.txt: -------------------------------------------------------------------------------- 1 | # Directed graph: graph.txt 2 | # Edge list format 3 | # Nodes: 10 Edges: 10 4 | # FromNodeId ToNodeId 5 | 0 7 6 | 1 9 7 | 2 3 8 | 3 8 9 | 4 0 10 | 5 2 11 | 6 4 12 | 7 8 13 | 8 3 14 | 9 9 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | testSnap 2 | scripts 3 | .ycm* 4 | .DS_Store 5 | /build* 6 | /trident* 7 | .vs/ 8 | *.exe 9 | *.dll 10 | *.obj 11 | *.lib 12 | *.log 13 | *.tlog 14 | *.pdb 15 | *.idb 16 | *.ilk 17 | *.exp 18 | *.ipdb 19 | *.iobj 20 | *.a 21 | *.o 22 | venv 23 | -------------------------------------------------------------------------------- /snap/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Makefile for non-Microsoft compilers 3 | # 4 | 5 | all: MakeAll 6 | 7 | test: TestAll 8 | 9 | MakeAll: 10 | $(MAKE) -C snap-core 11 | 12 | TestAll: 13 | $(MAKE) run -C test 14 | 15 | clean: 16 | $(MAKE) clean -C snap-core 17 | -------------------------------------------------------------------------------- /scripts/docker/update_and_make.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Make sure kognac is updated 3 | /app/kognac/scripts/docker/update_and_make.sh 4 | # Update trident 5 | cd /app/trident 6 | git pull 7 | cd build 8 | cmake .. 9 | make 10 | cd ../build_debug 11 | cmake .. 12 | make 13 | cd .. 14 | -------------------------------------------------------------------------------- /tests/test_httpserver.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | static void processRequest() { 4 | } 5 | 6 | int main(int argc, const char** argv) { 7 | auto f = std::bind(&processRequest); 8 | HttpServer server(8080, f); 9 | server.start(); 10 | } 11 | -------------------------------------------------------------------------------- /snap/snap-core/doc/util.h.txt: -------------------------------------------------------------------------------- 1 | /// SendVec 2 | Returns the number of bytes sent. If return value is negative, then some 3 | system call returned an error. 4 | /// 5 | 6 | /// SendVec64 7 | Returns the number of bytes sent. If return value is negative, then some 8 | system call returned an error. 9 | /// 10 | 11 | -------------------------------------------------------------------------------- /snap/glib-core/doc/bd.h.txt: -------------------------------------------------------------------------------- 1 | /// TPairHashImpl1 2 | Both \c TPairHashImpl1 and \c TPairHashImpl2 implement the same function: ((a + b) * (a + b + 1) / 2) + a. 3 | /// 4 | 5 | /// TPairHashImpl2 6 | Both \c TPairHashImpl1 and \c TPairHashImpl2 implement the same function: ((a + b) * (a + b + 1) / 2) + a. 7 | /// 8 | 9 | -------------------------------------------------------------------------------- /src/snap/undirected.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | Trident_UTNGraph::Trident_UTNGraph(KB *kb) { 5 | this->kb = kb; 6 | this->q = kb->query(); 7 | SnapReaders::loadAllFiles(kb); 8 | string path = kb->getPath() + string("/tree/flat"); 9 | mf = std::unique_ptr(new MemoryMappedFile(path)); 10 | rawnodes = mf->getData(); 11 | nnodes = mf->getLength() / 18; 12 | } 13 | -------------------------------------------------------------------------------- /include/snap/readers.h: -------------------------------------------------------------------------------- 1 | #ifndef _READERS_H 2 | #define _READERS_H 3 | 4 | #include 5 | 6 | #include 7 | 8 | class KB; 9 | class SnapReaders { 10 | public: 11 | typedef int64_t (*pReader)(const char*, const int64_t); 12 | const static pReader readers[256]; 13 | static std::vector f_sop; 14 | static std::vector f_osp; 15 | static void loadAllFiles(KB *kb); 16 | }; 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /src/trident/iterators/compositetermitr.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void CompositeTermItr::add(PairItr *itr) { 4 | currentCount = 0; 5 | children.push_back(itr); 6 | if (itr->hasNext()) { 7 | itr->next(); 8 | activechildren.push_back(itr); 9 | nc = true; 10 | } 11 | if (activechildren.size() > 1) { 12 | std::sort(activechildren.begin(), activechildren.end(), _sorter); 13 | } 14 | } 15 | 16 | 17 | -------------------------------------------------------------------------------- /snap/snap-core/numpy.h: -------------------------------------------------------------------------------- 1 | #ifndef NUMPY_H 2 | #define NUMPY_H 3 | 4 | namespace TSnap { 5 | /// Converts TIntV to Numpy array. 6 | void TIntVToNumpy(TIntV& IntV, int* IntNumpyVecOut, int n); 7 | /// Converts TFltV to Numpy array. 8 | void TFltVToNumpy(TFltV& FltV, float* FltNumpyVecOut, int n); 9 | /// Converts NumpyArray to TIntV 10 | void NumpyToTIntV(TIntV& IntV, int* IntNumpyVecIn, int n); 11 | /// Converts NumpyArray to TFltV 12 | void NumpyToTFltV(TFltV& FltV, float* FltNumpyVecIn, int n); 13 | } 14 | 15 | #endif //NUMPY_H 16 | -------------------------------------------------------------------------------- /snap/snap-core/coreper.h: -------------------------------------------------------------------------------- 1 | namespace TSnap { 2 | /// Girvan-Newman community detection algorithm based on Betweenness centrality. 3 | /// See: Girvan M. and Newman M. E. J., Community structure in social and biological networks, Proc. Natl. Acad. Sci. USA 99, 7821-7826 (2002) 4 | int FastCorePeriphery(PUNGraph& Graph, TIntIntH& out); 5 | int FastCorePeripheryGC(PUNGraph& Graph, TIntIntH& out); 6 | double BorgattiEverettMeasure(PUNGraph& Graph, TIntIntH& out, double coresize, int type); 7 | double PearsonCorrelation(PUNGraph& Graph, TIntIntH& out, int coresize); 8 | } -------------------------------------------------------------------------------- /tests/test_inv_dictionary_tree.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "../src/tree/root.h" 4 | #include 5 | #include 6 | 7 | using namespace std; 8 | 9 | int main(int argc, const char** argv) { 10 | 11 | Root *root = new Root(argv[1], 16 * 1024 * 1024, 10, false, true); 12 | 13 | TreeItr *itr = root->itr(); 14 | char text_term[1024]; 15 | Value value; 16 | 17 | while (itr->hasNext()) { 18 | int64_t key = itr->next(&value); 19 | value.text(text_term); 20 | cout << text_term << " " << key << endl; 21 | } 22 | 23 | return 0; 24 | } 25 | -------------------------------------------------------------------------------- /rdf3x/include/infra/Config.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_infra_Config 2 | #define H_infra_Config 3 | //--------------------------------------------------------------------------- 4 | #if !defined(__sparc__) 5 | #include 6 | #else 7 | // stdint.h is C99, not supported by SUN yet 8 | #include 9 | #endif 10 | //--------------------------------------------------------------------------- 11 | #if defined(__WIN32__)||defined(WIN32)||defined(_WIN32) 12 | #define CONFIG_WINDOWS 13 | #endif 14 | //--------------------------------------------------------------------------- 15 | #endif 16 | 17 | -------------------------------------------------------------------------------- /snap/snap-core/doc/gviz.h.txt: -------------------------------------------------------------------------------- 1 | /// DrawGViz1 2 | Useful for drawing small (<100 node) graphs. 3 | 4 | @param PltFNm Output filename (extension .ps, .png, .gif) determines the output format. 5 | @param NIdColorH Maps node ids to node colors (see GraphViz documentation for more details). 6 | /// 7 | 8 | 9 | /// DrawGViz2 10 | Useful for drawing small (<100 node) graphs. 11 | 12 | @param PltFNm Output filename (extension .ps, .png, .gif) determines the output format. 13 | @param NIdColorH Maps node ids to node colors (see GraphViz documentation for more details). 14 | /// 15 | -------------------------------------------------------------------------------- /snap/snap-core/flow.h: -------------------------------------------------------------------------------- 1 | namespace TSnap { 2 | 3 | // Mandatory name of edge capacity attribute for flow networks. 4 | const TStr CapAttrName = "capacity"; 5 | 6 | /// Returns the maximum integer valued flow in the network \c Net from source \c SrcNId to sink \c SnkNId. ##TSnap::GetMaxFlowIntEK 7 | int GetMaxFlowIntEK (PNEANet &Net, const int &SrcNId, const int &SnkNId); 8 | /// Returns the maximum integer valued flow in the network \c Net from source \c SrcNId to sink \c SnkNId. ##TSnap::GetMaxFlowIntEK 9 | int GetMaxFlowIntPR (PNEANet &Net, const int &SrcNId, const int &SnkNId); 10 | 11 | }; 12 | -------------------------------------------------------------------------------- /tests/test_ppr.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | int main(int argc, const char** argv) { 14 | KBConfig config; 15 | KB kb(argv[1], true, false, true, config); 16 | PTrident_TNGraph Graph = new Trident_TNGraph(&kb); 17 | 18 | std::vector weights(Graph->GetNodes()); 19 | TSnap::GetPageRank_stl(Graph, weights, false, 0.85, 0, 10); 20 | 21 | } 22 | -------------------------------------------------------------------------------- /snap/snap-core/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Small makefile for testing with non-Microsoft compilers 3 | # 4 | 5 | include ../Makefile.config 6 | 7 | all: testSnap 8 | 9 | testSnap: testSnap.cpp Snap.o 10 | $(CC) $(CXXFLAGS) -o testSnap testSnap.cpp Snap.o -I$(CGLIB) $(LDFLAGS) $(LIBS) 11 | 12 | Snap.o: *.h *.cpp ../glib-core/*.h ../glib-core/*.cpp 13 | $(CC) -c $(CXXFLAGS) Snap.cpp -I$(CGLIB) 14 | 15 | lib: Snap.o 16 | rm -f libsnap.a 17 | ar -cvrs libsnap.a Snap.o 18 | 19 | libinstall: lib 20 | cp libsnap.a /usr/local/lib/libsnap.a 21 | 22 | clean: 23 | rm -f *.o *.a testSnap testSnap.exe *.plt *.tab *.png 24 | 25 | -------------------------------------------------------------------------------- /tests/ontotranse2trident.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import sys 3 | import gzip 4 | 5 | dbin = pickle.load(open(sys.argv[1], 'rb')) 6 | fvalid = open(sys.argv[2] + "/valid_data", 'wt') 7 | for t in dbin['valid_subs']: 8 | fvalid.write(str(t[0]) + ' ' + str(t[2]) + ' ' + str(t[1]) + '\n') 9 | fvalid.close() 10 | 11 | triples = gzip.open(sys.argv[2] + '/triples.gz', 'wt') 12 | for t in dbin['train_subs']: 13 | triples.write(str(t[0]) + ' ' + str(t[2]) + ' ' + str(t[1]) + '\n') 14 | triples.close() 15 | dictfile = gzip.open(sys.argv[2] + '/dict.gz', 'wt') 16 | for k,v in dbin['entities'].items(): 17 | dictfile.write(str(k) + ' ' + str(len(v)) + ' ' + v + '\n') 18 | dictfile.close() 19 | -------------------------------------------------------------------------------- /tests/test_parallel_for.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | 12 | using namespace std; 13 | 14 | class Container { 15 | public: 16 | void operator()(const ParallelRange& r) const { 17 | LOG(ERRORL) << "begin " << r.begin() << " end " << r.end(); 18 | } 19 | 20 | }; 21 | 22 | int ParallelTasks::nthreads = -1; 23 | int main(int argc, const char** argv) { 24 | int nthreads = 8; 25 | 26 | Container container; 27 | ParallelTasks::parallel_for(0, 100, 60, container); 28 | return 0; 29 | } 30 | -------------------------------------------------------------------------------- /tests/test_lz4io.cpp: -------------------------------------------------------------------------------- 1 | #include "../src/utils/lz4io.h" 2 | #include "../src/compression/compressor.h" 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | using namespace std; 10 | 11 | int main(int argc, const char** argv) { 12 | 13 | // //Insert into a compressed file 14 | // int64_t startingValue = 0; 15 | // cout << "Starting from " << startingValue << endl; 16 | // { 17 | // LZ4Writer writer("/Users/jacopo/Desktop/file"); 18 | // for (int i = 0; i < 1000000; ++i) { 19 | // writer.writeVLong(i); 20 | // } 21 | // } 22 | 23 | { 24 | LZ4Reader reader(argv[1]); 25 | while (!reader.isEof()) { 26 | Term t; 27 | t.readFrom(&reader); 28 | } 29 | } 30 | cout << "Finished"; 31 | return 0; 32 | } 33 | -------------------------------------------------------------------------------- /snap/snap-core/doc/flow.h.txt: -------------------------------------------------------------------------------- 1 | /// TSnap::GetMaxFlowIntEK 2 | Implements max flow using the Edmonds-Karp algorithm. http://en.wikipedia.org/wiki/Edmonds%E2%80%93Karp_algorithm 3 | Although the asymptotic run time of Edmonds-Karp is worse than that of Push Relabel, in practice Edmonds Karp works very well, especially if the network is sparse. 4 | Unless the degree of each node is on the order of the number of nodes, it is best to use Edmonds Karp over Push Relabel. 5 | /// 6 | 7 | /// TSnap::GetMaxFlowIntPR 8 | Implements max flow using the Push-Relabel algorithm. http://en.wikipedia.org/wiki/Push%E2%80%93relabel_maximum_flow_algorithm 9 | The Global Relabel and Gap Relabel heuristics were also implemented to speed up the algorithm. http://link.springer.com/article/10.1007%2FPL00009180 10 | /// -------------------------------------------------------------------------------- /snap/snap-core/doc/gbase.h.txt: -------------------------------------------------------------------------------- 1 | /// TGraphFlag 2 | This is very useful for testing graph properties at compile time for partial template specialization as well as compile time assert (CAssert). 3 | /// 4 | 5 | /// TSnap::PrintInfo 6 | @param Fast true: only computes basic statistics (that can be computed fast). For more extensive information (and longer execution times) set Fast = false. 7 | /// 8 | 9 | /// TUnionFind 10 | For more info see: http://en.wikipedia.org/wiki/Disjoint-set_data_structure). 11 | /// 12 | 13 | /// THeap 14 | Data structure provides insertion of elements, and inspection and removal of the top element. It is guaranteed that the top element is the largest element in the heap, where the function object \c TCmp is used for comparisons. 15 | /// 16 | 17 | 18 | -------------------------------------------------------------------------------- /rdf3x/makefile: -------------------------------------------------------------------------------- 1 | #Include variables dependant on the platform 2 | OUTPUTDIR=. 3 | CPLUS=g++ 4 | 5 | LDFLAGS=-DBOOST_LOG_DYN_LINK -lboost_chrono -lboost_log-mt -lboost_log_setup-mt 6 | CPPFLAGS=-DBOOST_LOG_DYN_LINK -I include/ -c -std=c++11 -O3 7 | CFLAGS= -c -std=c99 8 | 9 | SRCDIR=src 10 | O_FILES=$(wildcard $(SRCDIR)/*.cpp) $(wildcard $(SRCDIR)/**/*.cpp) $(wildcard $(SRCDIR)/**/**/*.cpp) 11 | BFILES=$(subst $(SRCDIR),$(BUILDIR),$(O_FILES:.cpp=.o)) 12 | 13 | BUILDIR=$(OUTPUTDIR)/build 14 | RDF3X=$(BUILDIR_RELEASE)/rdf3x 15 | 16 | $(RDF3X): init $(BFILES) 17 | $(CPLUS) -o $@ $(BFILES) $(LDFLAGS) 18 | 19 | init: 20 | @mkdir -p $(BUILDIR) 21 | 22 | $(BUILDIR)/%.o: $(SRCDIR)/%.cpp 23 | @mkdir -p `dirname $@` 24 | $(CPLUS) $(CPPFLAGS) $< -o $@ 25 | 26 | .PHONY: clean 27 | clean: 28 | @rm -rf $(BUILDIR) 29 | @echo "Cleaning completed" 30 | -------------------------------------------------------------------------------- /include/trident/ml/hole.h: -------------------------------------------------------------------------------- 1 | #ifndef _HOLE_H 2 | #define _HOLE_H 3 | 4 | #include 5 | 6 | class HoleLearner : public PairwiseLearner { 7 | private: 8 | void update_gradient_matrix(std::unordered_map &gm, 9 | EntityGradient& eg1, 10 | EntityGradient& eg2, 11 | uint64_t term); 12 | 13 | bool shouldUpdate(uint32_t idx); 14 | 15 | public: 16 | HoleLearner(KB &kb, LearnParams &p) : 17 | PairwiseLearner(kb, p) { 18 | } 19 | 20 | void process_batch_withnegs(BatchIO &io, std::vector &oneg, 21 | std::vector &sneg); 22 | 23 | std::string getName() { 24 | return "HolE"; 25 | } 26 | 27 | float score(double*, double*, double*); 28 | }; 29 | #endif 30 | -------------------------------------------------------------------------------- /rdf3x/src/rts/LocalMakefile: -------------------------------------------------------------------------------- 1 | src_rts_operator:= \ 2 | rts/operator/Operator.cpp \ 3 | rts/operator/AggregatedIndexScan.cpp \ 4 | rts/operator/EmptyScan.cpp \ 5 | rts/operator/Filter.cpp \ 6 | rts/operator/FullyAggregatedIndexScan.cpp \ 7 | rts/operator/HashGroupify.cpp \ 8 | rts/operator/HashJoin.cpp \ 9 | rts/operator/IndexScan.cpp \ 10 | rts/operator/MergeJoin.cpp \ 11 | rts/operator/MergeUnion.cpp \ 12 | rts/operator/NestedLoopFilter.cpp \ 13 | rts/operator/NestedLoopJoin.cpp \ 14 | rts/operator/PlanPrinter.cpp \ 15 | rts/operator/ResultsPrinter.cpp \ 16 | rts/operator/Scheduler.cpp \ 17 | rts/operator/Selection.cpp \ 18 | rts/operator/SingletonScan.cpp \ 19 | rts/operator/Sort.cpp \ 20 | rts/operator/TableFunction.cpp \ 21 | rts/operator/Union.cpp \ 22 | rts/operator/Assignment.cpp \ 23 | rts/operator/DuplLimit.cpp 24 | -------------------------------------------------------------------------------- /include/trident/ml/transe.h: -------------------------------------------------------------------------------- 1 | #ifndef _TRANSE_H 2 | #define _TRANSE_H 3 | 4 | #include 5 | 6 | class TranseLearner : public PairwiseLearner { 7 | private: 8 | void update_gradient_matrix(std::vector &gm, 9 | std::vector> &signmatrix, 10 | std::vector &inputTripleID, 11 | std::vector &inputTerms, 12 | int pos, int neg); 13 | 14 | bool shouldUpdate(uint32_t idx); 15 | 16 | public: 17 | TranseLearner(KB &kb, LearnParams &p) : 18 | PairwiseLearner(kb, p) { 19 | } 20 | 21 | void process_batch_withnegs(BatchIO &io, std::vector &oneg, 22 | std::vector &sneg); 23 | 24 | std::string getName() { 25 | return "TranSE"; 26 | } 27 | }; 28 | #endif 29 | -------------------------------------------------------------------------------- /rdf3x/include/infra/osdep/Thread.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_infra_osdep_Thread 2 | #define H_infra_osdep_Thread 3 | //--------------------------------------------------------------------------- 4 | #include "infra/Config.hpp" 5 | //--------------------------------------------------------------------------- 6 | /// Thread functions 7 | class Thread 8 | { 9 | public: 10 | /// Create a new thread 11 | static bool start(void (*starter)(void*),void* arg,bool boost=false); 12 | 13 | /// Available physical memory 14 | static uint64_t guessPhysicalAvailable(); 15 | /// Wait x ms 16 | static void sleep(unsigned time); 17 | /// Get the thread id 18 | static int64_t threadID(); 19 | /// Activate the next thread 20 | static void yield(); 21 | /// Get the current time in milliseconds 22 | static uint64_t getTicks(); 23 | }; 24 | //--------------------------------------------------------------------------- 25 | #endif 26 | -------------------------------------------------------------------------------- /tests/test_graph.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | 5 | using namespace std; 6 | 7 | int main(int argc, char* argv[]) { 8 | Env = TEnv(argc, argv, TNotify::StdNotify); 9 | Env.PrepArgs(TStr::Fmt("Test. build: %s, %s. Time: %s", __TIME__, __DATE__, TExeTm::GetCurTm())); 10 | const TStr InFNm = Env.GetIfArgPrefixStr("-i:", "../as20graph.txt", "Input un/directed graph"); 11 | std::chrono::milliseconds durationPR; 12 | 13 | PNGraph Graph = TSnap::LoadEdgeList(InFNm); 14 | std::chrono::system_clock::time_point start = std::chrono::system_clock::now(); 15 | 16 | int64 out = TSnap::CountTriangles(Graph); 17 | //double out = TSnap::GetClustCf(Graph, -1); 18 | durationPR = std::chrono::duration_cast(std::chrono::system_clock::now() - start); 19 | cout << "Runtime algos: " << durationPR.count() << " ms. out=" << out << endl; 20 | } 21 | -------------------------------------------------------------------------------- /include/trident/sparql/sparql.h: -------------------------------------------------------------------------------- 1 | #ifndef _SPARQL_H 2 | #define _SPARQL_H 3 | 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | class SPARQLUtils { 12 | public: 13 | static void parseQuery(bool &success, 14 | SPARQLParser &parser, 15 | std::unique_ptr &queryGraph, 16 | QueryDict &queryDict, 17 | TridentLayer &db); 18 | 19 | static void execSPARQLQuery(string sparqlquery, 20 | bool explain, 21 | int64_t nterms, 22 | TridentLayer &db, 23 | bool printstdout, 24 | bool jsonoutput, 25 | JSON *jsonvars, 26 | JSON *jsonresults, 27 | JSON *jsonstats); 28 | }; 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /tests/test_insert_perm.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | int main(int argc, const char** argv) { 7 | KBConfig config; 8 | config.setParamInt(DICTPARTITIONS, 1); 9 | MemoryOptimizer::optimizeForWriting(1000000000, config); 10 | KB kb(argv[2], false, false, true, config); 11 | 12 | //Insert stuff 13 | TreeWriter *writer = new TreeWriter(argv[3]); 14 | 15 | Loader::sortAndInsert(0, 16 | 6, 17 | false, 18 | argv[1], 19 | NULL, 20 | writer, 21 | kb.insert(), 22 | false, 23 | false, 24 | false, 25 | NULL, 26 | 0, 27 | true); 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/snap/analytics.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | void Analytics::run(KB &kb, 6 | string nameTask, 7 | string outputfile, 8 | string params) { 9 | 10 | //Check what type of graph is stored in the KB 11 | if (kb.getGraphType() == GraphType::DIRECTED) { 12 | Analytics::runTask(kb, nameTask, outputfile, params); 13 | } else { 14 | //Graph should be undirected 15 | if ((kb.getGraphType() != GraphType::UNDIRECTED)) { 16 | LOG(ERRORL) << "Graph analytical operations work only on simple directed or simple undirected graphs"; 17 | throw 10; 18 | } 19 | Analytics::runTask(kb, nameTask, outputfile, params); 20 | } 21 | 22 | } 23 | 24 | -------------------------------------------------------------------------------- /tests/test_insert4.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | using namespace std; 22 | e 23 | int main(int argc, const char** argv) { 24 | string inputP2 = argv[1]; 25 | string inputP5 = argv[2]; 26 | 27 | Loader::sortPermutation(inputP2, 8, 72, true, 110000000000l, 30000000000l, 16, false); 28 | Loader::sortPermutation(inputP5, 8, 72, true, 110000000000l, 30000000000l, 16, false); 29 | //Loader::sortPermutation(inputP4, 8, 72, true, 110000000000l, 30000000000l, 16); 30 | } 31 | -------------------------------------------------------------------------------- /tests/test_readdictfiles.cpp: -------------------------------------------------------------------------------- 1 | #include "../src/kb/kbconfig.h" 2 | #include "../src/kb/kb.h" 3 | #include "../src/kb/memoryopt.h" 4 | #include "../src/kb/inserter.h" 5 | 6 | #include "../src/main/loader.h" 7 | 8 | #include "../src/sorting/sorter.h" 9 | #include "../src/sorting/filemerger.h" 10 | 11 | #include "../src/compression/compressor.h" 12 | 13 | #include "../src/utils/lz4io.h" 14 | #include "../src/utils/utils.h" 15 | 16 | #include "../src/utils/stringscol.h" 17 | 18 | #include "kognac/logs.h" 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | using namespace std; 28 | 29 | int main(int argc, const char** argv) { 30 | 31 | LZ4Reader *r = new LZ4Reader(argv[1]); 32 | 33 | while (!r->isEof()) { 34 | int64_t number = r->parseLong(); 35 | int size; 36 | const char *term = r->parseString(size); 37 | cout << number << " " << string(term + 2, size -2) << endl; 38 | } 39 | 40 | 41 | delete r; 42 | LOG(DEBUGL) << "finished"; 43 | } 44 | -------------------------------------------------------------------------------- /tests/test_mergesort.cpp: -------------------------------------------------------------------------------- 1 | #include "../src/utils/lz4io.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "../src/sorting/filemerger.h" 10 | #include "../src/compression/compressor.h" 11 | 12 | using namespace std; 13 | 14 | int main(int argc, const char** argv) { 15 | 16 | vector files; 17 | for(int i = 1; i < argc; ++i) { 18 | files.push_back(string(argv[i])); 19 | cerr << "Adding file " << argv[i] << endl; 20 | } 21 | 22 | FileMerger merger(files); 23 | 24 | int64_t count = 0; 25 | int64_t duplicates = 0; 26 | char *previousTerm = new char[1024]; 27 | check_again: 28 | while(!merger.isEmpty()) { 29 | Term t = merger.get(); 30 | 31 | if (t.equals(previousTerm)) { 32 | duplicates++; 33 | goto check_again; 34 | } 35 | 36 | memcpy(previousTerm, t.term,t.size); 37 | cout << "Process term " << string(t.term + 2, t.size -2) << endl; 38 | count++; 39 | } 40 | 41 | cerr << "Elements processed " << count << " " << duplicates << endl; 42 | 43 | } 44 | -------------------------------------------------------------------------------- /snap/snap-core/doc/mmnet.h.txt: -------------------------------------------------------------------------------- 1 | /// TModeNet::Class 2 | The class corresponding to one particular mode in a TMMNet. 3 | This class contains only the set of nodes in that mode, and does not explicitly model edges. 4 | For easy access to neighbors, there is a dense TIntV attribute for each CrossNet, which is a vector of edge ids of neighbors of our mode in that CrossNet. 5 | /// 6 | 7 | /// TCrossNet::Class 8 | The class corresponding to one particular CrossNet in a TMMNet. 9 | Has a CrossNet name, and two modes. Can be either undirected or directed. If it is directed, it is directed in one direction only: from mode 1 to mode 2. 10 | /// 11 | 12 | /// TMMNet::Class 13 | Represents a multimodal network. A mutimodal network is represented by composing TModeNets (disjoint sets of nodes) and TCrossNets (disjoint sets of edges that connect nodes in two TModeNets). 14 | TMMNet contains methods to create TModeNets and TCrossNets by name, and returns pointers to them. All operations more granular than this, like adding nodes and edges, are run directly using methods provided in the ModeNets and CrossNets. 15 | /// -------------------------------------------------------------------------------- /src/trident/java/native/karmaresearch_trident_Trident.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class karmaresearch_trident_Trident */ 4 | 5 | #ifndef _Included_karmaresearch_trident_Trident 6 | #define _Included_karmaresearch_trident_Trident 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | /* 11 | * Class: karmaresearch_trident_Trident 12 | * Method: load 13 | * Signature: (Ljava/lang/String;)V 14 | */ 15 | JNIEXPORT void JNICALL Java_karmaresearch_trident_Trident_load 16 | (JNIEnv *, jobject, jstring); 17 | 18 | /* 19 | * Class: karmaresearch_trident_Trident 20 | * Method: sparql 21 | * Signature: (Ljava/lang/String;)Ljava/lang/String; 22 | */ 23 | JNIEXPORT jstring JNICALL Java_karmaresearch_trident_Trident_sparql 24 | (JNIEnv *, jobject, jstring); 25 | 26 | /* 27 | * Class: karmaresearch_trident_Trident 28 | * Method: unload 29 | * Signature: ()V 30 | */ 31 | JNIEXPORT void JNICALL Java_karmaresearch_trident_Trident_unload 32 | (JNIEnv *, jobject); 33 | 34 | #ifdef __cplusplus 35 | } 36 | #endif 37 | #endif 38 | -------------------------------------------------------------------------------- /snap/snap-core/doc/bignet.h.txt: -------------------------------------------------------------------------------- 1 | /// Big_network 2 | This class implements similar interface to TNGraph and TUNGraph. 3 | The class is meant for storing particularly large static directed or undirected networks. 4 | The network representation is optimized for low memory footprint. 5 | This means that when a particular node is added to the network its (maximum) in- and out-degree need to be specified, so that the class allocates enough memory for that number of edges being adjacent to a node. 6 | The class nicely supports adding as well as deleting nodes, although the memory does not get freed. 7 | Deleting edges is supported, while adding edges is supported only up to the point until the node reaches its prespecified in- or out-degree. 8 | /// 9 | 10 | /// TBigNet::TNode 11 | For memory efficiency we store edge lists in a vector pool. 12 | /// 13 | 14 | /// TBigNet::OutVId 15 | If the graph is undirected (IsDir=false) then InVId==OutVId. 16 | /// 17 | 18 | /// TBigNet::TNodeI 19 | Only forward iteration (operator++) is supported. 20 | /// 21 | 22 | /// TBigNet::TEdgeI 23 | Only forward iteration (operator++) is supported. 24 | /// 25 | 26 | -------------------------------------------------------------------------------- /tests/test_httpclient.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | int main(int argc, const char** argv) { 9 | HttpClient client("localhost", 9200); 10 | if (client.connect()) { 11 | } else { 12 | std::cerr << "Not connected" << std::endl; 13 | } 14 | 15 | std::string req = "/wiki/_search?pretty"; 16 | std::string headers; 17 | std::string resp; 18 | std::map params; 19 | 20 | 21 | JSON json; 22 | JSON child; 23 | JSON child2; 24 | child.add_child("match_all", child2); 25 | json.add_child("query", child); 26 | std::stringstream ss; 27 | JSON::write(ss, json); 28 | std::string payload = ss.str(); 29 | params.insert(std::make_pair("", payload)); 30 | 31 | if (client.post(req, params, headers, resp, "application/json")) { 32 | //std::cout << resp << std::endl; 33 | std::cout << headers << std::endl; 34 | std::cout << "OK" << std::endl; 35 | } else { 36 | std::cerr << "Error!" << std::endl; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /win64/trident/trident.vcxproj.filters: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;ipp;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | Source Files 20 | 21 | 22 | Source Files 23 | 24 | 25 | -------------------------------------------------------------------------------- /rdf3x/src/rts/Operator.cpp: -------------------------------------------------------------------------------- 1 | #include "rts/operator/Operator.hpp" 2 | //--------------------------------------------------------------------------- 3 | // RDF-3X 4 | // (c) 2008 Thomas Neumann. Web site: http://www.mpi-inf.mpg.de/~neumann/rdf3x 5 | // 6 | // This work is licensed under the Creative Commons 7 | // Attribution-Noncommercial-Share Alike 3.0 Unported License. To view a copy 8 | // of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ 9 | // or send a letter to Creative Commons, 171 Second Street, Suite 300, 10 | // San Francisco, California, 94105, USA. 11 | //--------------------------------------------------------------------------- 12 | bool Operator::disableSkipping = false; 13 | //--------------------------------------------------------------------------- 14 | Operator::Operator(double expectedOutputCardinality) 15 | : expectedOutputCardinality(expectedOutputCardinality),observedOutputCardinality(0) 16 | // Constructor 17 | { 18 | } 19 | //--------------------------------------------------------------------------- 20 | Operator::~Operator() 21 | // Destructor 22 | { 23 | } 24 | //--------------------------------------------------------------------------- 25 | -------------------------------------------------------------------------------- /webinterface/yasgui/license.txt: -------------------------------------------------------------------------------- 1 | MIT License: 2 | 3 | Copyright (c) 2014 Laurens Rietveld 4 | 5 | Permission is hereby granted, free of charge, to any person 6 | obtaining a copy of this software and associated documentation 7 | files (the "Software"), to deal in the Software without 8 | restriction, including without limitation the rights to use, 9 | copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the 11 | Software is furnished to do so, subject to the following 12 | conditions: 13 | 14 | The above copyright notice and this permission notice shall be 15 | included in all copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 19 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 21 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 22 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 23 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 | OTHER DEALINGS IN THE SOFTWARE. 25 | -------------------------------------------------------------------------------- /snap/snap-core/numpy.cpp: -------------------------------------------------------------------------------- 1 | namespace TSnap { 2 | /// Fills the numpyvec array with TIntV vector values. 3 | /// Note that only the first n values are filled. 4 | void TIntVToNumpy(TIntV& IntV, int* IntNumpyVecOut, int n) { 5 | int limit = MIN(IntV.Len(), n); 6 | 7 | for (int i=0; i < limit; i++) { 8 | IntNumpyVecOut[i] = IntV[i]; 9 | } 10 | } 11 | 12 | 13 | /// Fills the numpyvec array with TFltV vector values. 14 | /// Note that only the first n values are filled. 15 | void TFltVToNumpy(TFltV& FltV, float* FltNumpyVecOut, int n) { 16 | int limit = MIN(FltV.Len(), n); 17 | 18 | for (int i=0; i < limit; i++) { 19 | FltNumpyVecOut[i] = static_cast(FltV[i]); 20 | } 21 | } 22 | 23 | /// Fills the IntV with Numpy Array values. 24 | /// Assumes that IntV is of size n. 25 | void NumpyToTIntV(TIntV& IntV, int* IntNumpyVecIn, int n) { 26 | for (int i = 0; i < n; ++i) { 27 | IntV[i] = IntNumpyVecIn[i]; 28 | } 29 | } 30 | 31 | /// Fills the FltV with Numpy Array values. 32 | /// Assumes that the vector is of size n. 33 | void NumpyToTFltV(TFltV& FltV, float* FltNumpyVecIn, int n) { 34 | for (int i = 0; i < n; ++i) { 35 | FltV[i] = FltNumpyVecIn[i]; 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /tests/test_insertdictionary.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | using namespace std; 19 | 20 | int main(int argc, const char** argv) { 21 | { 22 | KBConfig config; 23 | config.setParamInt(DICTPARTITIONS, 1); 24 | config.setParamInt(NINDICES, 6); 25 | config.setParamBool(AGGRINDICES, false); 26 | config.setParamBool(USEFIXEDSTRAT, false); 27 | config.setParamInt(FIXEDSTRAT, 96); 28 | config.setParamInt(THRESHOLD_SKIP_TABLE, 64); 29 | MemoryOptimizer::optimizeForWriting(10000000000l, config); 30 | KB kb(argv[1], false, false, true, config); 31 | 32 | nTerm *maxValues = new nTerm[1]; 33 | Loader::insertDictionary(0, kb.getDictMgmt(), argv[2], 34 | true, true, false, maxValues); 35 | kb.closeMainDict(); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /rdf3x/include/infra/util/Type.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_infra_util_Type 2 | #define H_infra_util_Type 3 | //--------------------------------------------------------------------------- 4 | // RDF-3X 5 | // (c) 2009 Thomas Neumann. Web site: http://www.mpi-inf.mpg.de/~neumann/rdf3x 6 | // 7 | // This work is licensed under the Creative Commons 8 | // Attribution-Noncommercial-Share Alike 3.0 Unported License. To view a copy 9 | // of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ 10 | // or send a letter to Creative Commons, 171 Second Street, Suite 300, 11 | // San Francisco, California, 94105, USA. 12 | //--------------------------------------------------------------------------- 13 | /// Information about the type system 14 | class Type { 15 | public: 16 | /// Different literal types 17 | enum ID { 18 | URI, Literal, CustomLanguage, CustomType, 19 | String, Integer, Decimal, Double, Boolean, Date 20 | }; 21 | /// Does the type have a sub-type? 22 | static inline bool hasSubType(ID t) { return (t==CustomLanguage)||(t==CustomType); } 23 | /// Get the type of the sub-type 24 | static inline ID getSubTypeType(ID t) { return (t==CustomLanguage)?Literal:URI; } 25 | }; 26 | //--------------------------------------------------------------------------- 27 | #endif 28 | -------------------------------------------------------------------------------- /include/trident/ml/feedback.h: -------------------------------------------------------------------------------- 1 | #ifndef _FEEDBACK_H 2 | #define _FEEDBACK_H 3 | 4 | #include 5 | 6 | #include 7 | 8 | class Feedback { 9 | private: 10 | struct QueryDetails { 11 | std::vector positions; 12 | double avg = -1; 13 | }; 14 | 15 | std::unordered_map queries_po; 16 | std::unordered_map queries_sp; 17 | uint16_t currentEpoch = 0; 18 | uint64_t excluded; 19 | uint64_t threshold = 0; 20 | uint32_t minFullEpochs = 0; 21 | 22 | typedef std::unordered_map::iterator QueriesItr; 23 | static bool _querySorter(const std::pair& a, 24 | const std::pair& b); 25 | 26 | public: 27 | Feedback(uint64_t threshold, uint32_t minFullEpochs) : 28 | threshold(threshold), 29 | minFullEpochs(minFullEpochs) {} 30 | 31 | bool shouldBeIncluded(int64_t s, int64_t p, int64_t o); 32 | 33 | void setCurrentEpoch(uint16_t epoch) { 34 | currentEpoch = epoch; 35 | } 36 | 37 | void addFeedbacks(std::shared_ptr::OutputTest>); 38 | }; 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /snap/Credits.txt: -------------------------------------------------------------------------------- 1 | Stanford Network Analysis Platform (SNAP) is a general purpose, high 2 | performance system for graph and network manipulation and analysis. 3 | 4 | Jure Leskovec started developing SNAP during his PhD studies in 2005. 5 | Rok Sosic has been maintaining the code since 2012. GLib, which is used 6 | extensively by SNAP, was started by Marko Grobelnik in 1996. 7 | 8 | The following people also contributed to the development of SNAP 9 | (in alphabetical order). 10 | 11 | Arijit Banerjee 12 | Austin Benson 13 | Justin Cheng 14 | Royce Cheng-Yue 15 | Blaz Fortuna 16 | Florian Geigl 17 | Stephen Dean Guo 18 | Agrim Gupta 19 | Shubham Gupta 20 | Tim Hsieh 21 | Richard Hsu 22 | Chentai Kao 23 | Mario Karlovcec 24 | Nikhil Khadke 25 | Vikesh Khanna 26 | Emmanouil Kiagias 27 | Myunghwan Kim 28 | Jason Jong 29 | Peter Lofgren 30 | Stephen Macke 31 | Carolina Mattsson 32 | Julian McAuley 33 | Morteza Milani 34 | Riyad Parvez 35 | Yonathan Perez 36 | Rohan Puttagunta 37 | Martin Raison 38 | Sheila Ramaswamy 39 | Manuel Gomez Rodriguez 40 | Kenny Root 41 | Davide Sardina 42 | Christoph Schaefer 43 | Pararth Shah 44 | Nicholas Shelly 45 | Guo Tai 46 | Ming Han Teh 47 | Ba-Quan Truong 48 | Utkarsh Upadhyay 49 | Theodore Vasiloudis 50 | Viswajith Venugopal 51 | Jaewon Yang 52 | David Zeng 53 | Chenguang Zhu 54 | Ziga Zupanec 55 | 56 | -------------------------------------------------------------------------------- /include/trident/iterators/tupleiterators.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Jacopo Urbani 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, 15 | * software distributed under the License is distributed on an 16 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | * KIND, either express or implied. See the License for the 18 | * specific language governing permissions and limitations 19 | * under the License. 20 | **/ 21 | 22 | 23 | #ifndef _TUPLE_ITR_H 24 | #define _TUPLE_ITR_H 25 | 26 | #include 27 | #include 28 | #include 29 | 30 | class TupleIterator { 31 | public: 32 | virtual bool hasNext() = 0; 33 | 34 | virtual void next() = 0; 35 | 36 | virtual size_t getTupleSize() = 0; 37 | 38 | virtual uint64_t getElementAt(const int pos) = 0; 39 | 40 | virtual ~TupleIterator() {} 41 | }; 42 | 43 | #endif 44 | -------------------------------------------------------------------------------- /tests/cmpoutput.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import pickle 4 | 5 | 6 | realset = pickle.load(open(sys.argv[2], 'rb')) 7 | realset = set(realset['train_subs']) 8 | 9 | debugdir = sys.argv[1] 10 | files = os.listdir(debugdir) 11 | for e in range(10): 12 | outputset = set() 13 | countp = 0 14 | allsneg = [] 15 | alloneg = [] 16 | for f in files: 17 | if "batch-" + str(e) + '-' in f: 18 | fin = open(debugdir + "/" + f, 'rb') 19 | b = fin.read(8) 20 | sizebuffer = int.from_bytes(b, 'little') 21 | for j in range(sizebuffer): 22 | a = fin.read(8) 23 | b = fin.read(8) 24 | c = fin.read(8) 25 | sneg = fin.read(8) 26 | oneg = fin.read(8) 27 | s = int.from_bytes(a, 'little') 28 | p = int.from_bytes(b, 'little') 29 | o = int.from_bytes(c, 'little') 30 | sneg = int.from_bytes(sneg, 'little') 31 | allsneg.append(sneg) 32 | oneg = int.from_bytes(oneg, 'little') 33 | alloneg.append(oneg) 34 | outputset.add((s,o,p)) 35 | if (sneg, o, p) in realset: 36 | countp += 1 37 | print("The set contains ", len(outputset), countp, outputset == realset) 38 | for i in range(10): 39 | print(alloneg[i]) 40 | -------------------------------------------------------------------------------- /include/trident/ml/pairwiselearner.h: -------------------------------------------------------------------------------- 1 | #ifndef _PAIRWISE_H 2 | #define _PAIRWISE_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | struct _PairwiseSorter { 11 | const std::vector &vec; 12 | _PairwiseSorter(const std::vector &vec) : vec(vec) {} 13 | bool operator() (const uint32_t v1, const uint32_t v2) const { 14 | return vec[v1] < vec[v2]; 15 | } 16 | }; 17 | 18 | class PairwiseLearner : public Learner { 19 | private: 20 | std::random_device rd; 21 | std::mt19937 gen; 22 | std::uniform_int_distribution<> dis; 23 | 24 | void gen_random(Querier *q, 25 | BatchIO &io, 26 | std::vector &input, 27 | const bool subjObjs, 28 | const uint16_t ntries); 29 | 30 | public: 31 | PairwiseLearner(KB &kb, LearnParams &p) : 32 | Learner(kb, p), gen(rd()), dis(0, ne - 1) { 33 | } 34 | 35 | void process_batch(BatchIO &io, const uint32_t epoch, const uint16_t 36 | nbatches); 37 | 38 | virtual void process_batch_withnegs(BatchIO &io, std::vector &oneg, 39 | std::vector &sneg) = 0; 40 | 41 | bool generateViolations() { 42 | return true; 43 | } 44 | }; 45 | #endif 46 | -------------------------------------------------------------------------------- /tests/test_insertlarge.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | using namespace std; 14 | 15 | class MyTreeInserter: public TreeInserter { 16 | private: 17 | public: 18 | MyTreeInserter() { 19 | } 20 | 21 | void addEntry(nTerm key, int64_t nElements, short file, int pos, 22 | char strategy) { 23 | } 24 | }; 25 | int main(int argc, const char** argv) { 26 | KBConfig config; 27 | config.setParamInt(DICTPARTITIONS, 1); 28 | config.setParamInt(NINDICES, 6); 29 | config.setParamBool(AGGRINDICES, false); 30 | config.setParamBool(USEFIXEDSTRAT, false); 31 | config.setParamInt(FIXEDSTRAT, 96); 32 | config.setParamInt(THRESHOLD_SKIP_TABLE, 64); 33 | MemoryOptimizer::optimizeForWriting(10000000000l, config); 34 | KB kb(argv[1], false, false, true, config); 35 | MyTreeInserter treeInserter; 36 | 37 | Inserter *ins = kb.insert(); 38 | for(int64_t i = 0; i < 1000000000; i += 2) { 39 | ins->insert(IDX_SPO, 0, i, i + 1, 1, NULL, NULL, false, false); 40 | if (i % 10000000 == 0) 41 | cout << "Idx " << i << endl; 42 | } 43 | ins->flush(IDX_SPO, NULL, &treeInserter, false, false); 44 | 45 | ins->stopInserts(IDX_SPO); 46 | delete ins; 47 | } 48 | -------------------------------------------------------------------------------- /include/trident/kb/memoryopt.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Jacopo Urbani 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, 15 | * software distributed under the License is distributed on an 16 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | * KIND, either express or implied. See the License for the 18 | * specific language governing permissions and limitations 19 | * under the License. 20 | **/ 21 | 22 | 23 | #ifndef MEMORYOPT_H_ 24 | #define MEMORYOPT_H_ 25 | 26 | #include 27 | 28 | class MemoryOptimizer { 29 | 30 | private: 31 | 32 | static int calculateBytesPerDictPair(int64_t nTerms, int leafSize); 33 | 34 | public: 35 | static void optimizeForWriting(int64_t inputTriples, KBConfig &config); 36 | 37 | static void optimizeForReading(int ndicts, KBConfig &config); 38 | 39 | static void optimizeForReasoning(int ndicts, KBConfig &config); 40 | 41 | }; 42 | 43 | #endif /* MEMORYOPT_H_ */ 44 | -------------------------------------------------------------------------------- /snap/snap-core/casc.h: -------------------------------------------------------------------------------- 1 | namespace TSnap { 2 | /// Takes as input the column names of the PTable \c P as \c C1, \c C2,\c C3 and \c C4 and returns a directed graph of \c W-adjacent events. For graph generation events are sorted by \c C1. 3 | PNGraph CascGraphSource(PTable P,const TStr C1,const TStr C2,const TStr C3,const TStr C4,const TInt W); 4 | /// Takes as input the column names of the PTable \c P as \c C1, \c C2,\c C3 and \c C4 and returns a directed graph of \c W-adjacent events. For graph generation events are sorted by \c C3. 5 | PNGraph CascGraphTime(PTable P,const TStr C1,const TStr C2,const TStr C3,const TStr C4,const TInt W); 6 | /// Takes as input the column names of the PTable \c P as \c C1, \c C2, \c C3 and \c C4 and returns a directed graph of \c W-adjacent events. By default calls CascGraphSource. Toggle SortParam to use CascGraphTime. 7 | PNGraph CascGraph(PTable P,const TStr C1,const TStr C2,const TStr C3,const TStr C4,const TInt W,bool SortParam = true); 8 | /// Takes as input a directed graph and returns all the top cascades in \c TopCascVV. 9 | void CascFind(PNGraph Graph,PTable P,const TStr C1,const TStr C2,const TStr C3,const TStr C4,TVec &TopCascVV,bool Print = false); 10 | /// Parallel implementaion of CascFind takes as input a directed graph and returns all the top cascades in \c TopCascVV. 11 | #ifdef USE_OPENMP 12 | void CascFindMP(PNGraph Graph,PTable P,const TStr C1,const TStr C2,const TStr C3,const TStr C4,TVec &TopCascVV); 13 | #endif 14 | } 15 | -------------------------------------------------------------------------------- /rdf3x/include/rts/operator/ValuesScan.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_rts_operator_ValuesScan 2 | #define H_rts_operator_ValuesScan 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | //--------------------------------------------------------------------------- 13 | /// A sort operator 14 | class ValuesScan: public Operator 15 | { 16 | private: 17 | std::vector regs; 18 | const std::vector &values; 19 | unsigned processed, toBeProcessed; 20 | 21 | 22 | public: 23 | /// Constructor 24 | ValuesScan(std::vector regs, 25 | const std::vector &values, 26 | double expectedOutputCardinality); 27 | 28 | /// Destructor 29 | ~ValuesScan(); 30 | 31 | /// Produce the first tuple 32 | uint64_t first(); 33 | /// Produce the next tuple 34 | uint64_t next(); 35 | /// Print the operator tree. Debugging only. 36 | void print(PlanPrinter& out); 37 | /// Add a merge join hint 38 | void addMergeHint(Register* reg1,Register* reg2); 39 | /// Register parts of the tree that can be executed asynchronous 40 | void getAsyncInputCandidates(Scheduler& scheduler); 41 | }; 42 | //--------------------------------------------------------------------------- 43 | #endif 44 | -------------------------------------------------------------------------------- /snap/glib-core/wch.cpp: -------------------------------------------------------------------------------- 1 | ///////////////////////////////////////////////// 2 | // Wide-Char 3 | const TWCh TWCh::Mn(0, 0); 4 | const TWCh TWCh::Mx(0xFF, 0xFF); 5 | const int TWCh::Vals=0x10000; 6 | 7 | const TWCh TWCh::StartWCh(0xFE, 0xFF); 8 | const TWCh TWCh::TabWCh(TCh::TabCh); 9 | const TWCh TWCh::LfWCh(TCh::LfCh); 10 | const TWCh TWCh::CrWCh(TCh::CrCh); 11 | const TWCh TWCh::SpaceWCh(' '); 12 | 13 | ///////////////////////////////////////////////// 14 | // Wide-Char-Array 15 | void TWChA::AddCStr(const char* CStr){ 16 | int CStrLen=int(strlen(CStr)); 17 | for (int ChN=0; ChNEof()){ 40 | WChA+=TWCh::LoadTxt(SIn);} 41 | } 42 | 43 | void TWChA::SaveTxt(const PSOut& SOut) const { 44 | TWCh::StartWCh.SaveTxt(SOut); 45 | for (int WChN=0; WChN 5 | 6 | #include 7 | #include 8 | 9 | class GradTracer { 10 | private: 11 | struct _Update { 12 | uint16_t epoch; 13 | uint32_t startidx; 14 | uint16_t n; 15 | }; 16 | const uint16_t dim; 17 | 18 | std::vector> updates_meta; 19 | std::vector> updates_values; 20 | const bool all; 21 | std::vector entities; 22 | std::unordered_set s_entities; 23 | public: 24 | GradTracer(uint32_t ne, int store_ents, uint16_t dim) : dim(dim), 25 | all(store_ents == -1) { 26 | if (!all) { 27 | std::random_device rd; 28 | std::mt19937 gen(rd()); 29 | std::uniform_int_distribution<> dis(0, ne -1); 30 | for(int j = 0; j < store_ents; ++j) { 31 | uint32_t ent = dis(gen); 32 | entities.push_back(ent); 33 | s_entities.insert(ent); 34 | } 35 | } 36 | updates_meta.resize(ne); 37 | updates_values.resize(ne); 38 | } 39 | 40 | void add(uint16_t epoch, uint32_t id, 41 | std::vector &features, 42 | uint32_t n); 43 | 44 | void store(std::string file); 45 | }; 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /rdf3x/include/rts/runtime/QueryDict.hpp: -------------------------------------------------------------------------------- 1 | #ifndef _QUERYDICT_H 2 | #define _QUERYDICT_H 3 | 4 | #include 5 | 6 | class QueryDict { 7 | private: 8 | uint64_t nextId; 9 | std::unordered_map map; 10 | std::unordered_map invertedMap; 11 | 12 | public: 13 | QueryDict(uint64_t nextId) : nextId(nextId) {} 14 | 15 | bool hasID(uint64_t id) const { 16 | return map.find(id) != map.end(); 17 | } 18 | 19 | bool isEmpty() const { 20 | return map.size() == 0; 21 | } 22 | 23 | std::string getLiteral(uint64_t id) { 24 | return map.find(id)->second; 25 | } 26 | 27 | uint64_t add(std::string &literal) { 28 | std::unordered_map::iterator itr = invertedMap.find(literal); 29 | if (itr == invertedMap.end()) { 30 | invertedMap.insert(std::make_pair(literal, nextId)); 31 | map.insert(std::make_pair(nextId, literal)); 32 | return nextId++; 33 | } else { 34 | return itr->second; 35 | } 36 | } 37 | 38 | std::pair getStringBoundaries(uint64_t id) { 39 | std::pair pair; 40 | std::unordered_map::iterator itr = map.find(id); 41 | const char* str = itr->second.c_str(); 42 | pair.first = (char*)str; 43 | pair.second = (char*)(str + itr->second.length()); 44 | return pair; 45 | } 46 | }; 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /rdf3x/include/rts/operator/EmptyScan.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_rts_operator_EmptyScan 2 | #define H_rts_operator_EmptyScan 3 | //--------------------------------------------------------------------------- 4 | // RDF-3X 5 | // (c) 2008 Thomas Neumann. Web site: http://www.mpi-inf.mpg.de/~neumann/rdf3x 6 | // 7 | // This work is licensed under the Creative Commons 8 | // Attribution-Noncommercial-Share Alike 3.0 Unported License. To view a copy 9 | // of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ 10 | // or send a letter to Creative Commons, 171 Second Street, Suite 300, 11 | // San Francisco, California, 94105, USA. 12 | //--------------------------------------------------------------------------- 13 | #include "rts/operator/Operator.hpp" 14 | //--------------------------------------------------------------------------- 15 | /// A scan over the empty set 16 | class EmptyScan : public Operator 17 | { 18 | public: 19 | /// Constructor 20 | EmptyScan(); 21 | /// Destructor 22 | ~EmptyScan(); 23 | 24 | /// Produce the first tuple 25 | uint64_t first(); 26 | /// Produce the next tuple 27 | uint64_t next(); 28 | 29 | /// Print the operator tree. Debugging only. 30 | void print(PlanPrinter& out); 31 | /// Add a merge join hint 32 | void addMergeHint(Register* reg1,Register* reg2); 33 | /// Register parts of the tree that can be executed asynchronous 34 | void getAsyncInputCandidates(Scheduler& scheduler); 35 | }; 36 | //--------------------------------------------------------------------------- 37 | #endif 38 | -------------------------------------------------------------------------------- /include/trident/ml/distmultester.h: -------------------------------------------------------------------------------- 1 | #ifndef _DISTMUL_TESTER_H 2 | #define _DISTMUL_TESTER_H 3 | 4 | #include 5 | 6 | template 7 | class DistMulTester : public Tester { 8 | public: 9 | DistMulTester(std::shared_ptr> E, 10 | std::shared_ptr> R) : Tester(E, R) { 11 | } 12 | 13 | double closeness(K *v1, uint64_t entity, uint16_t dim) { 14 | double res = 0; 15 | Embeddings *pE = (this->E).get(); 16 | K* v2 = pE->get(entity); 17 | for(uint16_t i = 0; i < dim; ++i) { 18 | res += v1[i] * v2[i]; 19 | } 20 | return -res; 21 | } 22 | 23 | void predictO(uint64_t sub, uint16_t dims, uint64_t pred, uint16_t dimp, K* o) { 24 | Embeddings *pE = (this->E).get(); 25 | Embeddings *pR = (this->R).get(); 26 | K* s = pE->get(sub); 27 | K* p = pR->get(pred); 28 | for (uint16_t i = 0; i < dims; ++i) { 29 | o[i] = s[i] * p[i]; 30 | } 31 | } 32 | 33 | void predictS(K *s, uint64_t pred, uint16_t dimp, uint64_t obj, uint16_t dimo) { 34 | Embeddings *pE = (this->E).get(); 35 | Embeddings *pR = (this->R).get(); 36 | K* o = pE->get(obj); 37 | K* p = pR->get(pred); 38 | for (uint16_t i = 0; i < dimo; ++i) { 39 | s[i] = o[i] * p[i]; 40 | } 41 | } 42 | }; 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /rdf3x/src/infra/Mutex.cpp: -------------------------------------------------------------------------------- 1 | #include "infra/Config.hpp" 2 | #ifdef CONFIG_WINDOWS 3 | #define _WIN32_WINNT 0x400 4 | #endif 5 | #include "infra/osdep/Mutex.hpp" 6 | //--------------------------------------------------------------------------- 7 | Mutex::Mutex() 8 | // Constructor 9 | { 10 | #ifdef CONFIG_WINDOWS 11 | InitializeCriticalSection(&mutex); 12 | #else 13 | pthread_mutex_init(&mutex,0); 14 | #endif 15 | } 16 | //--------------------------------------------------------------------------- 17 | Mutex::~Mutex() 18 | /// Destructor 19 | { 20 | #ifdef CONFIG_WINDOWS 21 | DeleteCriticalSection(&mutex); 22 | #else 23 | pthread_mutex_destroy(&mutex); 24 | #endif 25 | } 26 | //--------------------------------------------------------------------------- 27 | void Mutex::lock() 28 | // Lock 29 | { 30 | #ifdef CONFIG_WINDOWS 31 | EnterCriticalSection(&mutex); 32 | #else 33 | pthread_mutex_lock(&mutex); 34 | #endif 35 | } 36 | //--------------------------------------------------------------------------- 37 | bool Mutex::tryLock() 38 | // Try to lock 39 | { 40 | #ifdef CONFIG_WINDOWS 41 | return TryEnterCriticalSection(&mutex); 42 | #else 43 | return pthread_mutex_trylock(&mutex)==0; 44 | #endif 45 | } 46 | //--------------------------------------------------------------------------- 47 | void Mutex::unlock() 48 | // Unlock 49 | { 50 | #ifdef CONFIG_WINDOWS 51 | LeaveCriticalSection(&mutex); 52 | #else 53 | pthread_mutex_unlock(&mutex); 54 | #endif 55 | } 56 | //--------------------------------------------------------------------------- 57 | -------------------------------------------------------------------------------- /include/trident/tree/treeitr.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Jacopo Urbani 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, 15 | * software distributed under the License is distributed on an 16 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | * KIND, either express or implied. See the License for the 18 | * specific language governing permissions and limitations 19 | * under the License. 20 | **/ 21 | 22 | 23 | #ifndef TREEITR_H_ 24 | #define TREEITR_H_ 25 | 26 | #include 27 | 28 | class TermCoordinates; 29 | class Leaf; 30 | class Node; 31 | class Cache; 32 | 33 | class TreeItr { 34 | private: 35 | Node *root; 36 | Leaf *currentLeaf; 37 | int currentPos; 38 | //Cache *cache; 39 | 40 | public: 41 | TreeItr() {} 42 | 43 | TreeItr(Node *root, Leaf *firstLeaf); 44 | 45 | virtual bool hasNext(); 46 | 47 | virtual int64_t next(TermCoordinates *value); 48 | 49 | int64_t next(int64_t &value); 50 | 51 | virtual ~TreeItr() {} 52 | }; 53 | 54 | #endif /* TREEITR_H_ */ 55 | -------------------------------------------------------------------------------- /include/trident/tree/flattreeitr.h: -------------------------------------------------------------------------------- 1 | #ifndef _FLAT_TREEITR_H 2 | #define _FLAT_TREEITR_H 3 | 4 | #include 5 | 6 | class FlatTreeItr: public TreeItr { 7 | private: 8 | char *start; 9 | const char *end; 10 | const int sizeblock; 11 | int64_t idx; 12 | const bool unlabeled; 13 | const bool undirected; 14 | 15 | public: 16 | FlatTreeItr(char *start, char *end, int sizeblock, 17 | bool unlabeled, bool undirected) : start(start), 18 | end(end), sizeblock(sizeblock), idx(0), unlabeled(unlabeled), 19 | undirected(undirected) {} 20 | 21 | bool hasNext() { 22 | return start < end; 23 | } 24 | 25 | int64_t next(TermCoordinates *value) { 26 | if (start < end) { 27 | value->clear(); 28 | FlatRoot::__set(IDX_SOP, start + 5, value); 29 | if (!unlabeled || !undirected) { 30 | FlatRoot::__set(IDX_OSP, start + 18, value); 31 | } 32 | if (!unlabeled) { 33 | FlatRoot::__set(IDX_SPO, start + 31, value); 34 | FlatRoot::__set(IDX_OPS, start + 44, value); 35 | FlatRoot::__set(IDX_POS, start + 57, value); 36 | FlatRoot::__set(IDX_PSO, start + 70, value); 37 | } 38 | 39 | start += sizeblock; 40 | return idx++; 41 | } else { 42 | return 0; 43 | } 44 | } 45 | }; 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /rdf3x/include/rts/operator/SingletonScan.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_rts_operator_SingletonScan 2 | #define H_rts_operator_SingletonScan 3 | //--------------------------------------------------------------------------- 4 | // RDF-3X 5 | // (c) 2008 Thomas Neumann. Web site: http://www.mpi-inf.mpg.de/~neumann/rdf3x 6 | // 7 | // This work is licensed under the Creative Commons 8 | // Attribution-Noncommercial-Share Alike 3.0 Unported License. To view a copy 9 | // of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ 10 | // or send a letter to Creative Commons, 171 Second Street, Suite 300, 11 | // San Francisco, California, 94105, USA. 12 | //--------------------------------------------------------------------------- 13 | #include "rts/operator/Operator.hpp" 14 | //--------------------------------------------------------------------------- 15 | /// A scan over a single empty tuple 16 | class SingletonScan : public Operator 17 | { 18 | public: 19 | /// Constructor 20 | SingletonScan(); 21 | /// Destructor 22 | ~SingletonScan(); 23 | 24 | /// Produce the first tuple 25 | uint64_t first(); 26 | /// Produce the next tuple 27 | uint64_t next(); 28 | 29 | /// Print the operator tree. Debugging only. 30 | void print(PlanPrinter& out); 31 | /// Add a merge join hint 32 | void addMergeHint(Register* reg1,Register* reg2); 33 | /// Register parts of the tree that can be executed asynchronous 34 | void getAsyncInputCandidates(Scheduler& scheduler); 35 | }; 36 | //--------------------------------------------------------------------------- 37 | #endif 38 | -------------------------------------------------------------------------------- /include/trident/kb/statistics.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Jacopo Urbani 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, 15 | * software distributed under the License is distributed on an 16 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | * KIND, either express or implied. See the License for the 18 | * specific language governing permissions and limitations 19 | * under the License. 20 | **/ 21 | 22 | 23 | #ifndef _STATS_H 24 | #define _STATS_H 25 | 26 | 27 | class Stats { 28 | private: 29 | int64_t readIndexBlocks; 30 | int64_t readIndexBytes; 31 | public: 32 | 33 | Stats() : readIndexBlocks(0), readIndexBytes(0) {} 34 | 35 | void incrNReadIndexBlocks() { 36 | readIndexBlocks++; 37 | } 38 | 39 | void addNReadIndexBytes(const uint64_t bytes) { 40 | readIndexBytes += bytes; 41 | } 42 | 43 | uint64_t getNReadIndexBlocks() const { 44 | return readIndexBlocks; 45 | } 46 | 47 | uint64_t getNReadIndexBytes() const { 48 | return readIndexBytes; 49 | } 50 | }; 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /src/trident/ml/pairwiselearner.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | using namespace std; 11 | 12 | void PairwiseLearner::gen_random( 13 | Querier *q, 14 | BatchIO &io, 15 | std::vector &input, 16 | const bool subjObjs, 17 | const uint16_t ntries) { 18 | for(uint32_t i = 0; i < input.size(); ++i) { 19 | int64_t s, p, o; 20 | s = io.field1[i]; 21 | p = io.field2[i]; 22 | o = io.field3[i]; 23 | uint16_t attemptId; 24 | for(attemptId = 0; attemptId < ntries; ++attemptId) { 25 | input[i] = dis(gen); 26 | //Check if the resulting triple is existing ... 27 | if (subjObjs) { 28 | s = input[i]; 29 | } else { 30 | o = input[i]; 31 | } 32 | if (!q->exists(s, p, o)) { 33 | break; 34 | } 35 | } 36 | } 37 | } 38 | 39 | void PairwiseLearner::process_batch(BatchIO &io, const uint32_t epoch, 40 | const uint16_t nbatches) { 41 | //Generate negative samples 42 | std::vector oneg; 43 | std::vector sneg; 44 | uint32_t sizebatch = io.field1.size(); 45 | oneg.resize(sizebatch); 46 | sneg.resize(sizebatch); 47 | gen_random(io.q, io, sneg, true, 10); 48 | gen_random(io.q, io, oneg, false, 10); 49 | process_batch_withnegs(io, oneg, sneg); 50 | } 51 | -------------------------------------------------------------------------------- /rapidjson/include/rapidjson/internal/swap.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_INTERNAL_SWAP_H_ 16 | #define RAPIDJSON_INTERNAL_SWAP_H_ 17 | 18 | #include "../rapidjson.h" 19 | 20 | #if defined(__clang__) 21 | RAPIDJSON_DIAG_PUSH 22 | RAPIDJSON_DIAG_OFF(c++98-compat) 23 | #endif 24 | 25 | RAPIDJSON_NAMESPACE_BEGIN 26 | namespace internal { 27 | 28 | //! Custom swap() to avoid dependency on C++ header 29 | /*! \tparam T Type of the arguments to swap, should be instantiated with primitive C++ types only. 30 | \note This has the same semantics as std::swap(). 31 | */ 32 | template 33 | inline void Swap(T& a, T& b) RAPIDJSON_NOEXCEPT { 34 | T tmp = a; 35 | a = b; 36 | b = tmp; 37 | } 38 | 39 | } // namespace internal 40 | RAPIDJSON_NAMESPACE_END 41 | 42 | #if defined(__clang__) 43 | RAPIDJSON_DIAG_POP 44 | #endif 45 | 46 | #endif // RAPIDJSON_INTERNAL_SWAP_H_ 47 | -------------------------------------------------------------------------------- /rdf3x/include/infra/osdep/MemoryMappedFile.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_infra_osdep_MemoryMappedFile 2 | #define H_infra_osdep_MemoryMappedFile 3 | //--------------------------------------------------------------------------- 4 | // RDF-3X 5 | // (c) 2008 Thomas Neumann. Web site: http://www.mpi-inf.mpg.de/~neumann/rdf3x 6 | // 7 | // This work is licensed under the Creative Commons 8 | // Attribution-Noncommercial-Share Alike 3.0 Unported License. To view a copy 9 | // of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ 10 | // or send a letter to Creative Commons, 171 Second Street, Suite 300, 11 | // San Francisco, California, 94105, USA. 12 | //---------------------------------------------------------------------------- 13 | /// Maps a file read-only into memory 14 | class MemoryMappedFile 15 | { 16 | private: 17 | /// os dependent data 18 | struct Data; 19 | 20 | /// os dependen tdata 21 | Data* data; 22 | /// Begin of the file 23 | const char* begin; 24 | /// End of the file 25 | const char* end; 26 | 27 | public: 28 | /// Constructor 29 | MemoryMappedFile(); 30 | /// Destructor 31 | ~MemoryMappedFile(); 32 | 33 | /// Open 34 | bool open(const char* name); 35 | /// Close 36 | void close(); 37 | 38 | /// Get the begin 39 | const char* getBegin() const { return begin; } 40 | /// Get the end 41 | const char* getEnd() const { return end; } 42 | 43 | /// Ask the operating system to prefetch a part of the file 44 | void prefetch(const char* start,const char* end); 45 | }; 46 | //---------------------------------------------------------------------------- 47 | #endif 48 | -------------------------------------------------------------------------------- /rdf3x/src/rts/ValuesScan.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | ValuesScan::ValuesScan(std::vector regs, 5 | const std::vector &values, 6 | double expectedOutputCardinality) : Operator(expectedOutputCardinality), 7 | regs(regs), values(values) { 8 | if (regs.size() > 0) 9 | toBeProcessed = values.size() / regs.size(); 10 | else 11 | toBeProcessed = 0; 12 | processed = 0; 13 | } 14 | 15 | /// Destructor 16 | ValuesScan::~ValuesScan() { 17 | } 18 | 19 | /// Produce the first tuple 20 | uint64_t ValuesScan::first() { 21 | if (processed < toBeProcessed) { 22 | for (auto v : regs) { 23 | v->value = values[processed++]; 24 | } 25 | return 1; 26 | } else { 27 | return 0; 28 | } 29 | } 30 | 31 | /// Produce the next tuple 32 | uint64_t ValuesScan::next() { 33 | if (processed < toBeProcessed) { 34 | for (auto v : regs) { 35 | v->value = values[processed++]; 36 | } 37 | return 1; 38 | } else { 39 | return 0; 40 | } 41 | 42 | } 43 | 44 | /// Print the operator tree. Debugging only. 45 | void ValuesScan::print(PlanPrinter& out) { 46 | out.beginOperator("VALUES",expectedOutputCardinality, processed); 47 | out.endOperator(); 48 | } 49 | 50 | /// Add a merge join hint 51 | void ValuesScan::addMergeHint(Register* reg1,Register* reg2) { 52 | } 53 | 54 | /// Register parts of the tree that can be executed asynchronous 55 | void ValuesScan::getAsyncInputCandidates(Scheduler& scheduler) { 56 | } 57 | -------------------------------------------------------------------------------- /snap/snap-core/triad.cpp: -------------------------------------------------------------------------------- 1 | namespace TSnap { 2 | 3 | int GetCommon(TIntV& A, TIntV& B) { 4 | int i, j; 5 | int ret = 0; 6 | int alen, blen; 7 | int d; 8 | TInt ai; 9 | 10 | alen = A.Len(); 11 | blen = B.Len(); 12 | i = 0; 13 | j = 0; 14 | if (i >= alen || j >= blen) { 15 | return ret; 16 | } 17 | 18 | while (1) { 19 | d = A[i] - B[j]; 20 | if (d < 0) { 21 | i++; 22 | if (i >= alen) { 23 | break; 24 | } 25 | } else if (d > 0) { 26 | j++; 27 | if (j >= blen) { 28 | break; 29 | } 30 | } else { 31 | ret++; 32 | i++; 33 | if (i >= alen) { 34 | break; 35 | } 36 | j++; 37 | if (j >= blen) { 38 | break; 39 | } 40 | } 41 | } 42 | return ret; 43 | } 44 | 45 | int64_t GetCommon(const std::vector< int64_t>& A, const std::vector< int64_t>& B) { 46 | int64_t i, j; 47 | int64_t ret = 0; 48 | int64_t alen, blen; 49 | int64_t d; 50 | int64_t ai; 51 | 52 | alen = A.size(); 53 | blen = B.size(); 54 | i = 0; 55 | j = 0; 56 | if (i >= alen || j >= blen) { 57 | return ret; 58 | } 59 | 60 | while (1) { 61 | d = A[i] - B[j]; 62 | if (d < 0) { 63 | i++; 64 | if (i >= alen) { 65 | break; 66 | } 67 | } else if (d > 0) { 68 | j++; 69 | if (j >= blen) { 70 | break; 71 | } 72 | } else { 73 | ret++; 74 | i++; 75 | if (i >= alen) { 76 | break; 77 | } 78 | j++; 79 | if (j >= blen) { 80 | break; 81 | } 82 | } 83 | } 84 | return ret; 85 | } 86 | 87 | } // namespace TSnap 88 | -------------------------------------------------------------------------------- /tests/test_index.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "../src/tree/root.h" 5 | #include "../src/tree/coordinates.h" 6 | #include "../src/utils/propertymap.h" 7 | #include 8 | #include 9 | 10 | using namespace std; 11 | namespace timens = std::chrono; 12 | 13 | int main(int argc, const char** argv) { 14 | 15 | //Create the tree 16 | PropertyMap config; 17 | config.setBool(TEXT_KEYS, false); 18 | config.setBool(TEXT_VALUES, false); 19 | config.setBool(COMPRESSED_NODES, false); 20 | config.setInt(MAX_EL_PER_NODE, 2048); 21 | config.setInt(FILE_MAX_SIZE, 64 * 1024 * 1024); 22 | config.setLong(CACHE_MAX_SIZE, 1024 * 1024 * 1024); 23 | config.setInt(NODE_MIN_BYTES, 1); 24 | config.setInt(MAX_NODES_IN_CACHE, 100); 25 | 26 | config.setInt(LEAF_SIZE_FACTORY, 2050); 27 | config.setInt(LEAF_SIZE_PREALL_FACTORY, 2050); 28 | config.setInt(LEAF_ARRAYS_FACTORY_SIZE, 10); 29 | config.setInt(LEAF_ARRAYS_PREALL_FACTORY_SIZE, 2050); 30 | config.setInt(LEAF_MAX_INTERNAL_LINES, 2050); 31 | config.setInt(LEAF_MAX_PREALL_INTERNAL_LINES, 5000); 32 | config.setInt(NODE_KEYS_FACTORY_SIZE, 10); 33 | config.setInt(NODE_KEYS_PREALL_FACTORY_SIZE, 2050); 34 | 35 | Root *root = new Root("/Users/jacopo/Desktop/kb/tree", NULL, false, config); 36 | TreeItr *itr = root->itr(); 37 | TermCoordinates t; 38 | int64_t count = 0; 39 | int keys = 0; 40 | while (itr->hasNext()) { 41 | keys = (keys+1) % 100; 42 | int64_t k = itr->next(&t); 43 | if (keys == 0) { 44 | cout << k << endl; 45 | } 46 | ++count; 47 | } 48 | cout << count << endl; 49 | 50 | delete itr; 51 | delete root; 52 | return 0; 53 | } 54 | -------------------------------------------------------------------------------- /include/trident/utils/fft.h: -------------------------------------------------------------------------------- 1 | #ifndef _FFT_H 2 | #define _FFT_H 3 | 4 | #include 5 | typedef struct Complex { 6 | 7 | public: 8 | double real; 9 | double imag; 10 | 11 | Complex() { 12 | real = 0.0; 13 | imag = 0.0; 14 | } 15 | 16 | Complex(double r, double i) { 17 | real = r; 18 | imag = i; 19 | } 20 | 21 | Complex& operator +=(Complex c) { 22 | this->real += c.real; 23 | this->imag += c.imag; 24 | return *this; 25 | } 26 | 27 | Complex& operator = (Complex c) { 28 | this->real = c.real; 29 | this->imag = c.imag; 30 | return *this; 31 | } 32 | 33 | Complex operator * (Complex c) { 34 | return Complex((this->real * c.real) - (this->imag * c.imag), (this->real * c.imag) + (this->imag * c.real)); 35 | } 36 | 37 | Complex& operator *= (Complex c) { 38 | this->real = (this->real * c.real) - (this->imag * c.imag); 39 | this->imag = (this->real * c.imag) + (this->imag * c.real); 40 | return *this; 41 | } 42 | 43 | Complex operator ~() { 44 | return Complex(this->real, -this->imag); 45 | } 46 | 47 | Complex operator / (double d) { 48 | return Complex(this->real/d, -this->imag/d); 49 | } 50 | 51 | } Complex; 52 | 53 | 54 | void ccorr(double* , double* , uint16_t , std::vector&); 55 | void ccorr(double* , double* , uint16_t , std::vector&); 56 | void cconv(double* , double* , uint16_t , std::vector&); 57 | void cconv(double* , double* , uint16_t , std::vector&); 58 | double sigmoid(double x); 59 | double sigmoid_given_fun(double x); 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /snap/Makefile.config: -------------------------------------------------------------------------------- 1 | # 2 | # Global configuration for SNAP makefiles 3 | # 4 | 5 | GLIB = glib-core 6 | SNAP = snap-core 7 | GLIBADV = glib-adv 8 | SNAPADV = snap-adv 9 | SNAPEXP = snap-exp 10 | 11 | CGLIB = ../$(GLIB) 12 | CSNAP = ../$(SNAP) 13 | 14 | EXGLIB = ../../$(GLIB) 15 | EXSNAP = ../../$(SNAP) 16 | EXGLIBADV = ../../$(GLIBADV) 17 | EXSNAPADV = ../../$(SNAPADV) 18 | EXSNAPEXP = ../../$(SNAPEXP) 19 | 20 | UNAME := $(shell uname) 21 | 22 | ifeq ($(UNAME), Linux) 23 | # Linux flags 24 | CC = g++ 25 | CXXFLAGS += -std=c++11 -fpermissive -fPIC 26 | CXXFLAGS += -static -O3 -DNDEBUG -fopenmp 27 | # turn on for crash debugging, get symbols with 2>&1 | c++filt 28 | #CXXFLAGS += -g -rdynamic 29 | #CXXFLAGS += -ggdb 30 | # turn on for OpenMP 31 | #CXXOPENMP = -fopenmp 32 | LDFLAGS += 33 | LIBS += -lrt 34 | 35 | else ifeq ($(UNAME), Darwin) 36 | # OS X flags 37 | CC = g++ 38 | #CXXFLAGS += -std=c++98 -Wall 39 | CXXFLAGS += -std=c++11 -Wall 40 | CXXFLAGS += -O3 -DNDEBUG 41 | CXXFLAGS += $(MOREFLAGS) 42 | CLANG := $(shell g++ -v 2>&1 | grep clang | cut -d " " -f 2) 43 | ifneq ($(CLANG), LLVM) 44 | ifneq ($(CLANG), clang) 45 | CXXFLAGS += -fopenmp 46 | #CXXOPENMP += -fopenmp 47 | else 48 | CXXFLAGS += -DNOMP 49 | CXXOPENMP = 50 | endif 51 | else 52 | CXXFLAGS += -DNOMP 53 | CXXOPENMP = 54 | endif 55 | # LDFLAGS += -L$(TRID_LIBPATH) -L$(KOG_LIBPATH) 56 | # LIBS += $(TRID_DEP) 57 | 58 | else ifeq ($(shell uname -o), Cygwin) 59 | # Cygwin flags 60 | CC = g++ 61 | CXXFLAGS += -Wall -D__STDC_LIMIT_MACROS 62 | CXXFLAGS += -O3 -DNDEBUG 63 | CXXOPENMP = -fopenmp 64 | LDFLAGS += 65 | LIBS += 66 | 67 | endif 68 | 69 | -------------------------------------------------------------------------------- /include/trident/binarytables/newrowtableinserter.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Jacopo Urbani 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, 15 | * software distributed under the License is distributed on an 16 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | * KIND, either express or implied. See the License for the 18 | * specific language governing permissions and limitations 19 | * under the License. 20 | **/ 21 | 22 | 23 | #ifndef _NEWROWTABLEINSERTER_H 24 | #define _NEWROWTABLEINSERTER_H 25 | 26 | #include 27 | #include 28 | 29 | class NewRowTableInserter: public BinaryTableInserter { 30 | private: 31 | char reader1, reader2; 32 | 33 | void writeFirstTerm(int64_t t1); 34 | void writeSecondTerm(int64_t t2); 35 | 36 | public: 37 | 38 | int getType() { 39 | return NEWROW_ITR; 40 | } 41 | 42 | void setReaderSizes(char reader1, char reader2) { 43 | this->reader1 = reader1; 44 | this->reader2 = reader2; 45 | } 46 | 47 | void startAppend(); 48 | 49 | void append(int64_t t1, int64_t t2); 50 | 51 | void stopAppend(); 52 | 53 | }; 54 | #endif 55 | -------------------------------------------------------------------------------- /src/trident/kb/partial.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | Partial::Partial(int idx, std::string baseDir) { 8 | // Read index if present 9 | std::string ifile = baseDir + DIR_SEP + "partialIndex" + std::to_string(idx); 10 | std::string dfile = baseDir + DIR_SEP + "partialData" + std::to_string(idx); 11 | std::ifstream is(ifile, std::ios::in | std::ios::binary); 12 | if (is.is_open()) { 13 | STriple t; 14 | uint64_t off; 15 | 16 | while (! is.eof()) { 17 | is.read(reinterpret_cast(&t.s), sizeof(int64_t)); 18 | is.read(reinterpret_cast(&t.p), sizeof(int64_t)); 19 | is.read(reinterpret_cast(&t.o), sizeof(int64_t)); 20 | is.read(reinterpret_cast(&off), sizeof(uint64_t)); 21 | index[t] = off; 22 | } 23 | is.close(); 24 | } 25 | indexStream.open(ifile, std::ios::app | std::ios::binary); 26 | if (! indexStream.is_open()) { 27 | abort(); 28 | } 29 | dataStream.open(dfile, std::ios::in|std::ios::app|std::ios::binary); 30 | if (! dataStream.is_open()) { 31 | abort(); 32 | } 33 | } 34 | 35 | void Partial::dump(ReOrderItr *it) { 36 | it->dump(indexStream, dataStream, index); 37 | } 38 | 39 | ReOrderItr *Partial::getIterator(Querier *q, const int idx, const int64_t s, const int64_t p, const int64_t o) { 40 | STriple triple(s, p, o); 41 | auto i = index.find(triple); 42 | if (i != index.end()) { 43 | return new ReOrderItr(q, idx, s, p, o, dataStream, i->second); 44 | } 45 | return NULL; 46 | } 47 | -------------------------------------------------------------------------------- /include/trident/ml/transetester.h: -------------------------------------------------------------------------------- 1 | #ifndef _TRANSE_TESTER_H 2 | #define _TRANSE_TESTER_H 3 | 4 | #include 5 | 6 | template 7 | class TranseTester : public Tester { 8 | public: 9 | TranseTester(std::shared_ptr> E, 10 | std::shared_ptr> R, Querier* q) : Tester(E, R, q) { 11 | } 12 | 13 | TranseTester(std::shared_ptr> E, 14 | std::shared_ptr> R) : Tester(E, R) { 15 | } 16 | 17 | double closeness(K *v1, uint64_t entity, uint16_t dim) { 18 | double res = 0; 19 | Embeddings *pE = (this->E).get(); 20 | K* v2 = pE->get(entity); 21 | for(uint16_t i = 0; i < dim; ++i) { 22 | res += abs(v1[i] - v2[i]); 23 | } 24 | return res; 25 | } 26 | 27 | void predictO(uint64_t sub, uint16_t dims, uint64_t pred, uint16_t dimp, K* o) { 28 | Embeddings *pE = (this->E).get(); 29 | Embeddings *pR = (this->R).get(); 30 | K* s = pE->get(sub); 31 | K* p = pR->get(pred); 32 | for (uint16_t i = 0; i < dims; ++i) { 33 | o[i] = s[i] + p[i]; 34 | } 35 | } 36 | 37 | void predictS(K *s, uint64_t pred, uint16_t dimp, uint64_t obj, uint16_t dimo) { 38 | Embeddings *pE = (this->E).get(); 39 | Embeddings *pR = (this->R).get(); 40 | K* o = pE->get(obj); 41 | K* p = pR->get(pred); 42 | for (uint16_t i = 0; i < dimo; ++i) { 43 | s[i] = o[i] - p[i]; 44 | } 45 | } 46 | }; 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /snap/snap-core/gviz.cpp: -------------------------------------------------------------------------------- 1 | ///////////////////////////////////////////////// 2 | // GraphViz 3 | namespace TSnap { 4 | namespace TSnapDetail { 5 | void GVizDoLayout(const TStr& GraphInFNm, TStr OutFNm, const TGVizLayout& Layout) { 6 | TStr LayoutExe = TSnap::TSnapDetail::GVizGetLayoutStr(Layout), Ext = OutFNm.GetFExt(), GvPath; 7 | #if defined(GLib_WIN) 8 | GvPath = "C:\\Prog\\GraphViz\\bin\\"; 9 | #else 10 | GvPath = "/usr/bin/"; 11 | //OutFNm = OutFNm.GetFMid() + Ext; 12 | #endif 13 | IAssert(Ext==".ps" || Ext==".gif" || Ext==".png"); 14 | const TStr ExeCmd = TStr::Fmt("%s -T%s %s -o %s", LayoutExe.CStr(), 15 | Ext.CStr()+1, GraphInFNm.CStr(), OutFNm.CStr()); 16 | 17 | if (system(ExeCmd.CStr())==0) { return; } 18 | #if defined(GLib_WIN) 19 | if (system(TStr::Fmt(".\\%s", ExeCmd.CStr()).CStr())==0) { return; } 20 | #else 21 | if (system(TStr::Fmt("./%s", ExeCmd.CStr()).CStr())==0) { return; } 22 | #endif 23 | if (system(TStr::Fmt("%s%s", GvPath.CStr(), ExeCmd.CStr()).CStr())==0) { return; } 24 | fprintf(stderr, "[%s:%d] Cannot find GraphViz (%s). Set the PATH.\n", __FILE__, __LINE__, ExeCmd.CStr()); 25 | //#if defined(GLib_WIN) 26 | //if (ShowPlot) system(TStr::Fmt("start %s", OutFNm.CStr()).CStr()); 27 | //#endif 28 | } 29 | 30 | TStr GVizGetLayoutStr(const TGVizLayout& Layout) { 31 | switch(Layout) { 32 | case gvlDot : return "dot"; 33 | case gvlNeato : return "neato"; 34 | case gvlTwopi : return "twopi"; 35 | case gvlCirco: return "circo"; 36 | case gvlSfdp: return "sfdp"; 37 | default: Fail; 38 | } 39 | return TStr::GetNullStr(); 40 | } 41 | 42 | } // namespace TSnapDetail 43 | } // namespace TSnap 44 | 45 | -------------------------------------------------------------------------------- /tests/test_transe3.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | #define DIMS 50 14 | 15 | using namespace std; 16 | 17 | int main(int argc, const char** argv) { 18 | char buffer[24]; 19 | ifstream ifs; 20 | ifs.open(argv[1]); 21 | //Load the entity and relation embeddings 22 | ifs.read(buffer, 4); 23 | int nents = *(int*)buffer; 24 | std::vector embe_old; 25 | for(int j = 0; j < nents * DIMS; ++j) { 26 | ifs.read(buffer, 8); 27 | double v = *(double*)(buffer); 28 | embe_old.push_back(v); 29 | } 30 | ifs.read(buffer, 4); 31 | int nrels = *(int*)buffer; 32 | std::vector embr_old; 33 | for(int j = 0; j < nrels * DIMS; ++j) { 34 | ifs.read(buffer, 8); 35 | double v = *(double*)(buffer); 36 | embr_old.push_back(v); 37 | } 38 | ifs.close(); 39 | //Load embeddings 40 | std::shared_ptr> E = std::shared_ptr>(new Embeddings(embe_old.size() / DIMS, DIMS, embe_old)); 41 | std::shared_ptr> R = std::shared_ptr>(new Embeddings(embr_old.size() / DIMS, DIMS, embr_old)); 42 | std::unique_ptr pe2; 43 | std::unique_ptr pr2; 44 | 45 | //Test! 46 | std::vector testset; 47 | BatchCreator::loadTriples(argv[2], testset); 48 | TranseTester tester(E, R); 49 | auto result = tester.test("valid", testset, 1, 0); 50 | 51 | return 0; 52 | } 53 | -------------------------------------------------------------------------------- /rdf3x/include/infra/osdep/Mutex.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_infra_osdep_Mutex 2 | #define H_infra_osdep_Mutex 3 | //--------------------------------------------------------------------------- 4 | #include "infra/Config.hpp" 5 | //--------------------------------------------------------------------------- 6 | #ifdef CONFIG_WINDOWS 7 | #define WIN32_LEAN_AND_MEAN 8 | #include 9 | #else 10 | #include 11 | #endif 12 | //--------------------------------------------------------------------------- 13 | class Event; 14 | //--------------------------------------------------------------------------- 15 | /// Mutex 16 | /** System dependent. Under UNIX a thread cannot lock the same mutex swice. 17 | */ 18 | class Mutex 19 | { 20 | private: 21 | #ifdef CONFIG_WINDOWS 22 | CRITICAL_SECTION mutex; 23 | #else 24 | pthread_mutex_t mutex; 25 | #endif 26 | 27 | // Allow Event to access the handle for pthreads 28 | friend class Event; 29 | 30 | Mutex(const Mutex&); 31 | void operator=(const Mutex&); 32 | 33 | public: 34 | /// Constructor 35 | Mutex(); 36 | /// Destructor 37 | ~Mutex(); 38 | 39 | /// Lock the mutex 40 | void lock(); 41 | /// Try to lock the mutex 42 | bool tryLock(); 43 | /// Unlock the mutex 44 | void unlock(); 45 | }; 46 | //--------------------------------------------------------------------------- 47 | /// Locker object 48 | class auto_lock 49 | { 50 | private: 51 | Mutex& lock; 52 | 53 | auto_lock(const auto_lock&); 54 | void operator=(const auto_lock&); 55 | 56 | public: 57 | auto_lock(Mutex& l) : lock(l) { l.lock(); } 58 | ~auto_lock() { lock.unlock(); } 59 | }; 60 | //--------------------------------------------------------------------------- 61 | #endif 62 | -------------------------------------------------------------------------------- /webinterface/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 14 | 16 | 17 | Trident Web Interface 18 | 19 | 20 | 21 |
22 | 26 | 27 | 28 |
29 |
30 | 31 | 32 |
33 |
34 | 35 | 50 |
51 | 52 | 53 | -------------------------------------------------------------------------------- /include/trident/binarytables/newtable.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Jacopo Urbani 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, 15 | * software distributed under the License is distributed on an 16 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | * KIND, either express or implied. See the License for the 18 | * specific language governing permissions and limitations 19 | * under the License. 20 | **/ 21 | 22 | 23 | #ifndef _NEWTABLE_H 24 | #define _NEWTABLE_H 25 | 26 | #include 27 | 28 | 29 | 30 | class AbsNewTable : public PairItr { 31 | public: 32 | virtual char getReaderSize1() const = 0; 33 | 34 | virtual char getReaderSize2() const = 0; 35 | 36 | virtual char getReaderCountSize() const = 0; 37 | 38 | virtual void setup(const char* start, const char *end) = 0; 39 | 40 | virtual void setup(int64_t c1, const char* start, const char *end) = 0; 41 | 42 | virtual void setup(int64_t c1, int64_t c2, const char* start, const char *end) = 0; 43 | 44 | virtual int64_t _getValue1AtRow(const char *start, 45 | const int64_t rowId) const { 46 | throw 10; 47 | } 48 | }; 49 | 50 | #endif 51 | -------------------------------------------------------------------------------- /tests/test_sorting.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | using namespace std; 11 | 12 | void generateTriples(string input, int64_t n) { 13 | LZ4Writer writer(input); 14 | int64_t seed = 0; 15 | for(int64_t j = 0; j < n; ++j) { 16 | int64_t t1 = abs(seed++); 17 | int64_t t2 = abs(seed++); 18 | int64_t t3 = abs(seed++); 19 | writer.writeLong(t1); 20 | writer.writeLong(t2); 21 | writer.writeLong(t3); 22 | if (j % 10000 == 0) { 23 | seed = random() * 10000000000l; 24 | } 25 | if (j % 100000000 == 0) 26 | LOG(DEBUGL) << "Dumped " << j << " triples"; 27 | } 28 | } 29 | 30 | int main(int argc, const char** argv) { 31 | 32 | int64_t n = 100000000000; 33 | 34 | if (false) { 35 | //Generate random input 36 | cout << "Storing the input at " << argv[1] << endl; 37 | if (Utils::exists(argv[1])) { 38 | Utils::remove_all(argv[1]); 39 | } 40 | Utils::create_directories(argv[1]); 41 | 42 | std::thread threads[72]; 43 | for(int i = 0; i < 72; ++i) { 44 | string out = string(argv[1]) + "/input" + to_string(i); 45 | threads[i] = std::thread(generateTriples, out, n / 72); 46 | } 47 | for(int i = 0; i < 72; ++i) { 48 | threads[i].join(); 49 | } 50 | 51 | } 52 | 53 | //Sort 54 | LOG(DEBUGL) << "Start sorting"; 55 | Loader::sortPermutation(string(argv[1]), 8, 72, true, 100000000000l, 30000000000l , 16); 56 | LOG(DEBUGL) << "Stop sorting"; 57 | 58 | return 0; 59 | } 60 | -------------------------------------------------------------------------------- /include/trident/binarytables/rowtableinserter.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Jacopo Urbani 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, 15 | * software distributed under the License is distributed on an 16 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | * KIND, either express or implied. See the License for the 18 | * specific language governing permissions and limitations 19 | * under the License. 20 | **/ 21 | 22 | 23 | #ifndef _ROWTABLEINSERTER_H 24 | #define _ROWTABLEINSERTER_H 25 | 26 | #include 27 | #include 28 | 29 | class RowTableInserter: public BinaryTableInserter { 30 | private: 31 | int comprValue1; 32 | int comprValue2; 33 | int diffValue1; 34 | int64_t previousValue1; 35 | int64_t nElements; 36 | 37 | void writeFirstTerm(int64_t t1); 38 | void writeSecondTerm(int64_t t2); 39 | 40 | public: 41 | 42 | int getType() { 43 | return ROW_ITR; 44 | } 45 | 46 | void setCompressionMode(int v1, int v2); 47 | 48 | void setDifferenceMode(int d1); 49 | 50 | void startAppend(); 51 | 52 | void append(int64_t t1, int64_t t2); 53 | 54 | void stopAppend(); 55 | 56 | }; 57 | #endif 58 | -------------------------------------------------------------------------------- /include/trident/tests/timings.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Jacopo Urbani 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, 15 | * software distributed under the License is distributed on an 16 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | * KIND, either express or implied. See the License for the 18 | * specific language governing permissions and limitations 19 | * under the License. 20 | **/ 21 | 22 | 23 | #ifndef _TIMINGS_H 24 | #define _TIMINGS_H 25 | 26 | #include 27 | 28 | #include 29 | #include 30 | 31 | #define T_SPO 0 32 | #define T_OPS 1 33 | #define T_POS 2 34 | #define T_SOP 3 35 | #define T_OSP 4 36 | #define T_PSO 5 37 | 38 | class Timings { 39 | private: 40 | std::string queryfile; 41 | 42 | public: 43 | Timings(std::string filequeries) : queryfile(filequeries) {} 44 | 45 | virtual void init() = 0; 46 | 47 | virtual std::chrono::duration launchQuery(const int perm, 48 | const int64_t s, 49 | const int64_t p, 50 | const int64_t o, 51 | const int countIgnores, 52 | int64_t &c, 53 | int64_t &junk) = 0; 54 | 55 | LIBEXP void launchTests(); 56 | }; 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /rdf3x/include/infra/osdep/Event.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_infra_osdep_Event 2 | #define H_infra_osdep_Event 3 | //--------------------------------------------------------------------------- 4 | #include 5 | //--------------------------------------------------------------------------- 6 | #ifdef CONFIG_WINDOWS 7 | #define WIN32_LEAN_AND_MEAN 8 | #include 9 | #else 10 | #include 11 | #endif 12 | //--------------------------------------------------------------------------- 13 | class Mutex; 14 | //--------------------------------------------------------------------------- 15 | /// Wait primitive 16 | class Event 17 | { 18 | private: 19 | #ifdef CONFIG_WINDOWS 20 | CRITICAL_SECTION unblockLock; 21 | 22 | /// Auto-Reset event 23 | void* blockLock; 24 | /// Auto-Reset event 25 | void* blockQueueSignal; 26 | /// Manual-Reset event 27 | void* blockQueueBroadcast; 28 | //// Broadcast state 29 | enum { NoBroadcast, Broadcast, BroadcastAfterSignal } broadcast; 30 | /// Counters 31 | int waitersGone,waitersBlocked,waitersToUnblock; 32 | #else 33 | pthread_cond_t condVar; 34 | #endif 35 | 36 | Event(const Event&); 37 | void operator=(const Event&); 38 | 39 | public: 40 | /// Constructor 41 | Event(); 42 | /// Destructor 43 | ~Event(); 44 | 45 | /// Wait for event. The mutex must be locked. 46 | void wait(Mutex& mutex); 47 | /// Wait up to a certain time. The mutex must be locked. 48 | bool timedWait(Mutex& mutex,unsigned timeoutMilli); 49 | /// Notify at least one waiting thread 50 | void notify(Mutex& mutex); 51 | /// Notify all waiting threads 52 | void notifyAll(Mutex& mutex); 53 | }; 54 | //--------------------------------------------------------------------------- 55 | #endif 56 | -------------------------------------------------------------------------------- /rdf3x/include/rts/operator/NestedLoopJoin.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_rts_operator_NestedLoopJoin 2 | #define H_rts_operator_NestedLoopJoin 3 | //--------------------------------------------------------------------------- 4 | // RDF-3X 5 | // (c) 2008 Thomas Neumann. Web site: http://www.mpi-inf.mpg.de/~neumann/rdf3x 6 | // 7 | // This work is licensed under the Creative Commons 8 | // Attribution-Noncommercial-Share Alike 3.0 Unported License. To view a copy 9 | // of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ 10 | // or send a letter to Creative Commons, 171 Second Street, Suite 300, 11 | // San Francisco, California, 94105, USA. 12 | //--------------------------------------------------------------------------- 13 | #include "rts/operator/Operator.hpp" 14 | //--------------------------------------------------------------------------- 15 | /// A nested loop join 16 | class NestedLoopJoin : public Operator 17 | { 18 | private: 19 | /// The input 20 | Operator* left,*right; 21 | /// The count from the left side 22 | uint64_t leftCount; 23 | 24 | public: 25 | /// Constructor 26 | NestedLoopJoin(Operator* left,Operator* right,double expectedOutputCardinality); 27 | /// Destructor 28 | ~NestedLoopJoin(); 29 | 30 | /// Produce the first tuple 31 | uint64_t first(); 32 | /// Produce the next tuple 33 | uint64_t next(); 34 | 35 | /// Print the operator tree. Debugging only. 36 | void print(PlanPrinter& out); 37 | /// Add a merge join hint 38 | void addMergeHint(Register* reg1,Register* reg2); 39 | /// Register parts of the tree that can be executed asynchronous 40 | void getAsyncInputCandidates(Scheduler& scheduler); 41 | }; 42 | //--------------------------------------------------------------------------- 43 | #endif 44 | -------------------------------------------------------------------------------- /tests/test_insert_tree.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | #include 12 | 13 | using namespace std; 14 | 15 | int main(int argc, const char** argv) { 16 | 17 | Logger::setMinLevel(DEBUGL); 18 | //First test all 6 permutations 19 | char *supportBuffer; 20 | supportBuffer = new char[23]; 21 | for(int i = 0; i < 6; ++i) { 22 | string perm = string(argv[2]) + string("/tmpTree") + to_string(i); 23 | ifstream s; 24 | s.open(perm); 25 | LOG(DEBUGL) << perm; 26 | int64_t prevKey = -1; 27 | while (s.read(supportBuffer, 23)) { 28 | short file = Utils::decode_short((const char*) supportBuffer + 16); 29 | int64_t key = Utils::decode_long(supportBuffer); 30 | int64_t nElements = Utils::decode_long(supportBuffer + 8); 31 | int pos = Utils::decode_int(supportBuffer + 18); 32 | char strat = supportBuffer[22]; 33 | if (key <= prevKey) { 34 | LOG(DEBUGL) << "key=" << key << " prevkey=" << prevKey; 35 | } 36 | prevKey = key; 37 | } 38 | s.close(); 39 | } 40 | 41 | KBConfig config; 42 | config.setParamInt(DICTPARTITIONS, 1); 43 | config.setParamInt(NINDICES, 6); 44 | config.setParamBool(AGGRINDICES, false); 45 | config.setParamBool(USEFIXEDSTRAT, false); 46 | config.setParamInt(FIXEDSTRAT, 96); 47 | config.setParamInt(THRESHOLD_SKIP_TABLE, 64); 48 | MemoryOptimizer::optimizeForWriting(10000000000l, config); 49 | KB kb(argv[1], false, false, true, config); 50 | 51 | Loader loader; 52 | loader.testLoadingTree(argv[2], kb.insert(), 6); 53 | } 54 | -------------------------------------------------------------------------------- /rdf3x/include/rts/operator/DuplLimit.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_rts_operator_DuplLimit 2 | #define H_rts_operator_DuplLimit 3 | 4 | #include "rts/operator/Operator.hpp" 5 | #include 6 | //--------------------------------------------------------------------------- 7 | class Register; 8 | class Runtime; 9 | //--------------------------------------------------------------------------- 10 | /// Consumes its input and prints it. Produces a single empty tuple. 11 | class DuplLimit: public Operator { 12 | public: 13 | /// Duplicate handling 14 | enum DuplicateHandling { Duplicates, NoDuplicates }; 15 | 16 | private: 17 | std::vector output; 18 | /// The input 19 | Operator* input; 20 | /// The duplicate handling 21 | DuplicateHandling duplicateHandling; 22 | /// Maximum number of output tuples 23 | uint64_t limit; 24 | uint64_t offset; 25 | uint64_t entryCount, currentCount; 26 | 27 | public: 28 | /// Constructor 29 | DuplLimit(Operator* input, 30 | const std::vector& outputArgs, 31 | DuplicateHandling duplicateHandling, 32 | uint64_t limit = UINT64_MAX, uint64_t offset = 0); 33 | 34 | /// Destructor 35 | ~DuplLimit() { 36 | delete input; 37 | } 38 | 39 | /// Produce the first tuple 40 | uint64_t first(); 41 | /// Produce the next tuple 42 | uint64_t next(); 43 | 44 | /// Print the operator tree. Debugging only. 45 | void print(PlanPrinter& out); 46 | /// Add a merge join hint 47 | void addMergeHint(Register* reg1, Register* reg2); 48 | /// Register parts of the tree that can be executed asynchronous 49 | void getAsyncInputCandidates(Scheduler& scheduler); 50 | }; 51 | //--------------------------------------------------------------------------- 52 | #endif 53 | -------------------------------------------------------------------------------- /snap/snap-core/gbase.cpp: -------------------------------------------------------------------------------- 1 | ///////////////////////////////////////////////// 2 | // Graph Base 3 | namespace TSnap { 4 | 5 | TStr GetFlagStr(const TGraphFlag& GraphFlag) { 6 | switch (GraphFlag) { 7 | case gfUndef : return "Undef"; 8 | case gfDirected : return "Directed"; 9 | case gfMultiGraph : return "Multigraph"; 10 | case gfNodeDat : return "NodeDat"; 11 | case gfEdgeDat : return "EdgeDat"; 12 | case gfSources : return "Sources"; 13 | case gfBipart : return "Bipartite"; 14 | default: FailR("Unknown graph type"); 15 | }; 16 | return TStr(); 17 | } 18 | 19 | }; // namespace TSnap 20 | 21 | ///////////////////////////////////////////////// 22 | // Union Find 23 | int TUnionFind::Find(const int& Key) { 24 | int SetId = Key, parent = Parent(Key); 25 | // find set id 26 | while (parent != -1) { 27 | SetId = parent; 28 | parent = Parent(parent); 29 | } 30 | // flatten 31 | parent = Key; 32 | while (parent != -1) { 33 | const int tmp = Parent(parent); 34 | if (tmp != -1) { Parent(parent) = SetId; } 35 | parent = tmp; 36 | } 37 | return SetId; 38 | } 39 | 40 | void TUnionFind::Union(const int& Key1, const int& Key2) { 41 | const int root1 = Find(Key1); 42 | const int root2 = Find(Key2); 43 | TInt& rank1 = Rank(root1); 44 | TInt& rank2 = Rank(root2); 45 | if (rank1 > rank2) { Parent(root2) = root1; } 46 | else if (rank1 < rank2) { Parent(root1) = root2; } 47 | else if (root1 != root2) { 48 | Parent(root2) = root1; 49 | Rank(root1)++; 50 | } 51 | } 52 | 53 | void TUnionFind::Dump() { 54 | printf(" key\tset\n"); 55 | for (int i = 0; i < KIdSetH.Len(); i++) { 56 | printf(" %d\t%d\n", int(KIdSetH.GetKey(i)), Find(KIdSetH.GetKey(i))); 57 | } 58 | printf("\n"); 59 | } 60 | -------------------------------------------------------------------------------- /tests/test_hashnumbers.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | #include 14 | 15 | 16 | using namespace std; 17 | namespace timens = std::chrono; 18 | 19 | struct PairTermHash 20 | { 21 | template 22 | std::size_t operator() (const std::pair &pair) const 23 | { 24 | return std::hash{}(pair.first) ^ std::hash{}(pair.second); 25 | } 26 | }; 27 | 28 | struct _PairHash { 29 | typedef std::pair argument_type; 30 | typedef std::size_t result_type; 31 | result_type operator()(argument_type const& s) const { 32 | result_type const h1(std::hash{}(s.first)); 33 | result_type const h2(std::hash{}(s.second)); 34 | return h1 ^ (h2 << 1); 35 | } 36 | }; 37 | 38 | int main(int argc, const char** argv) 39 | { 40 | auto h = std::hash>{}(std::make_pair(0, 0)); 41 | std::cout << "*******" << h << "*******"; 42 | int n = 20000; 43 | google::dense_hash_set, _PairHash> cnt; 44 | cnt.set_empty_key(std::make_pair(n+1, n+1)); 45 | 46 | cout << "Testing speed ..." << endl; 47 | std::chrono::system_clock::time_point start = std::chrono::system_clock::now(); 48 | int64_t sum = 0; 49 | for (int i = 0; i < n; ++i) { 50 | cnt.insert(std::make_pair(i, i)); 51 | } 52 | std::chrono::duration sec = std::chrono::system_clock::now() 53 | - start; 54 | cout << "Time: " << sec.count() * 1000 << "ms. Size=" << cnt.size() << std::endl; 55 | return 0; 56 | } 57 | -------------------------------------------------------------------------------- /tests/test_map.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | struct eqint { 7 | bool operator()(int i1, int i2) const { 8 | return i1 == i2; 9 | } 10 | }; 11 | 12 | typedef std::list mylist; 13 | typedef google::dense_hash_map , eqint> map; 14 | 15 | using namespace std; 16 | namespace timens = std::chrono; 17 | 18 | int main(int argc, const char** args) { 19 | //Testing 20 | mylist l; 21 | map m; 22 | m.set_empty_key(-1); 23 | cout << "Insert keys into the map" << endl; 24 | timens::system_clock::time_point start = timens::system_clock::now(); 25 | for (int i = 0; i < 10; ++i) { 26 | unsigned int key = random() % 10; 27 | m.insert(make_pair(key, l.insert(l.end(), i))); 28 | } 29 | std::chrono::duration sec = std::chrono::system_clock::now() 30 | - start; 31 | cout << "Time insert " << m.size() << ": " << (sec.count() * 1000) << " ms" 32 | << endl; 33 | 34 | cout << "List: "; 35 | for (mylist::iterator itr = l.begin(); itr != l.end(); ++itr) { 36 | cout << *itr << " "; 37 | } 38 | cout << endl; 39 | 40 | start = timens::system_clock::now(); 41 | map::iterator itr; 42 | for (int i = 0; i < 10; ++i) { 43 | unsigned int key = random() % 10; 44 | cout << "Query " << key << endl; 45 | 46 | itr = m.find(key); 47 | if (itr != m.end()) { 48 | cout << "Splice" << *(itr->second) << endl; 49 | l.splice(l.end(), l, itr->second); 50 | } else { 51 | cout << "Not found" << endl; 52 | } 53 | 54 | cout << "List: "; 55 | for (mylist::iterator itr = l.begin(); itr != l.end(); ++itr) { 56 | cout << *itr << " "; 57 | } 58 | cout << endl; 59 | } 60 | sec = std::chrono::system_clock::now() - start; 61 | cout << "Time querying: " << (sec.count() * 1000) << " ms " << endl; 62 | } 63 | -------------------------------------------------------------------------------- /rdf3x/include/rts/operator/GroupBy.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_rts_operator_groupby 2 | #define H_rts_operator_groupby 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | 10 | //--------------------------------------------------------------------------- 11 | class GroupBy : public Operator 12 | { 13 | private: 14 | Operator *child; 15 | std::map bindings; 16 | std::vector regs; 17 | const bool distinct; 18 | std::vector>> values; 19 | std::vector keys; 20 | uint64_t index; 21 | std::vector fields; 22 | 23 | bool sameThanPrevious(uint64_t index); 24 | std::vector calculateFieldsToCompare( 25 | std::vector &keys, 26 | std::vector ®s); 27 | 28 | public: 29 | /// Constructor 30 | GroupBy(Operator* child, 31 | std::map bindings, 32 | std::vector regs, 33 | bool distinct, 34 | double expectedOutputCardinality); 35 | 36 | /// Destructor 37 | ~GroupBy(); 38 | 39 | /// Produce the first tuple 40 | uint64_t first(); 41 | /// Produce the next tuple 42 | uint64_t next(); 43 | /// Print the operator tree. Debugging only. 44 | void print(PlanPrinter& out); 45 | /// Add a merge join hint 46 | void addMergeHint(Register* reg1,Register* reg2); 47 | /// Register parts of the tree that can be executed asynchronous 48 | void getAsyncInputCandidates(Scheduler& scheduler); 49 | }; 50 | //--------------------------------------------------------------------------- 51 | #endif 52 | -------------------------------------------------------------------------------- /tests/test_lru.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // test_lru.cpp 3 | // trident 4 | // 5 | // Created by Jacopo Urbani on 15/06/14. 6 | // Copyright (c) 2014 Jacopo Urbani. All rights reserved. 7 | // 8 | 9 | #include "../src/utils/hashmap.h" 10 | #include "../src/utils/utils.h" 11 | 12 | #include 13 | 14 | using namespace std; 15 | 16 | char s[512]; 17 | 18 | void copy_string(char *output, const char* input) { 19 | Utils::encode_short(output, 0, strlen(input)); 20 | memcpy(output+2,input, strlen(input)); 21 | } 22 | 23 | int main(int argc, const char** args) { 24 | LRUByteArraySet set(5,100); 25 | 26 | //Add "a" 27 | cout << "Adding 'a'" << endl; 28 | copy_string(s, "a"); 29 | set.put(s); 30 | cout << set.toString() << endl; 31 | 32 | cout << "Adding 'b'" << endl; 33 | copy_string(s, "b"); 34 | set.put(s); 35 | cout << set.toString() << endl; 36 | 37 | cout << "Adding 'c'" << endl; 38 | copy_string(s, "c"); 39 | set.put(s); 40 | cout << set.toString() << endl; 41 | 42 | cout << "Adding 'd'" << endl; 43 | copy_string(s, "d"); 44 | set.put(s); 45 | cout << set.toString() << endl; 46 | 47 | cout << "Adding 'e'" << endl; 48 | copy_string(s, "e"); 49 | set.put(s); 50 | cout << set.toString() << endl; 51 | 52 | cout << "Adding 'd'" << endl; 53 | copy_string(s, "d"); 54 | set.put(s); 55 | cout << set.toString() << endl; 56 | 57 | cout << "Adding 'b'" << endl; 58 | copy_string(s, "b"); 59 | set.put(s); 60 | cout << set.toString() << endl; 61 | 62 | cout << "Adding 'b'" << endl; 63 | copy_string(s, "b"); 64 | set.put(s); 65 | cout << set.toString() << endl; 66 | 67 | cout << "Adding 'e'" << endl; 68 | copy_string(s, "e"); 69 | set.put(s); 70 | cout << set.toString() << endl; 71 | } -------------------------------------------------------------------------------- /tests/test_stxxl.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * test_stxxl.cpp 3 | * 4 | * Created on: Jan 15, 2014 5 | * Author: jacopo 6 | */ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | typedef struct Triple { 14 | int64_t t1, t2, t3; 15 | 16 | Triple(int64_t s, int64_t p, int64_t o) { 17 | this->t1 = s; 18 | this->t2 = p; 19 | this->t3 = o; 20 | } 21 | 22 | Triple() { 23 | t1 = t2 = t3 = 0; 24 | } 25 | } Triple; 26 | 27 | const Triple minv(std::numeric_limits::min(), 28 | std::numeric_limits::min(), std::numeric_limits::min()); 29 | 30 | const Triple maxv(std::numeric_limits::max(), 31 | std::numeric_limits::max(), std::numeric_limits::max()); 32 | 33 | struct cmp: std::less { 34 | bool operator ()(const Triple& a, const Triple& b) const { 35 | if (a.t1 < b.t1) { 36 | return true; 37 | } else if (a.t1 == b.t1) { 38 | if (a.t2 < b.t2) { 39 | return true; 40 | } else if (a.t2 == b.t2) { 41 | return a.t3 < b.t3; 42 | } 43 | } 44 | return false; 45 | } 46 | 47 | Triple min_value() const { 48 | return minv; 49 | } 50 | 51 | Triple max_value() const { 52 | return maxv; 53 | } 54 | }; 55 | 56 | typedef stxxl::VECTOR_GENERATOR::result vector_type; 57 | 58 | int main(int argc, const char** argv) { 59 | vector_type vector; 60 | vector_type::bufwriter_type writer(vector); 61 | for (int i = 0; i < 1000000000; ++i) { 62 | if (i % 10000000 == 0) 63 | std::cout << "Inserting element " << i << std::endl; 64 | Triple t; 65 | t.t1 = rand(); 66 | t.t2 = rand(); 67 | t.t3 = rand(); 68 | writer << t; 69 | } 70 | writer.finish(); 71 | 72 | //Sort the vector 73 | stxxl::sort(vector.begin(), vector.end(), cmp(), 1024*1024*1024); 74 | 75 | std::cout << vector.size() << std::endl; 76 | } 77 | 78 | -------------------------------------------------------------------------------- /include/trident/server/server.h: -------------------------------------------------------------------------------- 1 | #ifndef _SERVER_H 2 | #define _SERVER_H 3 | 4 | /* Code inspired by the tutorial available at http://pastebin.com/1KLsjJLZ */ 5 | 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | 17 | using namespace std; 18 | 19 | class TridentServer { 20 | protected: 21 | TridentLayer kb; 22 | 23 | private: 24 | string dirhtmlfiles; 25 | map cachehtml; 26 | std::thread t; 27 | string cmdArgs; 28 | 29 | bool isActive; 30 | int webport; 31 | std::shared_ptr server; 32 | int nthreads; 33 | 34 | void startThread(int port); 35 | 36 | void processRequest(std::string req, std::string &resp); 37 | 38 | public: 39 | //OK 40 | TridentServer(KB &kb, string htmlfiles, int nthreads = 1); 41 | 42 | //OK 43 | void start(int port); 44 | 45 | //OK 46 | void connect(); 47 | 48 | //OK 49 | void stop(); 50 | 51 | //OK 52 | string getDefaultPage(); 53 | 54 | //OK 55 | string getPage(string page); 56 | 57 | //OK 58 | void setActive() { 59 | isActive = true; 60 | } 61 | 62 | //OK 63 | void setInactive() { 64 | isActive = false; 65 | } 66 | 67 | //OK 68 | void join() { 69 | t.join(); 70 | } 71 | 72 | //OK 73 | string getCommandLineArgs() { 74 | return cmdArgs; 75 | } 76 | 77 | //OK 78 | static string lookup(string sId, TridentLayer &db); 79 | 80 | }; 81 | #endif 82 | -------------------------------------------------------------------------------- /include/trident/binarytables/newclustertableinserter.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Jacopo Urbani 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, 15 | * software distributed under the License is distributed on an 16 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | * KIND, either express or implied. See the License for the 18 | * specific language governing permissions and limitations 19 | * under the License. 20 | **/ 21 | 22 | 23 | #ifndef _NEWCLUSTERTABLEINSERTER_H 24 | #define _NEWCLUSTERTABLEINSERTER_H 25 | 26 | #include 27 | #include 28 | 29 | class NewClusterTableInserter: public BinaryTableInserter { 30 | private: 31 | char reader1, reader2, countsize; 32 | 33 | int64_t prevt1; 34 | std::vector v2; 35 | 36 | void writeFirstTerm(int64_t t1); 37 | void writeSecondTerm(int64_t t2); 38 | 39 | void writeGroup(); 40 | 41 | public: 42 | 43 | int getType() { 44 | return NEWCLUSTER_ITR; 45 | } 46 | 47 | void setSizes(char reader1, char reader2, char countsize) { 48 | this->reader1 = reader1; 49 | this->reader2 = reader2; 50 | this->countsize = countsize; 51 | } 52 | 53 | void startAppend(); 54 | 55 | void append(int64_t t1, int64_t t2); 56 | 57 | void stopAppend(); 58 | 59 | }; 60 | #endif 61 | -------------------------------------------------------------------------------- /include/trident/model/term.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Jacopo Urbani 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, 15 | * software distributed under the License is distributed on an 16 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | * KIND, either express or implied. See the License for the 18 | * specific language governing permissions and limitations 19 | * under the License. 20 | **/ 21 | 22 | 23 | #ifndef _TERM_H 24 | #define _TERM_H 25 | 26 | #include 27 | 28 | class Term { 29 | private: 30 | uint8_t id; //ID != 0 => variable. ID==0 => const value 31 | uint64_t value; 32 | public: 33 | Term() : id(0), value(0) {} 34 | Term(const uint8_t id, const uint64_t value) : id(id), value(value) {} 35 | uint8_t getId() const { 36 | return id; 37 | } 38 | uint64_t getValue() const { 39 | return value; 40 | } 41 | void setId(const uint8_t i) { 42 | id = i; 43 | } 44 | void setValue(const uint64_t v) { 45 | value = v; 46 | } 47 | bool isVariable() const { 48 | return id > 0; 49 | } 50 | bool operator==(const Term& rhs) const { 51 | return id == rhs.getId() && value == rhs.getValue(); 52 | } 53 | bool operator!=(const Term& rhs) const { 54 | return id != rhs.getId() || value != rhs.getValue(); 55 | } 56 | }; 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /include/trident/binarytables/columntableinserter.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Jacopo Urbani 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, 15 | * software distributed under the License is distributed on an 16 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | * KIND, either express or implied. See the License for the 18 | * specific language governing permissions and limitations 19 | * under the License. 20 | **/ 21 | 22 | 23 | #ifndef _COLUMNTABLEINSERTER_H 24 | #define _COLUMNTABLEINSERTER_H 25 | 26 | #include 27 | 28 | class ColumnTableInserter: public BinaryTableInserter { 29 | private: 30 | uint64_t largestElement, maxGroupSize; 31 | std::vector> tmpfirstpairs; 32 | std::vector tmpsecondpairs; 33 | int compr1; 34 | int compr2; 35 | uint8_t bytesPerFirstEntry, bytesPerPointer, bytesPerNElements; 36 | 37 | void writeFirstTerm(int64_t t1); 38 | 39 | void writeSecondTerm(int64_t t2); 40 | 41 | uint8_t getNBytes(const int comprType, const int64_t value) const; 42 | 43 | public: 44 | 45 | int getType() { 46 | return COLUMN_ITR; 47 | } 48 | 49 | void appendBlock(); 50 | 51 | void startAppend(); 52 | 53 | void append(int64_t t1, int64_t t2); 54 | 55 | void stopAppend(); 56 | 57 | void setCompressionMode(int v1, int v2); 58 | }; 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /snap/snap-core/doc/bfsdfs.h.txt: -------------------------------------------------------------------------------- 1 | /// GetBfsTree1 2 | Returns a directed graph where a parent points to its child node. 3 | Tree is created by following in-links (parameter FollowIn = true) and/or out-links (parameter FollowOut = true). 4 | /// 5 | 6 | 7 | /// GetSubTreeSz 8 | @IsDir false: ignore edge directions and consider edges/paths as undirected (in case they are directed). 9 | /// 10 | 11 | /// GetNodesAtHops 12 | @IsDir false: ignore edge directions and consider edges/paths as undirected (in case they are directed). 13 | /// 14 | 15 | /// GetShortPath1 16 | @param IsDir false: ignore edge directions and consider edges/paths as undirected (in case they are directed). 17 | /// 18 | 19 | /// GetShortPath2 20 | @param IsDir false: ignore edge directions and consider edges/paths as undirected (in case they are directed). 21 | @param MaxDist Maximum number of hops that BFS expands to. This is helpful for speeding-up the code if one in interested only in nodes less than MaxDist away from SrcNId. 22 | @param NIdToDistH Maps node ID to shortest path distance. NIdToDistH contains only nodes that are reachable from SrcNId. 23 | /// 24 | 25 | 26 | /// GetBfsFullDiam 27 | @param IsDir false: ignore edge directions and consider edges/paths as undirected (in case they are directed). 28 | /// 29 | 30 | /// GetBfsEffDiam1 31 | @param IsDir false: ignore edge directions and consider edges/paths as undirected (in case they are directed). 32 | /// 33 | 34 | /// GetBfsEffDiam2 35 | @param IsDir false: ignore edge directions and consider edges/paths as undirected (in case they are directed). 36 | /// 37 | 38 | /// GetBfsEffDiam3 39 | @param IsDir false: ignore edge directions and consider edges/paths as undirected (in case they are directed). 40 | /// 41 | 42 | /// GetBfsEffDiam4 43 | @param IsDir false: ignore edge directions and consider edges/paths as undirected (in case they are directed). 44 | /// 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /rdf3x/src/rts/Runtime.cpp: -------------------------------------------------------------------------------- 1 | #include "rts/runtime/Runtime.hpp" 2 | //--------------------------------------------------------------------------- 3 | // RDF-3X 4 | // (c) 2008 Thomas Neumann. Web site: http://www.mpi-inf.mpg.de/~neumann/rdf3x 5 | // 6 | // This work is licensed under the Creative Commons 7 | // Attribution-Noncommercial-Share Alike 3.0 Unported License. To view a copy 8 | // of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ 9 | // or send a letter to Creative Commons, 171 Second Street, Suite 300, 10 | // San Francisco, California, 94105, USA. 11 | //--------------------------------------------------------------------------- 12 | void Register::reset() 13 | // Reset the register (both value and domain) 14 | { 15 | value=UINT64_MAX; 16 | domain=0; 17 | } 18 | //--------------------------------------------------------------------------- 19 | Runtime::Runtime(DBLayer& db,/*DifferentialIndex* diff,*/TemporaryDictionary* temporaryDictionary, QueryDict *queryDict) 20 | : db(db),/*diff(diff),*/temporaryDictionary(temporaryDictionary), queryDict(queryDict) 21 | // Constructor 22 | { 23 | } 24 | //--------------------------------------------------------------------------- 25 | Runtime::~Runtime() 26 | // Destructor 27 | { 28 | } 29 | //--------------------------------------------------------------------------- 30 | void Runtime::allocateRegisters(unsigned count) 31 | // Set the number of registers 32 | { 33 | registers.clear(); 34 | registers.resize(count); 35 | 36 | for (unsigned index=0;index 5 | 6 | template 7 | class TranseBinaryTester : public Tester { 8 | public: 9 | TranseBinaryTester(std::shared_ptr> E, 10 | std::shared_ptr> R, Querier* q) : Tester(E, R, q) { 11 | } 12 | 13 | TranseBinaryTester(std::shared_ptr> E, 14 | std::shared_ptr> R) : Tester(E, R) { 15 | } 16 | 17 | double closeness(K *v1, uint64_t entity, uint16_t dim) { 18 | Embeddings *pE = (this->E).get(); 19 | K* v2 = pE->get(entity); 20 | uint64_t count = 0; 21 | for (uint16_t i = 0; i < dim; ++i) { 22 | uint64_t result = (((uint64_t)v1[i]) ^ ((uint64_t)v2[i])); 23 | count += __builtin_popcount(~result); 24 | } 25 | return (double)count / (double)(dim * sizeof(uint64_t) * 8); 26 | } 27 | 28 | void predictO(uint64_t sub, uint16_t dims, uint64_t pred, uint16_t dimp, K* o) { 29 | Embeddings *pE = (this->E).get(); 30 | Embeddings *pR = (this->R).get(); 31 | K* s = pE->get(sub); 32 | K* p = pR->get(pred); 33 | for (uint16_t i = 0; i < dims; ++i) { 34 | o[i] = (double) (((uint64_t)s[i]) | ((uint64_t)p[i])); 35 | } 36 | } 37 | 38 | void predictS(K *s, uint64_t pred, uint16_t dimp, uint64_t obj, uint16_t dimo) { 39 | Embeddings *pE = (this->E).get(); 40 | Embeddings *pR = (this->R).get(); 41 | K* o = pE->get(obj); 42 | K* p = pR->get(pred); 43 | for (uint16_t i = 0; i < dimp; ++i) { 44 | s[i] = (double)(~(((uint64_t)o[i]) & ((uint64_t)p[i]))); 45 | } 46 | } 47 | }; 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /snap/snap-core/Snap.cpp: -------------------------------------------------------------------------------- 1 | ///////////////////////////////////////////////// 2 | // SNAP library 3 | #include "stdafx.h" 4 | #include "Snap.h" 5 | 6 | #include "base.cpp" 7 | #include "gnuplot.cpp" 8 | #include "linalg.cpp" 9 | 10 | #include "gbase.cpp" 11 | #include "util.cpp" 12 | #include "attr.cpp" // sparse attributes 13 | 14 | // graph data structures 15 | #include "graph.cpp" // graphs 16 | #include "graphmp.cpp" // graphs 17 | //#include "mmgraph.cpp" // multimodal graphs 18 | #include "network.cpp" // networks 19 | #include "networkmp.cpp" // networks OMP 20 | #include "timenet.cpp" // time evolving networks 21 | #include "mmnet.cpp" // multimodal networks 22 | 23 | // algorithms 24 | #include "subgraph.cpp" // subgraph manipulations 25 | #include "anf.cpp" // approximate diameter calculation 26 | #include "cncom.cpp" // connected components 27 | #include "alg.cpp" // misc graph algorithms 28 | #include "gsvd.cpp" // SVD and eigenvector computations 29 | #include "gstat.cpp" // graph statistics 30 | #include "centr.cpp" // centrality measures 31 | #include "cmty.cpp" // community detection algorithms 32 | #include "flow.cpp" // network flow algorithms 33 | #include "coreper.cpp" // core-periphery algorithms 34 | #include "triad.cpp" // clustering coefficient and triads 35 | #include "casc.cpp" // cascade detection^M 36 | 37 | // graph generators 38 | #include "ggen.cpp" // graph generators (preferential attachment, small-world, ...) 39 | #include "ff.cpp" // forest fire graph generator 40 | 41 | #include "gviz.cpp" 42 | #include "ghash.cpp" 43 | #include "statplot.cpp" 44 | #include "gio.cpp" 45 | 46 | // table data structures and algorithms 47 | #include "table.cpp" // table 48 | #include "conv.cpp" 49 | #include "numpy.cpp" // numpy conversion 50 | 51 | -------------------------------------------------------------------------------- /include/trident/iterators/emptyitr.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Jacopo Urbani 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, 15 | * software distributed under the License is distributed on an 16 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | * KIND, either express or implied. See the License for the 18 | * specific language governing permissions and limitations 19 | * under the License. 20 | **/ 21 | 22 | 23 | #ifndef _EMPTY_ITR_H 24 | #define _EMPTY_ITR_H 25 | 26 | #include 27 | 28 | class EmptyItr: public PairItr { 29 | public: 30 | int getTypeItr() { 31 | return EMPTY_ITR; 32 | } 33 | 34 | int64_t getValue1() { 35 | return 0; 36 | } 37 | 38 | int64_t getValue2() { 39 | return 0; 40 | } 41 | 42 | int64_t getCount() { 43 | return 0; 44 | } 45 | 46 | bool hasNext() { 47 | return false; 48 | } 49 | 50 | void next() { 51 | } 52 | 53 | void ignoreSecondColumn() { 54 | } 55 | 56 | void clear() { 57 | } 58 | 59 | void mark() { 60 | } 61 | 62 | void reset(const char i) { 63 | } 64 | 65 | void moveto(const int64_t c1, const int64_t c2) { 66 | } 67 | 68 | bool allowMerge() { 69 | return false; 70 | } 71 | 72 | uint64_t estCardinality() { 73 | return 0; 74 | } 75 | 76 | uint64_t getCardinality() { 77 | return 0; 78 | } 79 | }; 80 | 81 | #endif 82 | -------------------------------------------------------------------------------- /include/trident/binarytables/newcolumntableinserter.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Jacopo Urbani 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, 15 | * software distributed under the License is distributed on an 16 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | * KIND, either express or implied. See the License for the 18 | * specific language governing permissions and limitations 19 | * under the License. 20 | **/ 21 | 22 | 23 | #ifndef _NEWCOLUMNTABLEINSERTER_H 24 | #define _NEWCOLUMNTABLEINSERTER_H 25 | 26 | #include 27 | 28 | class NewColumnTableInserter: public BinaryTableInserter { 29 | private: 30 | uint64_t largestElement1, largestElement2, largestGroup; 31 | 32 | int64_t prevel1, prevtotalsize2; 33 | std::vector> tmpfirstpairs; 34 | std::vector tmpsecondpairs; 35 | 36 | const size_t thresholdToOffload; 37 | int64_t offloadedElements1, offloadedElements2; 38 | bool fileopen1, fileopen2; 39 | ofstream offloadfile1; 40 | ifstream offloadfile1_r; 41 | ofstream offloadfile2; 42 | ifstream offloadfile2_r; 43 | 44 | public: 45 | 46 | NewColumnTableInserter() : thresholdToOffload(500000000) { 47 | } 48 | 49 | int getType() { 50 | return NEWCOLUMN_ITR; 51 | } 52 | 53 | void startAppend(); 54 | 55 | void append(int64_t t1, int64_t t2); 56 | 57 | void stopAppend(); 58 | }; 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /rdf3x/include/rts/operator/AggrFunctions.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_rts_operator_aggrfunctions 2 | #define H_rts_operator_aggrfunctions 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | #include 10 | #include 11 | 12 | //--------------------------------------------------------------------------- 13 | class AggrFunctions : public Operator { 14 | private: 15 | DBLayer &dict; 16 | Operator *child; 17 | AggregateHandler hdl; 18 | std::vector groupKeys; 19 | std::vector> varsToUpdate; 20 | std::vector> varsToReturn; 21 | std::vector currentGroupKeys; 22 | uint64_t currentCount; 23 | 24 | void readKeys(); 25 | void processGroup(); 26 | bool sameKeyAsCurrent(); 27 | void copyVars(); 28 | 29 | void updateVar(std::pair &var, uint64_t currentCount); 30 | 31 | public: 32 | /// Constructor 33 | AggrFunctions(DBLayer& db, Operator* child, 34 | std::map &bindings, 35 | const AggregateHandler &hdl, 36 | const std::vector &groupKeys, 37 | double expectedOutputCardinality); 38 | 39 | /// Destructor 40 | ~AggrFunctions(); 41 | 42 | /// Produce the first tuple 43 | uint64_t first(); 44 | /// Produce the next tuple 45 | uint64_t next(); 46 | /// Print the operator tree. Debugging only. 47 | void print(PlanPrinter& out); 48 | /// Add a merge join hint 49 | void addMergeHint(Register* reg1,Register* reg2); 50 | /// Register parts of the tree that can be executed asynchronous 51 | void getAsyncInputCandidates(Scheduler& scheduler); 52 | }; 53 | //--------------------------------------------------------------------------- 54 | #endif 55 | -------------------------------------------------------------------------------- /tests/test_io.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | 9 | using namespace std; 10 | 11 | namespace timens = std::chrono; 12 | 13 | void readChunk(std::vector *info) { 14 | timens::system_clock::time_point start = timens::system_clock::now(); 15 | ifstream ifs; 16 | 17 | size_t sizebuffer = 20000000; 18 | char *buffer = new char[sizebuffer]; 19 | 20 | for (int i = 0; i < info->size(); ++i) { 21 | const size_t size = info->at(i).size; 22 | //LOG(DEBUGL) << "Processing chunk " << i << " of size " << size; 23 | if (size > sizebuffer) { 24 | delete[] buffer; 25 | buffer = new char[size + size / 2]; 26 | sizebuffer = size + size / 2; 27 | } 28 | ifs.open(info->at(i).path); 29 | ifs.read(buffer, size); 30 | ifs.close(); 31 | } 32 | delete[] buffer; 33 | std::chrono::duration sec = std::chrono::system_clock::now() 34 | - start; 35 | LOG(DEBUGL) << "Reading time chunk = " << sec.count() * 1000 << "ms."; 36 | } 37 | 38 | int main(int argc, const char** argv) { 39 | 40 | string input(argv[1]); 41 | int nthreads = atoi(argv[2]); 42 | 43 | //Read all the files 44 | vector *chunks = Compressor::splitInputInChunks(input, nthreads); 45 | 46 | timens::system_clock::time_point start = timens::system_clock::now(); 47 | std::thread *threads = new std::thread[nthreads]; 48 | for (int i = 0; i < nthreads - 1; ++i) { 49 | threads[i] = std::thread(std::bind(readChunk, chunks + i + 1)); 50 | } 51 | readChunk(chunks); 52 | for (int i = 0; i < nthreads - 1; ++i) { 53 | threads[i].join(); 54 | } 55 | std::chrono::duration sec = std::chrono::system_clock::now() 56 | - start; 57 | cout << "Total reading time = " << sec.count() * 1000 << "ms." << endl; 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/trident/binarytables/newrowtableinserter.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Jacopo Urbani 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, 15 | * software distributed under the License is distributed on an 16 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | * KIND, either express or implied. See the License for the 18 | * specific language governing permissions and limitations 19 | * under the License. 20 | **/ 21 | 22 | 23 | #include 24 | 25 | void NewRowTableInserter::startAppend() { 26 | } 27 | 28 | void NewRowTableInserter::append(int64_t t1, int64_t t2) { 29 | writeFirstTerm(t1); 30 | writeSecondTerm(t2); 31 | } 32 | 33 | void NewRowTableInserter::stopAppend() { 34 | } 35 | 36 | void NewRowTableInserter::writeFirstTerm(int64_t t) { 37 | switch (reader1) { 38 | case 0: 39 | writeByte(t); 40 | return; 41 | case 1: 42 | writeShort(t); 43 | return; 44 | case 2: 45 | writeInt(t); 46 | return; 47 | case 3: 48 | writeLongInt(t); 49 | return; 50 | } 51 | } 52 | 53 | void NewRowTableInserter::writeSecondTerm(int64_t t) { 54 | switch (reader2) { 55 | case 0: 56 | writeByte(t); 57 | return; 58 | case 1: 59 | writeShort(t); 60 | return; 61 | case 2: 62 | writeInt(t); 63 | return; 64 | case 3: 65 | writeLongInt(t); 66 | return; 67 | case 4: 68 | writeLong(t); 69 | return; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /rdf3x/include/infra/osdep/GrowableMappedFile.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_infra_osdep_GrowableMappedFile 2 | #define H_infra_osdep_GrowableMappedFile 3 | //--------------------------------------------------------------------------- 4 | // RDF-3X 5 | // (c) 2009 Thomas Neumann. Web site: http://www.mpi-inf.mpg.de/~neumann/rdf3x 6 | // 7 | // This work is licensed under the Creative Commons 8 | // Attribution-Noncommercial-Share Alike 3.0 Unported License. To view a copy 9 | // of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ 10 | // or send a letter to Creative Commons, 171 Second Street, Suite 300, 11 | // San Francisco, California, 94105, USA. 12 | //---------------------------------------------------------------------------- 13 | /// Maps a file read-write into memory and supports growing the file. Note 14 | /// that mappings can be non-continuous after growth, the file might be 15 | /// mapped into multiple chunks. 16 | class GrowableMappedFile 17 | { 18 | public: 19 | /// Typedef for offsets and sizes 20 | typedef intptr_t ofs_t; 21 | 22 | private: 23 | /// os dependent data 24 | struct Data; 25 | 26 | /// os dependen tdata 27 | Data* data; 28 | 29 | public: 30 | /// Constructor 31 | GrowableMappedFile(); 32 | /// Destructor 33 | ~GrowableMappedFile(); 34 | 35 | /// Open 36 | bool open(const char* name,char*& begin,char*& end,bool readOnly); 37 | /// Create a new file 38 | bool create(const char* name); 39 | /// Close 40 | void close(); 41 | /// Flush the file 42 | bool flush(); 43 | 44 | /// Grow the underlying file physically 45 | bool growPhysically(ofs_t increment); 46 | /// Grow the mapping on the underlying file 47 | bool growMapping(ofs_t increment,char*& begin,char*& end); 48 | 49 | /// Read from the unmapped part of the file 50 | bool read(ofs_t ofs,void* data,unsigned len); 51 | /// Write to the unmapped part of the file 52 | bool write(ofs_t ofs,const void* data,unsigned len); 53 | }; 54 | //---------------------------------------------------------------------------- 55 | #endif 56 | -------------------------------------------------------------------------------- /src/trident/ml/graddebug.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | 6 | #include 7 | 8 | void GradTracer::add(uint16_t epoch, uint32_t id, 9 | std::vector &features, 10 | uint32_t n) { 11 | _Update up; 12 | up.epoch = epoch; 13 | up.n = n; 14 | up.startidx = updates_values[id].size(); 15 | //Add the values 16 | if (all || s_entities.count(id)) { 17 | for(uint16_t i = 0; i < dim; ++i) { 18 | updates_values[id].push_back(features[i]); 19 | } 20 | } 21 | updates_meta[id].push_back(up); 22 | } 23 | 24 | void GradTracer::store(std::string file) { 25 | LOG(DEBUGL) << "Storing the debug info about the gradients in " << file; 26 | //Write down the content as a JSON file 27 | JSON pt; 28 | pt.put("nents", std::to_string(updates_meta.size())); 29 | JSON traces; 30 | 31 | for(uint32_t idx = 0; idx < updates_meta.size(); ++idx) { 32 | uint32_t entityId = idx; 33 | LOG(DEBUGL) << "Process entity " << entityId << " ..."; 34 | auto trace = updates_meta[entityId]; 35 | JSON enttraces; 36 | for (auto t : trace) { 37 | JSON pent; 38 | pent.put("epoch", t.epoch); 39 | pent.put("n", t.n); 40 | pent.put("startidx", t.startidx); 41 | if (s_entities.count(entityId)) { 42 | JSON grad; 43 | for(uint16_t j = 0; j < dim; ++j) { 44 | grad.push_back(std::to_string(updates_values[entityId][t.startidx + j])); 45 | } 46 | pent.add_child("gradient", grad); 47 | } 48 | enttraces.push_back(pent); 49 | } 50 | JSON ent; 51 | ent.put("entity", std::to_string(entityId)); 52 | ent.add_child("trace", enttraces); 53 | traces.push_back(ent); 54 | } 55 | pt.add_child("traces", traces); 56 | 57 | //Store in gzip format 58 | zstr::ofstream ofs(file); 59 | JSON::write(ofs, pt); 60 | } 61 | -------------------------------------------------------------------------------- /rdf3x/include/rts/operator/CartProd.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_rts_operator_CartProd 2 | #define H_rts_operator_CartProd 3 | //--------------------------------------------------------------------------- 4 | #include 5 | #include 6 | #include 7 | #include 8 | //--------------------------------------------------------------------------- 9 | class Register; 10 | //--------------------------------------------------------------------------- 11 | /// A memory based hash join 12 | class CartProd: public Operator { 13 | private: 14 | /// The input 15 | Operator* left, *right; 16 | /// The non-join attributes 17 | std::vector leftTail, rightTail; 18 | bool leftOptional, rightOptional; 19 | bool leftExhausted; // To prevent calling next when first already delivers nothing, but leftOptional = true. 20 | 21 | uint64_t leftCount, rightCount; 22 | std::vector buffer; 23 | uint64_t idxbuffer; 24 | 25 | public: 26 | /// Constructor 27 | CartProd(Operator* left, 28 | const std::vector& leftTail, 29 | Operator* right, 30 | const std::vector& rightTail, 31 | double expectedOutputCardinality, 32 | bool leftOptional, 33 | bool rightOptional, 34 | int bitset); 35 | 36 | /// Destructor 37 | ~CartProd(); 38 | 39 | /// Produce the first tuple 40 | uint64_t first(); 41 | /// Produce the next tuple 42 | uint64_t next(); 43 | 44 | /// Print the operator tree. Debugging only. 45 | void print(PlanPrinter& out); 46 | /// Add a merge join hint 47 | void addMergeHint(Register* reg1, Register* reg2); 48 | /// Register parts of the tree that can be executed asynchronous 49 | void getAsyncInputCandidates(Scheduler& scheduler); 50 | }; 51 | //--------------------------------------------------------------------------- 52 | #endif 53 | -------------------------------------------------------------------------------- /rdf3x/include/rts/operator/Filter.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_rts_operator_Filter 2 | #define H_rts_operator_Filter 3 | //--------------------------------------------------------------------------- 4 | // RDF-3X 5 | // (c) 2008 Thomas Neumann. Web site: http://www.mpi-inf.mpg.de/~neumann/rdf3x 6 | // 7 | // This work is licensed under the Creative Commons 8 | // Attribution-Noncommercial-Share Alike 3.0 Unported License. To view a copy 9 | // of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ 10 | // or send a letter to Creative Commons, 171 Second Street, Suite 300, 11 | // San Francisco, California, 94105, USA. 12 | //--------------------------------------------------------------------------- 13 | #include "rts/operator/Operator.hpp" 14 | #include 15 | //--------------------------------------------------------------------------- 16 | class Register; 17 | //--------------------------------------------------------------------------- 18 | /// A selection that checks if a register is within a set of valid values 19 | class Filter : public Operator 20 | { 21 | private: 22 | /// The input 23 | Operator* input; 24 | /// The filter register 25 | Register* filter; 26 | /// The bounds 27 | uint64_t min,max; 28 | /// The valid values 29 | std::vector valid; 30 | /// Negative filter 31 | bool exclude; 32 | 33 | public: 34 | /// Constructor 35 | Filter(Operator* input,Register* filter,const std::vector& values,bool exclude,double expectedOutputCardinality); 36 | /// Destructor 37 | ~Filter(); 38 | 39 | /// Produce the first tuple 40 | uint64_t first(); 41 | /// Produce the next tuple 42 | uint64_t next(); 43 | 44 | /// Print the operator tree. Debugging only. 45 | void print(PlanPrinter& out); 46 | /// Add a merge join hint 47 | void addMergeHint(Register* reg1,Register* reg2); 48 | /// Register parts of the tree that can be executed asynchronous 49 | void getAsyncInputCandidates(Scheduler& scheduler); 50 | }; 51 | //--------------------------------------------------------------------------- 52 | #endif 53 | -------------------------------------------------------------------------------- /rdf3x/include/cts/codegen/CodeGen.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_cts_codegen_CodeGen 2 | #define H_cts_codegen_CodeGen 3 | //--------------------------------------------------------------------------- 4 | // RDF-3X 5 | // (c) 2008 Thomas Neumann. Web site: http://www.mpi-inf.mpg.de/~neumann/rdf3x 6 | // 7 | // This work is licensed under the Creative Commons 8 | // Attribution-Noncommercial-Share Alike 3.0 Unported License. To view a copy 9 | // of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ 10 | // or send a letter to Creative Commons, 171 Second Street, Suite 300, 11 | // San Francisco, California, 94105, USA. 12 | //--------------------------------------------------------------------------- 13 | #include 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | //--------------------------------------------------------------------------- 20 | class Operator; 21 | struct Plan; 22 | class Register; 23 | class Runtime; 24 | //--------------------------------------------------------------------------- 25 | /// Interfact to the code generation part of the compiletime system 26 | class CodeGen 27 | { 28 | public: 29 | /// Prepare runtime 30 | static void prepareRuntime(Runtime &runtime, 31 | const QueryGraph &query, 32 | std::map ®isters); 33 | 34 | /// Collect all variables contained in a plan 35 | static void collectVariables(std::set& variables,Plan* plan); 36 | /// Translate an execution plan into an operator tree without output generation 37 | static Operator* translateIntern(Runtime& runtime, const QueryGraph& query, Plan* plan, std::vector& output, const std::map ®isters); 38 | /// Translate an execution plan into an operator tree 39 | SLIBEXP static Operator* translate(Runtime& runtime,const QueryGraph& query,Plan* plan, bool silent=false); 40 | }; 41 | //--------------------------------------------------------------------------- 42 | #endif 43 | -------------------------------------------------------------------------------- /include/trident/sparql/plan.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Jacopo Urbani 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, 15 | * software distributed under the License is distributed on an 16 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | * KIND, either express or implied. See the License for the 18 | * specific language governing permissions and limitations 19 | * under the License. 20 | **/ 21 | 22 | 23 | #ifndef _PLAN_H 24 | #define _PLAN_H 25 | 26 | #include 27 | #include 28 | #include 29 | 30 | //#include 31 | 32 | #define SIMPLE 0 33 | //#define BOTTOMUP 1 34 | #define NONE 2 35 | 36 | class Querier; 37 | //class Plan; 38 | class TridentQueryPlan { 39 | private: 40 | 41 | Querier *q; 42 | 43 | std::map mapVars1; 44 | std::map mapVars2; 45 | 46 | std::shared_ptr root; 47 | 48 | /* std::shared_ptr translateJoin(Plan *plan); 49 | 50 | std::shared_ptr translateIndexScan(Plan *plan); 51 | 52 | std::shared_ptr translateProjection(Plan *plan); 53 | 54 | std::shared_ptr translateOperator(Plan *plan);*/ 55 | 56 | public: 57 | 58 | TridentQueryPlan(Querier *q) : q(q) { 59 | } 60 | 61 | void create(Query & query, int typePlanning); 62 | 63 | TupleIterator *getIterator(); 64 | 65 | void releaseIterator(TupleIterator * itr); 66 | 67 | void print(); 68 | }; 69 | 70 | #endif 71 | -------------------------------------------------------------------------------- /include/trident/sparql/resultprinter.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Jacopo Urbani 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, 15 | * software distributed under the License is distributed on an 16 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | * KIND, either express or implied. See the License for the 18 | * specific language governing permissions and limitations 19 | * under the License. 20 | **/ 21 | 22 | 23 | #ifndef _OUTPUTQUERY_H 24 | #define _OUTPUTQUERY_H 25 | 26 | #include 27 | 28 | class OutputBuffer { 29 | 30 | int64_t buffer1[OUTPUT_BUFFER_SIZE]; 31 | int64_t buffer2[OUTPUT_BUFFER_SIZE]; 32 | 33 | int64_t *currentBuffer; 34 | bool firstBufferActive; 35 | int currentSize; 36 | 37 | public: 38 | OutputBuffer() { 39 | currentBuffer = buffer1; 40 | firstBufferActive = true; 41 | currentSize = 0; 42 | } 43 | 44 | void addTerm(int64_t t) { 45 | currentBuffer[currentSize++] = t; 46 | } 47 | 48 | int64_t *getBuffer() { 49 | int64_t *o = currentBuffer; 50 | if (firstBufferActive) { 51 | currentBuffer = buffer2; 52 | } else { 53 | currentBuffer = buffer1; 54 | } 55 | firstBufferActive = !firstBufferActive; 56 | currentSize = 0; 57 | return o; 58 | } 59 | }; 60 | 61 | class ResultPrinter { 62 | private: 63 | 64 | void printHeader(); 65 | 66 | void printNumericRow(int64_t *row, int size); 67 | 68 | void printFooter(); 69 | 70 | public: 71 | 72 | 73 | }; 74 | 75 | #endif 76 | -------------------------------------------------------------------------------- /rdf3x/include/infra/osdep/Timestamp.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_infra_osdep_Timestamp 2 | #define H_infra_osdep_Timestamp 3 | //--------------------------------------------------------------------------- 4 | // RDF-3X 5 | // (c) 2008 Thomas Neumann. Web site: http://www.mpi-inf.mpg.de/~neumann/rdf3x 6 | // 7 | // This work is licensed under the Creative Commons 8 | // Attribution-Noncommercial-Share Alike 3.0 Unported License. To view a copy 9 | // of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ 10 | // or send a letter to Creative Commons, 171 Second Street, Suite 300, 11 | // San Francisco, California, 94105, USA. 12 | //--------------------------------------------------------------------------- 13 | class AvgTime; 14 | //--------------------------------------------------------------------------- 15 | /// A high resolution timestamp 16 | class Timestamp 17 | { 18 | private: 19 | /// The data 20 | char data[64]; 21 | 22 | friend class AvgTime; 23 | 24 | /// Get the raw storage space 25 | void* ptr() { return data; } 26 | /// Get the raw storage space 27 | const void* ptr() const { return data; } 28 | 29 | public: 30 | /// Constructor 31 | Timestamp(); 32 | 33 | /// Hash 34 | unsigned long long getHash() const { return *static_cast(ptr()); } 35 | 36 | /// Difference in ms 37 | unsigned operator-(const Timestamp& other) const; 38 | }; 39 | //--------------------------------------------------------------------------- 40 | /// Aggregate 41 | class AvgTime { 42 | private: 43 | /// The data 44 | char data[64]; 45 | /// Count 46 | unsigned count; 47 | 48 | /// Get the raw storage space 49 | void* ptr() { return data; } 50 | /// Get the raw storage space 51 | const void* ptr() const { return data; } 52 | 53 | public: 54 | /// Constructor 55 | AvgTime(); 56 | 57 | /// Add an interval 58 | void add(const Timestamp& start,const Timestamp& end); 59 | /// The avg time in ms 60 | double avg() const; 61 | }; 62 | //--------------------------------------------------------------------------- 63 | #endif 64 | -------------------------------------------------------------------------------- /rdf3x/src/infra/Pool.cpp: -------------------------------------------------------------------------------- 1 | #include "infra/util/Pool.hpp" 2 | //--------------------------------------------------------------------------- 3 | // RDF-3X 4 | // (c) 2008 Thomas Neumann. Web site: http://www.mpi-inf.mpg.de/~neumann/rdf3x 5 | // 6 | // This work is licensed under the Creative Commons 7 | // Attribution-Noncommercial-Share Alike 3.0 Unported License. To view a copy 8 | // of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ 9 | // or send a letter to Creative Commons, 171 Second Street, Suite 300, 10 | // San Francisco, California, 94105, USA. 11 | //--------------------------------------------------------------------------- 12 | void* PoolHelper::sort(void* data) 13 | // Sort a single-linked list using merge sort 14 | { 15 | #define next(x) *reinterpret_cast(x) 16 | // Special cases first 17 | if (!data) 18 | return data; 19 | 20 | // Split 21 | void* left=data,*leftIter=data; 22 | if ((data=next(data))==0) return left; 23 | void *right=data,*rightIter=data; 24 | if ((data=next(data))==0) { 25 | if (left 15 | //--------------------------------------------------------------------------- 16 | class Register; 17 | //--------------------------------------------------------------------------- 18 | /// A binding filter that calls its input multiple times with different 19 | /// variable bindings 20 | class NestedLoopFilter : public Operator 21 | { 22 | private: 23 | /// The input 24 | Operator* input; 25 | /// The filter register 26 | Register* filter; 27 | /// The valid values 28 | std::vector values; 29 | /// The current position 30 | uint64_t pos; 31 | 32 | public: 33 | /// Constructor 34 | NestedLoopFilter(Operator* input,Register* filter,const std::vector& values,double expectedOutputCardinality); 35 | /// Destructor 36 | ~NestedLoopFilter(); 37 | 38 | /// Produce the first tuple 39 | uint64_t first(); 40 | /// Produce the next tuple 41 | uint64_t next(); 42 | 43 | /// Print the operator tree. Debugging only. 44 | void print(PlanPrinter& out); 45 | /// Add a merge join hint 46 | void addMergeHint(Register* reg1,Register* reg2); 47 | /// Register parts of the tree that can be executed asynchronous 48 | void getAsyncInputCandidates(Scheduler& scheduler); 49 | }; 50 | //--------------------------------------------------------------------------- 51 | #endif 52 | -------------------------------------------------------------------------------- /rdf3x/src/rts/Assignment.cpp: -------------------------------------------------------------------------------- 1 | #include "rts/operator/Assignment.hpp" 2 | #include "rts/operator/PlanPrinter.hpp" 3 | #include "rts/operator/TableFunction.hpp" 4 | #include "rts/operator/PlanPrinter.hpp" 5 | #include "rts/runtime/Runtime.hpp" 6 | #include "rts/runtime/TemporaryDictionary.hpp" 7 | 8 | //#include "rts/database/Database.hpp" 9 | //#include "rts/segment/DictionarySegment.hpp" 10 | 11 | #include 12 | #include 13 | 14 | //--------------------------------------------------------------------------- 15 | Assignment::Assignment(Operator *input, const std::vector& inputArgs, 16 | const std::vector& outputVars, 17 | double expectedOutputCardinality) : 18 | Operator(expectedOutputCardinality), input(input), inputArgs(inputArgs), 19 | outputVars(outputVars) { 20 | } 21 | //--------------------------------------------------------------------------- 22 | void Assignment::print(PlanPrinter& out) 23 | // Print the operator tree. Debugging only. 24 | { 25 | out.beginOperator("Bind", expectedOutputCardinality, observedOutputCardinality); 26 | input->print(out); 27 | out.endOperator(); 28 | } 29 | //--------------------------------------------------------------------------- 30 | uint64_t Assignment::first() { 31 | if (input->first() == 0) 32 | return false; 33 | for (size_t j = 0; j < inputArgs.size(); ++j) { 34 | if (inputArgs[j].reg != NULL) 35 | outputVars[j]->value = inputArgs[j].reg->value; 36 | else 37 | outputVars[j]->value = inputArgs[j].defaultValue; 38 | } 39 | return true; 40 | } 41 | //--------------------------------------------------------------------------- 42 | uint64_t Assignment::next() { 43 | if (input->next()) { 44 | for (size_t j = 0; j < inputArgs.size(); ++j) { 45 | if (inputArgs[j].reg != NULL) 46 | outputVars[j]->value = inputArgs[j].reg->value; 47 | else 48 | outputVars[j]->value = inputArgs[j].defaultValue; 49 | } 50 | return true; 51 | } 52 | return false; 53 | } 54 | -------------------------------------------------------------------------------- /rdf3x/include/rts/operator/MergeUnion.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_rts_operator_MergeUnion 2 | #define H_rts_operator_MergeUnion 3 | //--------------------------------------------------------------------------- 4 | // RDF-3X 5 | // (c) 2008 Thomas Neumann. Web site: http://www.mpi-inf.mpg.de/~neumann/rdf3x 6 | // 7 | // This work is licensed under the Creative Commons 8 | // Attribution-Noncommercial-Share Alike 3.0 Unported License. To view a copy 9 | // of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ 10 | // or send a letter to Creative Commons, 171 Second Street, Suite 300, 11 | // San Francisco, California, 94105, USA. 12 | //--------------------------------------------------------------------------- 13 | #include "rts/operator/Operator.hpp" 14 | //--------------------------------------------------------------------------- 15 | /// A union that merges two sorted streams 16 | class MergeUnion : public Operator 17 | { 18 | private: 19 | /// Possible states 20 | enum State { done, stepLeft, stepRight, stepBoth, leftEmpty, rightEmpty }; 21 | 22 | /// The input 23 | Operator* left,*right; 24 | /// The input registers 25 | Register* leftReg,*rightReg; 26 | /// The result register 27 | Register* result; 28 | /// The values 29 | uint64_t leftValue,rightValue; 30 | /// The counts 31 | uint64_t leftCount,rightCount; 32 | /// The state 33 | State state; 34 | 35 | public: 36 | /// Constructor 37 | MergeUnion(Register* result,Operator* left,Register* leftReg,Operator* right,Register* rightReg,double expectedOutputCardinality); 38 | /// Destructor 39 | ~MergeUnion(); 40 | 41 | /// Produce the first tuple 42 | uint64_t first(); 43 | /// Produce the next tuple 44 | uint64_t next(); 45 | 46 | /// Print the operator tree. Debugging only. 47 | void print(PlanPrinter& out); 48 | /// Add a merge join hint 49 | void addMergeHint(Register* reg1,Register* reg2); 50 | /// Register parts of the tree that can be executed asynchronous 51 | void getAsyncInputCandidates(Scheduler& scheduler); 52 | }; 53 | //--------------------------------------------------------------------------- 54 | #endif 55 | -------------------------------------------------------------------------------- /include/trident/model/tuple.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Jacopo Urbani 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, 15 | * software distributed under the License is distributed on an 16 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | * KIND, either express or implied. See the License for the 18 | * specific language governing permissions and limitations 19 | * under the License. 20 | **/ 21 | 22 | 23 | #ifndef _TUPLE_H 24 | #define _TUPLE_H 25 | 26 | #include 27 | #include 28 | 29 | #define MAXTUPLESIZE 3 30 | class Tuple { 31 | private: 32 | const uint8_t sizetuple; 33 | Term terms[MAXTUPLESIZE]; 34 | public: 35 | Tuple(const uint8_t sizetuple) : sizetuple(sizetuple) {} 36 | size_t getSize() const { 37 | return sizetuple; 38 | } 39 | Term get(const int pos) const { 40 | return terms[pos]; 41 | } 42 | void set(const Term term, const int pos) { 43 | terms[pos] = term; 44 | } 45 | 46 | std::vector> getRepeatedVars() const { 47 | std::vector> output; 48 | for (uint8_t i = 0; i < sizetuple; ++i) { 49 | Term t1 = get(i); 50 | if (t1.isVariable()) { 51 | for (uint8_t j = i + 1; j < sizetuple; ++j) { 52 | Term t2 = get(j); 53 | if (t2.getId() == t1.getId()) { 54 | output.push_back(std::make_pair(i, j)); 55 | } 56 | } 57 | } 58 | } 59 | return output; 60 | } 61 | }; 62 | 63 | 64 | #endif 65 | -------------------------------------------------------------------------------- /tests/test_insert8.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | 14 | using namespace std; 15 | 16 | int main(int argc, const char** argv) { 17 | int maxReadingThreads = 2; 18 | int parallelProcesses = 8; 19 | int nindices = 6; 20 | int partsPerFiles = parallelProcesses / maxReadingThreads; 21 | uint64_t ntriples = 3500000000l; 22 | uint64_t estimatedSize = ntriples * 15; 23 | uint64_t max = ntriples / parallelProcesses; 24 | 25 | int n = 1; 26 | if(*(char *)&n != 1) { 27 | LOG(ERRORL) << "Some features of Trident rely on little endianness. " 28 | "Change machine ...sorry"; 29 | } 30 | 31 | std::string inputDir = "/Users/jacopo/Desktop/test3/"; 32 | std::vector permDirs(6); 33 | permDirs[0] = inputDir + "permtmp-0"; 34 | permDirs[1] = inputDir + "permtmp-1"; 35 | permDirs[2] = inputDir + "permtmp-2"; 36 | permDirs[3] = inputDir + "permtmp-3"; 37 | permDirs[4] = inputDir + "permtmp-4"; 38 | permDirs[5] = inputDir + "permtmp-5"; 39 | std::vector> permutations; 40 | permutations.push_back(std::make_pair(permDirs[0], IDX_SPO)); 41 | permutations.push_back(std::make_pair(permDirs[3], IDX_SOP)); 42 | permutations.push_back(std::make_pair(permDirs[4], IDX_OSP)); 43 | permutations.push_back(std::make_pair(permDirs[1], IDX_OPS)); 44 | permutations.push_back(std::make_pair(permDirs[2], IDX_POS)); 45 | permutations.push_back(std::make_pair(permDirs[5], IDX_PSO)); 46 | 47 | std::chrono::system_clock::time_point start = std::chrono::system_clock::now(); 48 | PermSorter::sortChunks2(permutations, maxReadingThreads, 49 | parallelProcesses, 50 | estimatedSize); 51 | 52 | Loader::mergeDiskFragments(ParamsMergeDiskFragments(inputDir)); 53 | std::chrono::duration sec = std::chrono::system_clock::now() - start; 54 | LOG(INFOL) << "Time (sec) " << sec.count(); 55 | } 56 | -------------------------------------------------------------------------------- /tests/test_intermediatetripledata.cpp: -------------------------------------------------------------------------------- 1 | #include "../src/kb/kbconfig.h" 2 | #include "../src/kb/kb.h" 3 | #include "../src/kb/memoryopt.h" 4 | #include "../src/kb/inserter.h" 5 | 6 | #include "../src/main/loader.h" 7 | 8 | #include "../src/sorting/sorter.h" 9 | #include "../src/sorting/filemerger.h" 10 | 11 | #include "../src/utils/lz4io.h" 12 | #include "../src/utils/utils.h" 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | using namespace std; 22 | 23 | int main(int argc, const char** argv) { 24 | LZ4Reader *reader = new LZ4Reader(argv[1]); 25 | int64_t count = 0; 26 | 27 | int idx = atoi(argv[2]); 28 | 29 | while (!reader->isEof()) { 30 | int64_t comb = reader->parseLong(); 31 | int64_t tripleId = comb >> 2; 32 | int64_t pos = comb & 3; 33 | int64_t term = reader->parseLong(); 34 | 35 | if (tripleId % 16 != idx || tripleId < 0 || tripleId > 20000000000l 36 | || pos < 0 || pos > 2 || term < 0 || term > 5000000000l) { 37 | cout << "Found problem with " << tripleId << " " << term << endl; 38 | exit(1); 39 | } 40 | 41 | if (tripleId == 0 && pos == 0) { 42 | cout << "Stop. I found it" << endl; 43 | exit(0); 44 | } 45 | 46 | if (count % 1000000 == 0) { 47 | cout << "Processed " << count << " records. Last term is " 48 | << tripleId << " " << pos << " " << term << endl; 49 | } 50 | count++; 51 | } 52 | 53 | cout << "Terminated. Processed " << count << " records" << endl; 54 | 55 | // while (!reader->isEof()) { 56 | // int flag = reader->parseByte(); 57 | // if (flag == 1) { 58 | // int64_t term = reader->parseLong(); 59 | // } else if (flag == 0) { 60 | // int sizeTerm; 61 | // const char *term = reader->parseString(sizeTerm); 62 | // cout << "Term not recognized " << string(term + 2, sizeTerm - 2) 63 | // << " triple " << (count / 3) << " pos " << (count % 3) 64 | // << endl; 65 | // exit(1); 66 | // } else { 67 | // cout << "Flag " << flag << endl; 68 | // exit(1); 69 | // } 70 | // count++; 71 | // } 72 | // 73 | // cout << "Processed " << count << " records" << endl; 74 | 75 | delete reader; 76 | 77 | } 78 | -------------------------------------------------------------------------------- /rdf3x/src/rts/SingletonScan.cpp: -------------------------------------------------------------------------------- 1 | #include "rts/operator/SingletonScan.hpp" 2 | #include "rts/operator/PlanPrinter.hpp" 3 | //--------------------------------------------------------------------------- 4 | // RDF-3X 5 | // (c) 2008 Thomas Neumann. Web site: http://www.mpi-inf.mpg.de/~neumann/rdf3x 6 | // 7 | // This work is licensed under the Creative Commons 8 | // Attribution-Noncommercial-Share Alike 3.0 Unported License. To view a copy 9 | // of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ 10 | // or send a letter to Creative Commons, 171 Second Street, Suite 300, 11 | // San Francisco, California, 94105, USA. 12 | //--------------------------------------------------------------------------- 13 | SingletonScan::SingletonScan() 14 | : Operator(1) 15 | // Constructor 16 | { 17 | } 18 | //--------------------------------------------------------------------------- 19 | SingletonScan::~SingletonScan() 20 | // Destructor 21 | { 22 | } 23 | //--------------------------------------------------------------------------- 24 | uint64_t SingletonScan::first() 25 | // Produce the first tuple 26 | { 27 | observedOutputCardinality=1; 28 | return 1; 29 | } 30 | //--------------------------------------------------------------------------- 31 | uint64_t SingletonScan::next() 32 | // Produce the next tuple 33 | { 34 | return false; 35 | } 36 | //--------------------------------------------------------------------------- 37 | void SingletonScan::print(PlanPrinter& out) 38 | // Print the operator tree. Debugging only. 39 | { 40 | out.beginOperator("SingletonScan",expectedOutputCardinality,observedOutputCardinality); 41 | out.endOperator(); 42 | } 43 | //--------------------------------------------------------------------------- 44 | void SingletonScan::addMergeHint(Register* /*reg1*/,Register* /*reg2*/) 45 | // Add a merge join hint 46 | { 47 | } 48 | //--------------------------------------------------------------------------- 49 | void SingletonScan::getAsyncInputCandidates(Scheduler& /*scheduler*/) 50 | // Register parts of the tree that can be executed asynchronous 51 | { 52 | } 53 | //--------------------------------------------------------------------------- 54 | -------------------------------------------------------------------------------- /include/snap/tedge.h: -------------------------------------------------------------------------------- 1 | #ifndef _T_EDGE_H 2 | #define _T_EDGE_H 3 | 4 | class TEdgeI { 5 | private: 6 | PairItr *itr; 7 | Querier *q; 8 | 9 | public: 10 | TEdgeI(PairItr *itr, Querier *q) : itr(itr), q(q) { 11 | if (itr && itr->hasNext()) 12 | itr->next(); 13 | } 14 | 15 | /// Increment iterator. 16 | TEdgeI& operator++ (int) { 17 | if (itr) { 18 | if (itr->hasNext()) { 19 | itr->next(); 20 | } else { 21 | q->releaseItr(itr); 22 | itr = NULL; 23 | q = NULL; 24 | } 25 | } 26 | return *this; 27 | } 28 | 29 | bool operator < (const TEdgeI& EdgeI) const { 30 | if (itr && EdgeI.itr) { 31 | return itr->getValue1() < EdgeI.itr->getValue1() || (itr->getValue1() == EdgeI.itr->getValue1() && itr->getValue2() < EdgeI.itr->getValue2()); 32 | } else { 33 | if (itr) { 34 | return true; 35 | } else { 36 | return false; 37 | } 38 | } 39 | } 40 | 41 | bool operator == (const TEdgeI& EdgeI) const { 42 | return itr->getValue1() == EdgeI.itr->getValue1() && itr->getValue2() == EdgeI.itr->getValue2(); 43 | } 44 | 45 | /// Returns edge ID. Always returns -1 since only edges in multigraphs have explicit IDs. 46 | int GetId() const { 47 | return 0; 48 | } 49 | 50 | /// Returns the source of the edge. Since the graph is undirected, this is the node with a smaller ID of the edge endpoints. 51 | int64_t GetSrcNId() const { 52 | return itr->getValue1(); 53 | } 54 | 55 | /// Returns the destination of the edge. Since the graph is undirected, this is the node with a greater ID of the edge endpoints. 56 | int64_t GetDstNId() const { 57 | return itr->getValue2(); 58 | } 59 | 60 | ~TEdgeI() { 61 | if (q) 62 | q->releaseItr(itr); 63 | } 64 | }; 65 | 66 | #endif 67 | -------------------------------------------------------------------------------- /rdf3x/include/rts/operator/Union.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_rts_operator_Union 2 | #define H_rts_operator_Union 3 | //--------------------------------------------------------------------------- 4 | // RDF-3X 5 | // (c) 2008 Thomas Neumann. Web site: http://www.mpi-inf.mpg.de/~neumann/rdf3x 6 | // 7 | // This work is licensed under the Creative Commons 8 | // Attribution-Noncommercial-Share Alike 3.0 Unported License. To view a copy 9 | // of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ 10 | // or send a letter to Creative Commons, 171 Second Street, Suite 300, 11 | // San Francisco, California, 94105, USA. 12 | //--------------------------------------------------------------------------- 13 | #include "rts/operator/Operator.hpp" 14 | #include 15 | //--------------------------------------------------------------------------- 16 | /// A union of multiple input operators. The operators are concatenated, not 17 | /// duplicates are eliminated. 18 | class Union : public Operator 19 | { 20 | private: 21 | /// The parts 22 | std::vector parts; 23 | /// The register mappings 24 | std::vector > mappings; 25 | /// The initialization lists 26 | std::vector > initializations; 27 | /// The current slot 28 | uint64_t current; 29 | 30 | // Get the first tuple from the current part 31 | uint64_t firstFromPart(); 32 | 33 | public: 34 | /// Constructor 35 | Union(const std::vector& parts,const std::vector >& mappings,const std::vector >& initializations,double expectedOutputCardinality); 36 | /// Destructor 37 | ~Union(); 38 | 39 | /// Produce the first tuple 40 | uint64_t first(); 41 | /// Produce the next tuple 42 | uint64_t next(); 43 | 44 | /// Print the operator tree. Debugging only. 45 | void print(PlanPrinter& out); 46 | /// Add a merge join hint 47 | void addMergeHint(Register* reg1,Register* reg2); 48 | /// Register parts of the tree that can be executed asynchronous 49 | void getAsyncInputCandidates(Scheduler& scheduler); 50 | }; 51 | //--------------------------------------------------------------------------- 52 | #endif 53 | -------------------------------------------------------------------------------- /src/trident/ml/tester.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | PredictParams::PredictParams() { 8 | nametestset = ""; 9 | nthreads = 1; 10 | path_modele = ""; 11 | path_modelr = ""; 12 | binary = "false"; 13 | } 14 | 15 | string PredictParams::changeable_tostring() { 16 | string out = "nametestset=" + nametestset; 17 | out += ";nthreads=" + to_string(nthreads); 18 | out += ";path_modele=" + path_modele; 19 | out += ";path_modelr=" + path_modelr; 20 | out += ";binary=" + binary; 21 | return out; 22 | } 23 | 24 | void Predictor::launchPrediction(KB &kb, string algo, PredictParams &p) { 25 | //Load model 26 | std::shared_ptr> E; 27 | std::shared_ptr> R; 28 | if (p.binary == "true") { 29 | E = Embeddings::loadBinary(p.path_modele); 30 | R = Embeddings::loadBinary(p.path_modelr); 31 | } else { 32 | E = Embeddings::load(p.path_modele); 33 | R = Embeddings::load(p.path_modelr); 34 | } 35 | 36 | //Load test files 37 | std::vector testset; 38 | string pathtest; 39 | if (p.nametestset == "valid") { 40 | pathtest = BatchCreator::getValidPath(kb.getPath(), 2); 41 | } else { 42 | pathtest = BatchCreator::getTestPath(kb.getPath(), 2); 43 | } 44 | BatchCreator::loadTriples(pathtest, testset); 45 | 46 | if (algo == "transe") { 47 | if (p.binary == "true") { 48 | TranseBinaryTester tester(E, R, kb.query()); 49 | auto result = tester.test(p.nametestset, testset, p.nthreads, 0); 50 | } else { 51 | TranseTester tester(E,R, kb.query()); 52 | auto result = tester.test(p.nametestset, testset, p.nthreads, 0); 53 | } 54 | } else if (algo == "hole") { 55 | HoleTester tester(E,R, kb.query()); 56 | auto result = tester.test(p.nametestset, testset, p.nthreads, 0); 57 | } else { 58 | LOG(ERRORL) << "Not yet supported"; 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /rdf3x/include/infra/osdep/Latch.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_infra_osdep_Latch 2 | #define H_infra_osdep_Latch 3 | //--------------------------------------------------------------------------- 4 | // RDF-3X 5 | // (c) 2009 Thomas Neumann. Web site: http://www.mpi-inf.mpg.de/~neumann/rdf3x 6 | // 7 | // This work is licensed under the Creative Commons 8 | // Attribution-Noncommercial-Share Alike 3.0 Unported License. To view a copy 9 | // of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ 10 | // or send a letter to Creative Commons, 171 Second Street, Suite 300, 11 | // San Francisco, California, 94105, USA. 12 | //--------------------------------------------------------------------------- 13 | #include "infra/Config.hpp" 14 | //--------------------------------------------------------------------------- 15 | #ifdef CONFIG_WINDOWS 16 | #define WIN32_LEAN_AND_MEAN 17 | #include 18 | #else 19 | #include 20 | #endif 21 | //--------------------------------------------------------------------------- 22 | /// A read writer lock 23 | class Latch 24 | { 25 | private: 26 | #ifdef CONFIG_WINDOWS 27 | /// Protection for the lock 28 | CRITICAL_SECTION lock; 29 | /// The event 30 | void* event; 31 | /// THe latch state 32 | int state; 33 | /// Waiting threads 34 | int exclusiveWaitCounter,sharedWaitCounter; 35 | 36 | /// Lock exclusive. A conflict occurred 37 | void doLockExclusive(); 38 | /// Lock shared. A conflict occurred 39 | void doLockShared(); 40 | /// Notify 41 | void notify(); 42 | #else 43 | /// The real latch 44 | pthread_rwlock_t lock; 45 | #endif 46 | 47 | Latch(const Latch&); 48 | void operator=(const Latch&); 49 | 50 | public: 51 | /// Constructor 52 | Latch(); 53 | /// Destructor 54 | ~Latch(); 55 | 56 | /// Lock exclusive 57 | void lockExclusive(); 58 | /// Try to lock exclusive 59 | bool tryLockExclusive(); 60 | /// Lock shared 61 | void lockShared(); 62 | /// Try to lock shared 63 | bool tryLockShared(); 64 | /// Release the lock. Returns true if the lock seems to be free now (hint only) 65 | bool unlock(); 66 | }; 67 | //--------------------------------------------------------------------------- 68 | #endif 69 | -------------------------------------------------------------------------------- /tests/test_tree.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | using namespace std; 11 | 12 | int main(int argc, const char** argv) { 13 | 14 | int n = 10000000; 15 | std::vector> pairs; 16 | for (int i = 0; i < n; ++i) { 17 | pairs.push_back(std::make_pair(i + 1024, 0)); 18 | } 19 | cout << "Finished filling the array" << endl; 20 | 21 | //Create the tree 22 | PropertyMap config; 23 | config.setBool(TEXT_KEYS, false); 24 | config.setBool(TEXT_VALUES, true); 25 | config.setBool(COMPRESSED_NODES, false); 26 | config.setInt(MAX_EL_PER_NODE, 2048); 27 | config.setInt(FILE_MAX_SIZE, 1 * 1024 * 1024); 28 | config.setLong(CACHE_MAX_SIZE, 4 * 1024 * 1024); 29 | config.setInt(NODE_MIN_BYTES, 1); 30 | config.setInt(MAX_NODES_IN_CACHE, 10); 31 | 32 | config.setInt(LEAF_SIZE_FACTORY, 2050); 33 | config.setInt(LEAF_SIZE_PREALL_FACTORY, 2050); 34 | config.setInt(LEAF_ARRAYS_FACTORY_SIZE, 10); 35 | config.setInt(LEAF_ARRAYS_PREALL_FACTORY_SIZE, 2050); 36 | config.setInt(LEAF_MAX_INTERNAL_LINES, 2050); 37 | config.setInt(LEAF_MAX_PREALL_INTERNAL_LINES, 5000); 38 | config.setInt(NODE_KEYS_FACTORY_SIZE, 10); 39 | config.setInt(NODE_KEYS_PREALL_FACTORY_SIZE, 2050); 40 | config.setInt(MAX_N_OPENED_FILES, 2048); 41 | 42 | Root *root = new Root("/Users/jacopo/Desktop/tree", NULL, false, config); 43 | 44 | //Populate the tree 45 | std::chrono::system_clock::time_point start = 46 | std::chrono::system_clock::now(); 47 | cout << "Start inserting" << endl; 48 | for (int i = 0; i < n; i++) { 49 | if (i % 1000000 == 0) { 50 | cout << "Inserting " << pairs[i].first << endl; 51 | } 52 | root->put(pairs[i].first, pairs[i].second); 53 | } 54 | 55 | //Add the first 1024 56 | for(int i = 0; i < 1024; ++i) { 57 | cout << "Inserting " << i << endl; 58 | root->put((int64_t)i, (int64_t)0); 59 | } 60 | 61 | std::chrono::duration sec = std::chrono::system_clock::now() 62 | - start; 63 | cout << "Duration insertion " << sec.count() * 1000 << endl; 64 | 65 | delete root; 66 | return 0; 67 | } 68 | -------------------------------------------------------------------------------- /snap/snap-core/gviz.h: -------------------------------------------------------------------------------- 1 | //#////////////////////////////////////////////// 2 | /// GraphViz Layout Engines 3 | typedef enum TGVizLayout_ { gvlDot, gvlNeato, gvlTwopi, gvlCirco, gvlSfdp } TGVizLayout; 4 | 5 | ///////////////////////////////////////////////// 6 | // GraphViz 7 | namespace TSnap { 8 | /// Draws a given Graph using a selected GraphViz Layout engine with nodes colored. ##DrawGViz1 9 | template void DrawGViz(const PGraph& Graph, const TGVizLayout& Layout, const TStr& PltFNm, const TStr& Desc=TStr(), const bool& NodeLabels=false, const TIntStrH& NIdColorH=TIntStrH()); 10 | /// Draws a given Graph using a selected GraphViz Layout engine with nodes labeled. ##DrawGViz2 11 | template void DrawGViz(const PGraph& Graph, const TGVizLayout& Layout, const TStr& PltFNm, const TStr& Desc, const TIntStrH& NodeLabelH); 12 | 13 | namespace TSnapDetail { 14 | /// Runs GraphViz layout engine over a graph saved in the file GraphInFNm with output saved to OutFNm. 15 | void GVizDoLayout(const TStr& GraphInFNm, TStr OutFNm, const TGVizLayout& Layout); 16 | /// Generates the GraphViz command string based on the selected Layout engine. 17 | TStr GVizGetLayoutStr(const TGVizLayout& Layout); 18 | } // namespace TSnapDetail 19 | 20 | ///////////////////////////////////////////////// 21 | // GraphViz implementation 22 | 23 | template 24 | void DrawGViz(const PGraph& Graph, const TGVizLayout& Layout, const TStr& PltFNm, const TStr& Desc, const bool& NodeLabels, const TIntStrH& NIdColorH) { 25 | const TStr Ext = PltFNm.GetFExt(); 26 | const TStr GraphFNm = PltFNm.GetSubStr(0, PltFNm.Len()-Ext.Len()) + "dot"; 27 | SaveGViz(Graph, GraphFNm, Desc, NodeLabels, NIdColorH); 28 | TSnap::TSnapDetail::GVizDoLayout(GraphFNm, PltFNm, Layout); 29 | } 30 | 31 | template 32 | void DrawGViz(const PGraph& Graph, const TGVizLayout& Layout, const TStr& PltFNm, const TStr& Desc, const TIntStrH& NodeLabelH) { 33 | const TStr Ext = PltFNm.GetFExt(); 34 | const TStr GraphFNm = PltFNm.GetSubStr(0, PltFNm.Len()-Ext.Len()) + "dot"; 35 | SaveGViz(Graph, GraphFNm, Desc, NodeLabelH); 36 | TSnap::TSnapDetail::GVizDoLayout(GraphFNm, PltFNm, Layout); 37 | } 38 | 39 | } // namespace TSNap 40 | -------------------------------------------------------------------------------- /snap/glib-core/doc/ss.h.txt: -------------------------------------------------------------------------------- 1 | /// TSs 2 | Useful for parsing small spreadsheets that represent tables of X columns and Y rows. The whole spreadsheet is read directly into memory. 3 | /// 4 | 5 | /// TSsParser 6 | Useful for fast parsing large files that contain variable number of fields separated by a particular character. See \v TSsFmt for the definitions of separators (space, tab, comma, etc.) supported by the class. The class reads the input file one line at a time. Each line is parsed and a vector of fields is returned. The class can process raw text files as well as compressed files (.gz, .7z, .zip, .7z). This means there is no need to first uncompress a file and load it. Refer to \v TZipIn for documentation on how to directly load compressed files. Lines starting with '#' can be considered as comments and the parser can skip them. 7 | /// 8 | 9 | /// TSsParser::TSsParser1 10 | @param FNm Input filename. Can be a text file or a compressed file. 11 | @param _SsFmt Spread-sheet separator format. Each line will be broken in a set of fields, where the boundary between the fields is defined by the \c _SsFmt. 12 | @param _SkipLeadBlanks If \c true leading/trailing white-spaces of the line will be ignored. 13 | @param _SkipCmt If \c true lines starting with '#' will be considered as comments and will be skipped. 14 | @param _SkipEmptyFld If \c true then empty fields (consecutive occurrences of the separator) will be ignored. 15 | /// 16 | 17 | /// TSsParser::TSsParser2 18 | @param FNm Input filename. Can be a text file or a compressed file. 19 | @param Separator Spread-sheet separator character. Each line will be broken in a set of fields, where the boundary between the fields is the \c Separator character. 20 | @param _SkipLeadBlanks If \c true leading/trailing white-spaces of the line will be ignored. 21 | @param _SkipCmt If \c true lines starting with '#' will be considered as comments and will be skipped. 22 | @param _SkipEmptyFld If \c true then empty fields (consecutive occurrences of the separator) will be ignored. 23 | /// 24 | 25 | /// TSsParser::Next 26 | If end of file is reached, return value is \c false. 27 | /// 28 | 29 | /// TSsParser::NextSlow 30 | If end of file is reached, return value is \c false. 31 | This function is deprecated, use Next instead. 32 | /// 33 | -------------------------------------------------------------------------------- /include/python/trident.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Jacopo Urbani 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, 15 | * software distributed under the License is distributed on an 16 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | * KIND, either express or implied. See the License for the 18 | * specific language governing permissions and limitations 19 | * under the License. 20 | **/ 21 | 22 | 23 | #ifndef _PYTHON_H 24 | #define _PYTHON_H 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION 33 | 34 | #include 35 | 36 | typedef struct { 37 | PyObject_HEAD 38 | KB *kb = NULL; 39 | Querier *q = NULL; 40 | std::unique_ptr db; 41 | bool rmKbOnDelete = false; 42 | } trident_Db; 43 | 44 | typedef struct { 45 | PyObject_HEAD 46 | Querier *q; 47 | PairItr *itr; 48 | char pos; 49 | } trident_Itr; 50 | 51 | typedef struct { 52 | PyObject_HEAD 53 | std::shared_ptr> E; 54 | std::shared_ptr> R; 55 | } trident_Emb; 56 | 57 | 58 | extern PyTypeObject trident_ItrType; 59 | extern PyTypeObject trident_DbType; 60 | extern PyTypeObject trident_EmbType; 61 | 62 | typedef struct { 63 | PyObject_HEAD 64 | std::unique_ptr creator; 65 | std::vector batch1; 66 | std::vector batch2; 67 | std::vector batch3; 68 | int64_t batchsize; 69 | } trident_Batcher; 70 | 71 | extern PyTypeObject trident_BatcherType; 72 | 73 | PyMODINIT_FUNC PyInit_analytics(void); 74 | 75 | #endif 76 | -------------------------------------------------------------------------------- /snap/snap-core/Snap.h: -------------------------------------------------------------------------------- 1 | #ifndef SNAP_H 2 | #define SNAP_H 3 | 4 | /// ##mainpage 5 | #define _USE_MATH_DEFINES // to use cmath's constants for VS 6 | 7 | ///////////////////////////////////////////////// 8 | // SNAP library 9 | #include "base.h" 10 | #include "gnuplot.h" 11 | #include "linalg.h" 12 | 13 | #include "gbase.h" 14 | #include "util.h" 15 | #include "attr.h" // sparse attributes 16 | 17 | // graph data structures 18 | #include "graph.h" // graphs 19 | #include "graphmp.h" // graphs 20 | #include "network.h" // networks 21 | #include "networkmp.h" // networks OMP 22 | #include "bignet.h" // large networks 23 | #include "timenet.h" // time evolving networks 24 | #include "mmnet.h" // multimodal networks 25 | 26 | // algorithms 27 | #include "subgraph.h" // subgraph manipulations 28 | #include "anf.h" // approximate diameter calculation 29 | #include "bfsdfs.h" // breadth and depth first search 30 | #include "cncom.h" // connected components 31 | #include "kcore.h" // k-core decomposition 32 | #include "alg.h" // misc graph algorithms 33 | #include "triad.h" // clustering coefficient and triads 34 | #include "gsvd.h" // SVD and eigenvector computations 35 | #include "gstat.h" // graph statistics 36 | #include "centr.h" // centrality measures 37 | #include "cmty.h" // community detection algorithms 38 | #include "flow.h" // network flow algorithms 39 | #include "coreper.h" // core-periphery algorithms 40 | #include "randwalk.h" // Personalized PageRank 41 | 42 | // graph generators 43 | #include "ggen.h" // graph generators (preferential attachment, small-world, ...) 44 | #include "ff.h" // forest fire graph generator 45 | 46 | #include "gio.h" 47 | #include "gviz.h" 48 | #include "ghash.h" 49 | #include "statplot.h" 50 | 51 | // table data structures and algorithms 52 | #include "table.h" // table 53 | #include "conv.h" // conversion functions - table to graph 54 | #include "numpy.h" // numpy conversion 55 | #include "casc.h" // cascade detection^M 56 | 57 | #endif // SNAP_H 58 | -------------------------------------------------------------------------------- /src/trident/tree/treeitr.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Jacopo Urbani 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, 15 | * software distributed under the License is distributed on an 16 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | * KIND, either express or implied. See the License for the 18 | * specific language governing permissions and limitations 19 | * under the License. 20 | **/ 21 | 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | TreeItr::TreeItr(Node *root, Leaf *firstLeaf) { 29 | this->root = root; 30 | currentLeaf = firstLeaf; 31 | currentPos = 0; 32 | 33 | //I temporary remove this leaf from the caching system. 34 | //cache = root->getContext()->getCache(); 35 | //cache->unregisterNode(currentLeaf); 36 | } 37 | 38 | bool TreeItr::hasNext() { 39 | if (currentPos >= currentLeaf->getCurrentSize()) { 40 | 41 | Node *n = currentLeaf->getRightSibling(); 42 | 43 | if (n == currentLeaf) { 44 | // cache->registerNode(currentLeaf); 45 | return false; 46 | } 47 | 48 | // cache->registerNode(currentLeaf); 49 | currentLeaf = (Leaf *) n; 50 | // cache->unregisterNode(currentLeaf); 51 | 52 | currentPos = 0; 53 | } 54 | return true; 55 | } 56 | 57 | int64_t TreeItr::next(TermCoordinates *value) { 58 | currentLeaf->getValueAtPos(currentPos, value); 59 | return currentLeaf->getKey(currentPos++); 60 | } 61 | 62 | int64_t TreeItr::next(int64_t &value) { 63 | nTerm v; 64 | currentLeaf->getValueAtPos(currentPos, &v); 65 | value = v; 66 | return currentLeaf->getKey(currentPos++); 67 | } 68 | -------------------------------------------------------------------------------- /snap/glib-core/doc/ssmp.h.txt: -------------------------------------------------------------------------------- 1 | /// TSs 2 | Useful for parsing small spreadsheets that represent tables of X columns and Y rows. The whole spreadsheet is read directly into memory. 3 | /// 4 | 5 | /// TSsParserMP 6 | Useful for fast parsing large files that contain variable number of fields separated by a particular character. See \v TSsFmt for the definitions of separators (space, tab, comma, etc.) supported by the class. The class reads the input file one line at a time. Each line is parsed and a vector of fields is returned. The class can process raw text files as well as compressed files (.gz, .7z, .zip, .7z). This means there is no need to first uncompress a file and load it. Refer to \v TZipIn for documentation on how to directly load compressed files. Lines starting with '#' can be considered as comments and the parser can skip them. 7 | /// 8 | 9 | /// TSsParserMP::TSsParserMP1 10 | @param FNm Input filename. Can be a text file or a compressed file. 11 | @param _SsFmt Spread-sheet separator format. Each line will be broken in a set of fields, where the boundary between the fields is defined by the \c _SsFmt. 12 | @param _SkipLeadBlanks If \c true leading/trailing white-spaces of the line will be ignored. 13 | @param _SkipCmt If \c true lines starting with '#' will be considered as comments and will be skipped. 14 | @param _SkipEmptyFld If \c true then empty fields (consecutive occurrences of the separator) will be ignored. 15 | /// 16 | 17 | /// TSsParserMP::TSsParserMP2 18 | @param FNm Input filename. Can be a text file or a compressed file. 19 | @param Separator Spread-sheet separator character. Each line will be broken in a set of fields, where the boundary between the fields is the \c Separator character. 20 | @param _SkipLeadBlanks If \c true leading/trailing white-spaces of the line will be ignored. 21 | @param _SkipCmt If \c true lines starting with '#' will be considered as comments and will be skipped. 22 | @param _SkipEmptyFld If \c true then empty fields (consecutive occurrences of the separator) will be ignored. 23 | /// 24 | 25 | /// TSsParserMP::Next 26 | If end of file is reached, return value is \c false. 27 | /// 28 | 29 | /// TSsParserMP::NextSlow 30 | If end of file is reached, return value is \c false. 31 | This function is deprecated, use Next instead. 32 | /// 33 | -------------------------------------------------------------------------------- /snap/snap-core/anf.cpp: -------------------------------------------------------------------------------- 1 | ///////////////////////////////////////////////// 2 | // Approximate Neighborhood Function 3 | namespace TSnap { 4 | namespace TSnapDetail { 5 | 6 | // interpolate effective diameter 7 | double CalcEffDiam(const TIntFltKdV& DistNbrsCdfV, const double& Percentile) { 8 | const double EffPairs = Percentile * DistNbrsCdfV.Last().Dat; 9 | int ValN; 10 | for (ValN = 0; ValN < DistNbrsCdfV.Len(); ValN++) { 11 | if (DistNbrsCdfV[ValN].Dat() > EffPairs) { break; } 12 | } 13 | if (ValN >= DistNbrsCdfV.Len()) return DistNbrsCdfV.Last().Key; 14 | if (ValN == 0) return 1; 15 | // interpolate 16 | const double DeltaNbrs = DistNbrsCdfV[ValN].Dat - DistNbrsCdfV[ValN-1].Dat; 17 | if (DeltaNbrs == 0) return DistNbrsCdfV[ValN].Key; 18 | return DistNbrsCdfV[ValN-1].Key + (EffPairs - DistNbrsCdfV[ValN-1].Dat)/DeltaNbrs; 19 | } 20 | 21 | double CalcEffDiam(const TFltPrV& DistNbrsCdfV, const double& Percentile) { 22 | TIntFltKdV KdV(DistNbrsCdfV.Len(), 0); 23 | for (int i = 0; i < DistNbrsCdfV.Len(); i++) { 24 | KdV.Add(TIntFltKd(int(DistNbrsCdfV[i].Val1()), DistNbrsCdfV[i].Val2)); 25 | } 26 | return CalcEffDiam(KdV, Percentile); 27 | } 28 | 29 | double CalcEffDiamPdf(const TIntFltKdV& DistNbrsPdfV, const double& Percentile) { 30 | TIntFltKdV CdfV; 31 | TGUtil::GetCdf(DistNbrsPdfV, CdfV); 32 | return CalcEffDiam(CdfV, Percentile); 33 | } 34 | 35 | double CalcEffDiamPdf(const TFltPrV& DistNbrsPdfV, const double& Percentile) { 36 | TFltPrV CdfV; 37 | TGUtil::GetCdf(DistNbrsPdfV, CdfV); 38 | return CalcEffDiam(CdfV, Percentile); 39 | } 40 | 41 | double CalcAvgDiamPdf(const TIntFltKdV& DistNbrsPdfV) { 42 | double Paths=0, SumLen=0; 43 | for (int i = 0; i < DistNbrsPdfV.Len(); i++) { 44 | SumLen += DistNbrsPdfV[i].Key * DistNbrsPdfV[i].Dat; 45 | Paths += DistNbrsPdfV[i].Dat; 46 | } 47 | return SumLen/Paths; 48 | } 49 | 50 | double CalcAvgDiamPdf(const TFltPrV& DistNbrsPdfV) { 51 | double Paths=0, SumLen=0; 52 | for (int i = 0; i < DistNbrsPdfV.Len(); i++) { 53 | SumLen += DistNbrsPdfV[i].Val1 * DistNbrsPdfV[i].Val2; 54 | Paths += DistNbrsPdfV[i].Val2; 55 | } 56 | return SumLen/Paths; 57 | } 58 | 59 | } // namespace TSnapDetail 60 | } // namespace TSnap 61 | 62 | -------------------------------------------------------------------------------- /snap/snap-core/doc/Snap.h.txt: -------------------------------------------------------------------------------- 1 | /// mainpage 2 | \mainpage 3 | 4 | \section general_doc About this Document 5 | 6 | This document provides reference documentation for the SNAP APIs. 7 | The documentation also includes a complete reference to the source code. 8 | To start, browse the sections about Classes or Files in the left column or 9 | search for a specific item in the search box in the upper right. 10 | 11 | Two versions of this document are available, User Reference and Developer 12 | Reference. The Developer Reference document is more extensive. It includes 13 | inheritance and collaboration diagrams for classes, and call and caller graphs 14 | for functions. These diagrams and graphs are not included in the 15 | User Reference document. 16 | 17 | \section general_snap About Snap 18 | 19 | Stanford Network Analysis Platform (SNAP) is a general purpose, high 20 | performance system for analysis and manipulation of large networks. 21 | Graphs consists of nodes and directed/undirected/multiple edges between 22 | the graph nodes. Networks are graphs with data on nodes and/or edges of 23 | the network. 24 | 25 | The core SNAP library is written in C++ and optimized for maximum 26 | performance and compact graph representation. It easily scales to 27 | massive networks with hundreds of millions of nodes, and billions of 28 | edges. It efficiently manipulates large graphs, calculates structural 29 | properties, generates regular and random graphs, and supports attributes 30 | on nodes and edges. Besides scalability to large graphs, an additional 31 | strength of SNAP is that nodes, edges and attributes in a graph or 32 | a network can be changed dynamically during the computation. 33 | 34 | SNAP works on Windows with Visual Studio or Cygwin with GCC, Mac OS X, 35 | Linux and other Unix variants with GCC installed. SNAP is largely 36 | self-contained and has minimal dependency requirements on other packages. 37 | 38 | SNAP uses a general purpose STL (Standard Template Library)-like library 39 | GLib developed at Jozef Stefan Institute. SNAP and GLib are being actively 40 | developed and used in numerous academic and industrial projects. 41 | 42 | Complete source code for the core SNAP and GLib libraries is available 43 | under the BSD license. Download the package at http://snap.stanford.edu. 44 | /// 45 | -------------------------------------------------------------------------------- /tests/test_number_flags.cpp: -------------------------------------------------------------------------------- 1 | #include "../src/utils/utils.h" 2 | #include 3 | #include 4 | 5 | int main(int argc, const char** args) { 6 | 7 | int64_t startOffset = 0; 8 | int64_t n = 10000000000; 9 | int incr = 10; 10 | char test[8]; 11 | 12 | std::chrono::system_clock::time_point start = 13 | std::chrono::system_clock::now(); 14 | for (int64_t i = 0; i < n; i += incr) { 15 | int64_t key = (int64_t) (i) + startOffset; 16 | Utils::encode_vlongWithHeader1(test, key); 17 | int offset = 0; 18 | int64_t j = Utils::decode_vlongWithHeader1(test, 8, &offset); 19 | if (j != key) { 20 | cout << "Wrong j=" << j << " key=" << key << endl; 21 | break; 22 | } 23 | } 24 | std::chrono::duration sec = std::chrono::system_clock::now() 25 | - start; 26 | cout << "Duration encode vlong1 " << sec.count() << endl; 27 | 28 | start = std::chrono::system_clock::now(); 29 | for (int64_t i = 0; i < n; i += incr) { 30 | int64_t key = (int64_t) (i) + startOffset; 31 | Utils::encode_vlongWithHeader0(test, key); 32 | int offset = 0; 33 | int64_t j = Utils::decode_vlongWithHeader0(test, 8, &offset); 34 | if (j != key) { 35 | cout << "Wrong j=" << j << " key=" << key << endl; 36 | break; 37 | } 38 | } 39 | sec = std::chrono::system_clock::now() - start; 40 | cout << "Duration encode vlong0 " << sec.count() << endl; 41 | 42 | start = std::chrono::system_clock::now(); 43 | for (int64_t i = 0; i < n; i += incr) { 44 | int64_t key = (int64_t) (i) + startOffset; 45 | Utils::encode_longWithHeader0(test, key); 46 | int64_t j = Utils::decode_longWithHeader(test); 47 | if (j != key) { 48 | cout << "Wrong j=" << j << " key=" << key << endl; 49 | break; 50 | } 51 | } 52 | sec = std::chrono::system_clock::now() - start; 53 | cout << "Duration encode long0 " << sec.count() << endl; 54 | 55 | start = std::chrono::system_clock::now(); 56 | for (int64_t i = 0; i < n; i += incr) { 57 | int64_t key = (int64_t) (i) + startOffset; 58 | Utils::encode_longWithHeader1(test, key); 59 | int64_t j = Utils::decode_longWithHeader(test); 60 | if (j != key) { 61 | cout << "Wrong j=" << j << " key=" << key << endl; 62 | break; 63 | } 64 | } 65 | sec = std::chrono::system_clock::now() - start; 66 | cout << "Duration encode long1 " << sec.count() << endl; 67 | } 68 | -------------------------------------------------------------------------------- /rdf3x/include/rts/operator/HashGroupify.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_rts_operator_HashGroupify 2 | #define H_rts_operator_HashGroupify 3 | //--------------------------------------------------------------------------- 4 | // RDF-3X 5 | // (c) 2008 Thomas Neumann. Web site: http://www.mpi-inf.mpg.de/~neumann/rdf3x 6 | // 7 | // This work is licensed under the Creative Commons 8 | // Attribution-Noncommercial-Share Alike 3.0 Unported License. To view a copy 9 | // of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ 10 | // or send a letter to Creative Commons, 171 Second Street, Suite 300, 11 | // San Francisco, California, 94105, USA. 12 | //--------------------------------------------------------------------------- 13 | #include "rts/operator/Operator.hpp" 14 | #include "infra/util/VarPool.hpp" 15 | #include 16 | //--------------------------------------------------------------------------- 17 | /// A hash based aggregation 18 | class HashGroupify : public Operator 19 | { 20 | private: 21 | /// A group 22 | struct Group { 23 | /// The next group 24 | Group* next; 25 | /// The hash value 26 | uint64_t hash; 27 | /// The count 28 | uint64_t count; 29 | /// The values 30 | uint64_t values[]; 31 | }; 32 | /// Helper 33 | class Rehasher; 34 | /// Helper 35 | class Chainer; 36 | 37 | /// The input registers 38 | std::vector values; 39 | /// The input 40 | Operator* input; 41 | /// The groups 42 | Group* groups,*groupsIter; 43 | /// The groups pool 44 | VarPool groupsPool; 45 | 46 | public: 47 | /// Constructor 48 | HashGroupify(Operator* input,const std::vector& values,double expectedOutputCardinality); 49 | /// Destructor 50 | ~HashGroupify(); 51 | 52 | /// Produce the first tuple 53 | uint64_t first(); 54 | /// Produce the next tuple 55 | uint64_t next(); 56 | 57 | /// Print the operator tree. Debugging only. 58 | void print(PlanPrinter& out); 59 | /// Add a merge join hint 60 | void addMergeHint(Register* reg1,Register* reg2); 61 | /// Register parts of the tree that can be executed asynchronous 62 | void getAsyncInputCandidates(Scheduler& scheduler); 63 | }; 64 | //--------------------------------------------------------------------------- 65 | #endif 66 | -------------------------------------------------------------------------------- /tests/test_treeitr.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | using namespace std; 15 | 16 | int main(int argc, const char** argv) { 17 | //Initialize the tree 18 | KBConfig config; 19 | PropertyMap map; 20 | map.setBool(TEXT_KEYS, false); 21 | map.setBool(TEXT_VALUES, false); 22 | map.setBool(COMPRESSED_NODES, false); 23 | map.setInt(LEAF_SIZE_PREALL_FACTORY, 24 | config.getParamInt(TREE_MAXPREALLLEAVESCACHE)); 25 | map.setInt(LEAF_SIZE_FACTORY, config.getParamInt(TREE_MAXLEAVESCACHE)); 26 | map.setInt(MAX_NODES_IN_CACHE, config.getParamInt(TREE_MAXNODESINCACHE)); 27 | map.setInt(NODE_MIN_BYTES, config.getParamInt(TREE_NODEMINBYTES)); 28 | map.setLong(CACHE_MAX_SIZE, config.getParamLong(TREE_MAXSIZECACHETREE)); 29 | map.setInt(FILE_MAX_SIZE, config.getParamInt(TREE_MAXFILESIZE)); 30 | map.setInt(MAX_N_OPENED_FILES, config.getParamInt(TREE_MAXNFILES)); 31 | map.setInt(MAX_EL_PER_NODE, config.getParamInt(TREE_MAXELEMENTSNODE)); 32 | 33 | map.setInt(LEAF_MAX_PREALL_INTERNAL_LINES, 34 | config.getParamInt(TREE_MAXPREALLINTERNALLINES)); 35 | map.setInt(LEAF_MAX_INTERNAL_LINES, 36 | config.getParamInt(TREE_MAXINTERNALLINES)); 37 | map.setInt(LEAF_ARRAYS_FACTORY_SIZE, config.getParamInt(TREE_FACTORYSIZE)); 38 | map.setInt(LEAF_ARRAYS_PREALL_FACTORY_SIZE, 39 | config.getParamInt(TREE_ALLOCATEDELEMENTS)); 40 | 41 | map.setInt(NODE_KEYS_FACTORY_SIZE, 42 | config.getParamInt(TREE_NODE_KEYS_FACTORY_SIZE)); 43 | map.setInt(NODE_KEYS_PREALL_FACTORY_SIZE, 44 | config.getParamInt(TREE_NODE_KEYS_PREALL_FACTORY_SIZE)); 45 | 46 | Root root(string(argv[1]), NULL, true, map); 47 | TreeItr *itr = root.itr(); 48 | TermCoordinates coord; 49 | while (itr->hasNext()) { 50 | int64_t key = itr->next(&coord); 51 | std::cout << key << endl; 52 | } 53 | 54 | delete itr; 55 | } 56 | -------------------------------------------------------------------------------- /rdf3x/include/rts/operator/Assignment.hpp: -------------------------------------------------------------------------------- 1 | #ifndef H_rts_operator_Assignment 2 | #define H_rts_operator_Assignment 3 | //--------------------------------------------------------------------------- 4 | // RDF-3X 5 | // (c) 2008 Thomas Neumann. Web site: http://www.mpi-inf.mpg.de/~neumann/rdf3x 6 | // 7 | // This work is licensed under the Creative Commons 8 | // Attribution-Noncommercial-Share Alike 3.0 Unported License. To view a copy 9 | // of this license, visit http://creativecommons.org/licenses/by-nc-sa/3.0/ 10 | // or send a letter to Creative Commons, 171 Second Street, Suite 300, 11 | // San Francisco, California, 94105, USA. 12 | //--------------------------------------------------------------------------- 13 | #include "rts/operator/Operator.hpp" 14 | #include "rts/operator/TableFunction.hpp" 15 | //--------------------------------------------------------------------------- 16 | /// The SPARQL 1.1 BIND operator 17 | class Register; 18 | class Assignment: public Operator { 19 | public: 20 | // A function argument 21 | struct AssignmentArgument { 22 | /// The register if any 23 | Register* reg; 24 | uint64_t defaultValue; 25 | }; 26 | 27 | private: 28 | Operator* input; 29 | std::vector inputArgs; 30 | std::vector outputVars; 31 | 32 | public: 33 | /// Constructor 34 | Assignment(Operator *input, const std::vector& inputArgs, 35 | const std::vector& outputVars, 36 | double expectedOutputCardinality); 37 | 38 | /// Destructor 39 | ~Assignment() { 40 | delete input; 41 | } 42 | 43 | /// Produce the first tuple 44 | uint64_t first(); 45 | /// Produce the next tuple 46 | uint64_t next(); 47 | 48 | /// Print the operator tree. Debugging only. 49 | void print(PlanPrinter& out); 50 | 51 | /// Add a merge join hint 52 | void addMergeHint(Register* reg1, Register* reg2) { 53 | input->addMergeHint(reg1, reg2); 54 | } 55 | 56 | /// Register parts of the tree that can be executed asynchronous 57 | void getAsyncInputCandidates(Scheduler& scheduler) { 58 | input->getAsyncInputCandidates(scheduler); 59 | } 60 | }; 61 | //--------------------------------------------------------------------------- 62 | #endif 63 | --------------------------------------------------------------------------------