├── .gitignore ├── AUTHORS ├── COPYING ├── ChangeLog ├── INSTALL ├── LICENSE ├── Makefile.am ├── NEWS ├── README.md ├── bindings └── python │ ├── .gitignore │ ├── __init__.py │ └── client.pyx ├── chronos ├── chronos.cc ├── chronos.h ├── chronos_c_wrappers.cc ├── chronos_cmp_encode.cc ├── chronos_cmp_encode.h ├── chronos_stats_encode.cc ├── chronos_stats_encode.h ├── chronosd.cc ├── event_dependency_graph.cc ├── event_dependency_graph.h ├── network_constants.h └── replicant-shim.c ├── client ├── c.cc ├── client.cc ├── client.h ├── client_constants.h ├── comm_wrapper.cc ├── comm_wrapper.h ├── datastructures.h ├── weaver │ ├── weaver_client.h │ └── weaver_returncode.h └── weaver_returncode.cc ├── common ├── .gitignore ├── MurmurHash3.cpp ├── MurmurHash3.h ├── bool_vector.h ├── cache_constants.h ├── clock.cc ├── clock.h ├── comm_wrapper.cc ├── comm_wrapper.h ├── config_constants.cc ├── config_constants.h ├── configuration.cc ├── configuration.h ├── enum_serialization.cc ├── enum_serialization.h ├── event_order.cc ├── event_order.h ├── hyper_stub_base.cc ├── hyper_stub_base.h ├── ids.cc ├── ids.h ├── message.cc ├── message.h ├── message_cache_context.cc ├── message_constants.h ├── message_graph_elem.cc ├── prog_write_and_dlopen.cc ├── prog_write_and_dlopen.h ├── property_predicate.cc ├── property_predicate.h ├── server.cc ├── server.h ├── server_manager_link.cc ├── server_manager_link.h ├── server_manager_link_wrapper.cc ├── server_manager_link_wrapper.h ├── server_manager_returncode.h ├── stl_serialization.cc ├── stl_serialization.h ├── transaction.cc ├── transaction.h ├── types.h ├── utils.h ├── vclock.cc ├── vclock.h ├── weaver_constants.h ├── weaver_serialization.cc └── weaver_serialization.h ├── conf └── weaver.yaml ├── configure.ac ├── coordinator ├── .gitignore ├── blocked_prog.h ├── current_prog.h ├── hyper_stub.cc ├── hyper_stub.h ├── register_node_prog_state.h ├── server_barrier.cc ├── server_barrier.h ├── server_manager.cc ├── server_manager.h ├── symtable.c ├── timestamper.cc ├── timestamper.h ├── transitions.cc ├── transitions.h ├── util.h └── vt_constants.h ├── db ├── .gitignore ├── async_nodeprog_state.h ├── cache_entry.h ├── deferred_write.h ├── del_obj.h ├── edge.cc ├── edge.h ├── element.cc ├── element.h ├── hyper_stub.cc ├── hyper_stub.h ├── message_wrapper.h ├── node.cc ├── node.h ├── node_entry.h ├── node_prog_running_state.h ├── property.cc ├── property.h ├── queue_manager.cc ├── queue_manager.h ├── queued_request.h ├── remote_node.h ├── shard.cc ├── shard.h ├── shard_constants.h ├── types.h └── utils.h ├── m4 └── .gitignore ├── node_prog ├── base_classes.h ├── boilerplate.h ├── cache_response.h ├── cause_and_effect.cc ├── cause_and_effect.h ├── clustering_program.cc ├── clustering_program.h ├── dijkstra_program.h ├── discover_paths.cc ├── discover_paths.h ├── dynamic_prog_table.cc ├── dynamic_prog_table.h ├── edge.h ├── edge_count_program.cc ├── edge_count_program.h ├── edge_get_program.cc ├── edge_get_program.h ├── edge_list.cc ├── edge_list.h ├── get_btc_addr.cc ├── get_btc_addr.h ├── get_btc_block.cc ├── get_btc_block.h ├── get_btc_tx.cc ├── get_btc_tx.h ├── n_gram_path.cc ├── n_gram_path.h ├── n_hop_reach_program.h ├── neural_net_infer.cc ├── neural_net_infer.h ├── node.h ├── node_get_program.cc ├── node_get_program.h ├── node_prog_type.h ├── parse_btc_tx.cc ├── parse_btc_tx.h ├── pathless_reach_program.cc ├── pathless_reach_program.h ├── prop_list.cc ├── prop_list.h ├── property.h ├── reach_program.cc ├── reach_program.h ├── read_n_edges_program.cc ├── read_n_edges_program.h ├── read_node_props_program.cc ├── read_node_props_program.h ├── traverse_with_props.cc ├── traverse_with_props.h ├── triangle_program.h ├── two_neighborhood_program.cc └── two_neighborhood_program.h ├── startup_scripts ├── docker_kill_weaver.sh ├── docker_start_weaver.sh ├── kill_weaver.sh ├── parse_config.cc ├── ps_weaver.sh ├── start_weaver.sh └── start_weaver_no_hdex.sh ├── tests ├── .gitignore ├── c │ └── simple_test.c ├── cpp │ ├── basic_client.h │ ├── basic_migration_test.h │ ├── cache_test.h │ ├── clique_reach_program.h │ ├── clustering_prog_test.h │ ├── consistency_test.h │ ├── create_graph.h │ ├── dijkstra_prog_test.h │ ├── dijkstra_tree_test.h │ ├── hyper_stub_perf.cc │ ├── line_reach_program.h │ ├── message_test.h │ ├── message_tx.h │ ├── multiple_caching.h │ ├── multiple_clustering.h │ ├── multiple_locality.h │ ├── multiple_nhop_locality.h │ ├── multiple_reach_program.h │ ├── multiple_widest_path.h │ ├── n_hop_reachability_test.h │ ├── new_reachability_test.h │ ├── read_only_vertex_bench.h │ ├── repetitive_reach_program.h │ ├── run.cc │ ├── scalability.h │ ├── test_base.h │ ├── test_dynamic.cc │ ├── triangle_counting_test.h │ ├── tx_msg_nmap.h │ ├── unreachable_reach_program.h │ └── vc_ordering.h ├── python │ ├── .gitignore │ ├── assoc_count_basic.py │ ├── assoc_range_basic.py │ ├── benchmarks │ │ ├── clustering_bench.py │ │ ├── clustering_migration.py │ │ ├── nwx_reachability.py │ │ ├── reachability_bench.py │ │ ├── reachable_migration.py │ │ ├── read_only_vertex_bench.py │ │ ├── sequential_pathless_reachability_bench.py │ │ ├── soc_net_bench.py │ │ └── two_neighborhood_bench.py │ ├── correctness │ │ ├── clustering_basic_test.py │ │ ├── concurrent_clients.py │ │ ├── delete_tx.py │ │ ├── dijkstra_basic_test.py │ │ ├── empty_graph_sanity_checks.py │ │ ├── ft_shard_basic.py │ │ ├── ft_shard_deletes.py │ │ ├── graph_of_gods_demo.py │ │ ├── line_properties.py │ │ ├── line_reachability.py │ │ ├── multiline_cache_testing.py │ │ ├── multiple_del.py │ │ ├── paths.py │ │ ├── read_properties.py │ │ ├── simple_test.py │ │ ├── simple_test_aux_index.py │ │ ├── transactions.py │ │ ├── transactions_ring.py │ │ └── two_neighborhood_testing.py │ ├── create_graph.py │ ├── simple_client.py │ ├── star_cache_testing.py │ └── test_base.py ├── sh │ ├── clean.sh │ ├── concurrent_clients.sh │ ├── empty_graph.sh │ ├── env.sh │ ├── line_properties.sh │ ├── line_reachability.sh │ ├── multiple_del.sh │ ├── read_properties.sh │ ├── setup.sh │ ├── simple_test.sh │ ├── simple_test_aux_index.sh │ └── transactions.sh ├── snap_cleanup.py ├── snap_to_weaver.py └── static_partitioning │ └── stream_partition.py ├── weaver.cc ├── weaver.upack └── weaver_alpha.png /.gitignore: -------------------------------------------------------------------------------- 1 | *.dirstamp 2 | *.bin 3 | *.o 4 | *.rec 5 | *.txt 6 | *.log 7 | *.profile 8 | ltmain.sh 9 | *.depend 10 | *.out 11 | *.lo 12 | *.la 13 | */*.[0-9][0-9]* 14 | */*.eps 15 | */*.pdf 16 | */*.supp 17 | Makefile 18 | Makefile.in 19 | aclocal.m4 20 | autom4te.cache 21 | chronos/chronosbench/__init__.pyc 22 | chronos/chronosbench/util.pyc 23 | chronos/chronos_cmp_encode.lo 24 | chronos/chronos_cmp_encode.o 25 | chronos/chronos_c_wrappers.lo 26 | chronos/chronos_c_wrappers.o 27 | chronos/chronosd 28 | chronos/chronosd-chronos_cmp_encode.o 29 | chronos/chronosd-chronosd.o 30 | chronos/chronosd-chronos_stats_encode.o 31 | chronos/chronosd-event_dependency_graph.o 32 | chronos/chronosd.lo 33 | chronos/chronosd.o 34 | chronos/chronos.la 35 | chronos/chronos_la-python-chronos.lo 36 | chronos/chronos_la-python-chronos.o 37 | chronos/chronos.lo 38 | chronos/chronos.o 39 | chronos/chronos_stats_encode.lo 40 | chronos/chronos_stats_encode.o 41 | chronos/event_dependency_graph.lo 42 | chronos/event_dependency_graph.o 43 | chronos/libchronosd.la 44 | chronos/libchronosd.so 45 | chronos/libchronos.la 46 | chronos/python-chronos.c 47 | chronos/replicant-shim.lo 48 | chronos/replicant-shim.o 49 | config.guess 50 | config.h 51 | config.h.in 52 | config.h.in~ 53 | config.log 54 | config.status 55 | config.sub 56 | configure 57 | coordinator/run_sm 58 | depcomp 59 | .deps 60 | install-sh 61 | libtool 62 | .libs 63 | m4/*.m4 64 | missing 65 | stamp-h1 66 | tests/cpp/.dirstamp 67 | tests/hyperdex_test 68 | weaver-shard 69 | weaver-timestamper 70 | weaver-parse-config 71 | weaver-test-bench 72 | weaver-test-hs 73 | Makefile.deb.weaver 74 | weaver-0.1.tar.gz 75 | compile 76 | test-driver 77 | weaver-test-dynamic 78 | weaver 79 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Ayush Dubey, dubey@cs.cornell.edu 2 | Greg Hill, gdh39@cornell.edu 3 | Robert Escriva, escriva@cs.cornell.edu 4 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dubey/weaver/56a42fd2d0bbb14867ba792ca5461d16310a7387/COPYING -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dubey/weaver/56a42fd2d0bbb14867ba792ca5461d16310a7387/ChangeLog -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-14, Cornell University 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | * Neither the name of HyperDex nor the names of its contributors 13 | may be used to endorse or promote products derived from this software 14 | without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 17 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 20 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 23 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dubey/weaver/56a42fd2d0bbb14867ba792ca5461d16310a7387/NEWS -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | Weaver is a distributed, scalable, and strongly consistent graph store. 4 | 5 | Visit the homepage at [weaver.systems](http://weaver.systems). 6 | -------------------------------------------------------------------------------- /bindings/python/.gitignore: -------------------------------------------------------------------------------- 1 | *.cpp 2 | *.so 3 | -------------------------------------------------------------------------------- /bindings/python/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # 3 | # =============================================================== 4 | # Description: Empty init file for py client. 5 | # 6 | # Created: 2014-03-17 10:09:16 7 | # 8 | # Author: Ayush Dubey, dubey@cs.cornell.edu 9 | # 10 | # Copyright (C) 2013-2014, Cornell University, see the LICENSE 11 | # file for licensing agreement 12 | # =============================================================== 13 | # 14 | 15 | 16 | -------------------------------------------------------------------------------- /chronos/chronos_cmp_encode.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Cornell University 2 | // All rights reserved. 3 | // 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 7 | // * Redistributions of source code must retain the above copyright notice, 8 | // this list of conditions and the following disclaimer. 9 | // * Redistributions in binary form must reproduce the above copyright 10 | // notice, this list of conditions and the following disclaimer in the 11 | // documentation and/or other materials provided with the distribution. 12 | // * Neither the name of Chronos nor the names of its contributors may be 13 | // used to endorse or promote products derived from this software without 14 | // specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 | // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 | // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 | // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 | // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 | // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 | // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 | // POSSIBILITY OF SUCH DAMAGE. 27 | 28 | // Chronos 29 | #include "chronos_cmp_encode.h" 30 | 31 | uint8_t 32 | chronos_cmp_to_byte(chronos_cmp c) 33 | { 34 | switch (c) 35 | { 36 | case CHRONOS_HAPPENS_BEFORE: 37 | return '<'; 38 | case CHRONOS_HAPPENS_AFTER: 39 | return '>'; 40 | case CHRONOS_CONCURRENT: 41 | return '?'; 42 | case CHRONOS_WOULDLOOP: 43 | return 'O'; 44 | case CHRONOS_NOEXIST: 45 | return 'X'; 46 | default: 47 | return 'E'; 48 | } 49 | } 50 | 51 | chronos_cmp 52 | byte_to_chronos_cmp(uint8_t b) 53 | { 54 | switch (b) 55 | { 56 | case '<': 57 | return CHRONOS_HAPPENS_BEFORE; 58 | case '>': 59 | return CHRONOS_HAPPENS_AFTER; 60 | case '?': 61 | return CHRONOS_CONCURRENT; 62 | case 'O': 63 | return CHRONOS_WOULDLOOP; 64 | case 'X': 65 | default: 66 | return CHRONOS_NOEXIST; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /chronos/chronos_cmp_encode.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Cornell University 2 | // All rights reserved. 3 | // 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 7 | // * Redistributions of source code must retain the above copyright notice, 8 | // this list of conditions and the following disclaimer. 9 | // * Redistributions in binary form must reproduce the above copyright 10 | // notice, this list of conditions and the following disclaimer in the 11 | // documentation and/or other materials provided with the distribution. 12 | // * Neither the name of Chronos nor the names of its contributors may be 13 | // used to endorse or promote products derived from this software without 14 | // specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 | // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 | // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 | // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 | // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 | // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 | // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 | // POSSIBILITY OF SUCH DAMAGE. 27 | 28 | #ifndef chronos_cmp_encode_h_ 29 | #define chronos_cmp_encode_h_ 30 | 31 | // C 32 | #include 33 | 34 | // Chronos 35 | #include "chronos.h" 36 | 37 | uint8_t 38 | chronos_cmp_to_byte(chronos_cmp c); 39 | 40 | chronos_cmp 41 | byte_to_chronos_cmp(uint8_t b); 42 | 43 | #endif // chronos_cmp_encode_h_ 44 | -------------------------------------------------------------------------------- /chronos/chronos_stats_encode.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Cornell University 2 | // All rights reserved. 3 | // 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 7 | // * Redistributions of source code must retain the above copyright notice, 8 | // this list of conditions and the following disclaimer. 9 | // * Redistributions in binary form must reproduce the above copyright 10 | // notice, this list of conditions and the following disclaimer in the 11 | // documentation and/or other materials provided with the distribution. 12 | // * Neither the name of Chronos nor the names of its contributors may be 13 | // used to endorse or promote products derived from this software without 14 | // specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 | // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 | // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 | // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 | // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 | // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 | // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 | // POSSIBILITY OF SUCH DAMAGE. 27 | 28 | // Chronos 29 | #include "chronos_stats_encode.h" 30 | 31 | e::packer 32 | operator << (e::packer lhs, const chronos_stats& rhs) 33 | { 34 | return lhs << rhs.time << rhs.utime << rhs.stime << rhs.maxrss << rhs.events 35 | << rhs.count_create_event << rhs.count_acquire_references 36 | << rhs.count_release_references << rhs.count_query_order 37 | << rhs.count_assign_order; 38 | } 39 | 40 | e::unpacker 41 | operator >> (e::unpacker lhs, chronos_stats& rhs) 42 | { 43 | return lhs >> rhs.time >> rhs.utime >> rhs.stime >> rhs.maxrss >> rhs.events 44 | >> rhs.count_create_event >> rhs.count_acquire_references 45 | >> rhs.count_release_references >> rhs.count_query_order 46 | >> rhs.count_assign_order; 47 | } 48 | -------------------------------------------------------------------------------- /chronos/chronos_stats_encode.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Cornell University 2 | // All rights reserved. 3 | // 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 7 | // * Redistributions of source code must retain the above copyright notice, 8 | // this list of conditions and the following disclaimer. 9 | // * Redistributions in binary form must reproduce the above copyright 10 | // notice, this list of conditions and the following disclaimer in the 11 | // documentation and/or other materials provided with the distribution. 12 | // * Neither the name of Chronos nor the names of its contributors may be 13 | // used to endorse or promote products derived from this software without 14 | // specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 | // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 | // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 | // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 | // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 | // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 | // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 | // POSSIBILITY OF SUCH DAMAGE. 27 | 28 | #ifndef chronos_stats_encode_h_ 29 | #define chronos_stats_encode_h_ 30 | 31 | // C 32 | #include 33 | 34 | // e 35 | #include 36 | 37 | // Chronos 38 | #include "chronos/chronos.h" 39 | 40 | e::packer 41 | operator << (e::packer lhs, const chronos_stats& rhs); 42 | 43 | e::unpacker 44 | operator >> (e::unpacker lhs, chronos_stats& rhs); 45 | 46 | #endif // chronos_stats_encode_h_ 47 | -------------------------------------------------------------------------------- /chronos/network_constants.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Cornell University 2 | // All rights reserved. 3 | // 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 7 | // * Redistributions of source code must retain the above copyright notice, 8 | // this list of conditions and the following disclaimer. 9 | // * Redistributions in binary form must reproduce the above copyright 10 | // notice, this list of conditions and the following disclaimer in the 11 | // documentation and/or other materials provided with the distribution. 12 | // * Neither the name of Chronos nor the names of its contributors may be 13 | // used to endorse or promote products derived from this software without 14 | // specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 | // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 | // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 | // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 | // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 | // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 | // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 | // POSSIBILITY OF SUCH DAMAGE. 27 | 28 | #ifndef network_constants_h_ 29 | #define network_constants_h_ 30 | 31 | enum network_constant 32 | { 33 | CHRONOSNC_CREATE_EVENT, 34 | CHRONOSNC_ACQUIRE_REF, 35 | CHRONOSNC_RELEASE_REF, 36 | CHRONOSNC_QUERY_ORDER, 37 | CHRONOSNC_ASSIGN_ORDER, 38 | CHRONOSNC_GET_STATS 39 | }; 40 | 41 | #endif // network_constants_h_ 42 | -------------------------------------------------------------------------------- /client/client_constants.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Constants for client implementation. 4 | * 5 | * Created: 2014-06-19 15:20:57 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #ifndef weaver_client_client_constants_h_ 15 | #define weaver_client_client_constants_h_ 16 | 17 | #define CLIENT_MSGRECV_TIMEOUT 1000 // busybee recv timeout (ms) for clients 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /client/comm_wrapper.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Implement client busybee wrapper. 4 | * 5 | * Created: 2014-07-02 16:07:30 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #include 15 | 16 | #define weaver_debug_ 17 | #include "common/weaver_constants.h" 18 | #include "common/config_constants.h" 19 | #include "common/message_constants.h" 20 | #include "client/comm_wrapper.h" 21 | 22 | using cl::comm_wrapper; 23 | 24 | comm_wrapper :: weaver_mapper :: weaver_mapper(const configuration &config) 25 | { 26 | std::vector servers = config.get_servers(); 27 | 28 | for (const server &srv: servers) { 29 | if (srv.type == server::VT && srv.state == server::AVAILABLE) { 30 | assert(mlist.find(WEAVER_TO_BUSYBEE(srv.virtual_id)) == mlist.end()); 31 | mlist[WEAVER_TO_BUSYBEE(srv.virtual_id)] = srv.bind_to; 32 | } 33 | } 34 | } 35 | 36 | bool 37 | comm_wrapper :: weaver_mapper :: lookup(uint64_t server_id, po6::net::location *loc) 38 | { 39 | assert(server_id < NumVts); 40 | auto mlist_iter = mlist.find(WEAVER_TO_BUSYBEE(server_id)); 41 | if (mlist_iter == mlist.end()) { 42 | WDEBUG << "busybee mapper lookup fail for " << server_id << std::endl; 43 | return false; 44 | } else { 45 | *loc = mlist_iter->second; 46 | return true; 47 | } 48 | } 49 | 50 | comm_wrapper :: comm_wrapper(uint64_t bbid, const configuration &new_config) 51 | : config(new_config) 52 | , wmap(new weaver_mapper(new_config)) 53 | , bb(new busybee_st(wmap.get(), busybee_generate_id())) 54 | , bb_id(bbid) 55 | { } 56 | 57 | comm_wrapper :: ~comm_wrapper() 58 | { } 59 | 60 | void 61 | comm_wrapper :: reconfigure(const configuration &new_config) 62 | { 63 | config = new_config; 64 | wmap.reset(new weaver_mapper(new_config)); 65 | bb.reset(new busybee_st(wmap.get(), busybee_generate_id())); 66 | } 67 | 68 | #pragma GCC diagnostic push 69 | #pragma GCC diagnostic ignored "-Wdeprecated-declarations" 70 | busybee_returncode 71 | comm_wrapper :: send(uint64_t send_to, std::auto_ptr msg) 72 | { 73 | return bb->send(send_to, msg); 74 | } 75 | 76 | busybee_returncode 77 | comm_wrapper :: recv(std::auto_ptr *msg) 78 | { 79 | return bb->recv(&recv_from, msg); 80 | } 81 | #pragma GCC diagnostic pop 82 | 83 | void 84 | comm_wrapper :: drop() 85 | { 86 | bb->drop(recv_from); 87 | } 88 | -------------------------------------------------------------------------------- /client/comm_wrapper.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Client busybee wrapper. 4 | * 5 | * Created: 2014-07-02 15:58:10 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #ifndef weaver_client_comm_wrapper_h_ 15 | #define weaver_client_comm_wrapper_h_ 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "common/configuration.h" 23 | 24 | namespace cl 25 | { 26 | 27 | class comm_wrapper 28 | { 29 | public: 30 | class weaver_mapper : public busybee_mapper 31 | { 32 | private: 33 | std::unordered_map mlist; 34 | 35 | public: 36 | weaver_mapper(const configuration &config); 37 | virtual ~weaver_mapper() throw () { } 38 | virtual bool lookup(uint64_t server_id, po6::net::location *loc); 39 | 40 | private: 41 | weaver_mapper(const weaver_mapper&); 42 | weaver_mapper& operator=(const weaver_mapper&); 43 | }; 44 | 45 | private: 46 | configuration config; 47 | std::unique_ptr wmap; 48 | std::unique_ptr bb; 49 | uint64_t bb_id; 50 | uint64_t recv_from; 51 | 52 | public: 53 | comm_wrapper(uint64_t bb_id, const configuration &config); 54 | ~comm_wrapper(); 55 | void reconfigure(const configuration &new_config); 56 | #pragma GCC diagnostic push 57 | #pragma GCC diagnostic ignored "-Wdeprecated-declarations" 58 | busybee_returncode send(uint64_t send_to, std::auto_ptr msg); 59 | busybee_returncode recv(std::auto_ptr *msg); 60 | #pragma GCC diagnostic pop 61 | void drop(); 62 | }; 63 | 64 | } 65 | 66 | #endif 67 | -------------------------------------------------------------------------------- /client/datastructures.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Client datastructures. 4 | * 5 | * Author: Ayush Dubey, dubey@cs.cornell.edu 6 | * 7 | * Copyright (C) 2014, Cornell University, see the LICENSE file 8 | * for licensing agreement 9 | * =============================================================== 10 | */ 11 | 12 | #ifndef weaver_client_datastructures_h_ 13 | #define weaver_client_datastructures_h_ 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "node_prog/property.h" 22 | 23 | namespace cl 24 | { 25 | using property = node_prog::property; 26 | 27 | struct edge 28 | { 29 | std::string handle; 30 | std::string start_node, end_node; 31 | std::vector> properties; 32 | }; 33 | 34 | struct hash_edge 35 | { 36 | size_t operator()(const edge &e) const 37 | { 38 | return std::hash()(e.handle); 39 | } 40 | }; 41 | 42 | struct equals_edge 43 | { 44 | bool operator()(const edge &e1, const edge &e2) const 45 | { 46 | return e1.handle == e2.handle; 47 | } 48 | }; 49 | 50 | struct node 51 | { 52 | std::string handle; 53 | std::vector> properties; 54 | std::unordered_map out_edges; 55 | std::unordered_set aliases; 56 | }; 57 | 58 | struct hash_node 59 | { 60 | size_t operator()(const node &n) const 61 | { 62 | return std::hash()(n.handle); 63 | } 64 | }; 65 | 66 | struct equals_node 67 | { 68 | bool operator()(const node &n1, const node &n2) const 69 | { 70 | return n1.handle == n2.handle; 71 | } 72 | }; 73 | } 74 | 75 | #endif 76 | -------------------------------------------------------------------------------- /client/weaver/weaver_returncode.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Weaver client returncodes. 4 | * 5 | * Author: Ayush Dubey, dubey@cs.cornell.edu 6 | * 7 | * Copyright (C) 2015, Cornell University, see the LICENSE file 8 | * for licensing agreement 9 | * =============================================================== 10 | */ 11 | 12 | #ifndef weaver_client_weaver_returncode_h_ 13 | #define weaver_client_weaver_returncode_h_ 14 | 15 | #ifdef __cplusplus 16 | extern "C" 17 | { 18 | #endif 19 | 20 | enum weaver_client_returncode 21 | { 22 | WEAVER_CLIENT_SUCCESS, 23 | WEAVER_CLIENT_INITERROR, 24 | WEAVER_CLIENT_LOGICALERROR, 25 | WEAVER_CLIENT_NOMEM, 26 | WEAVER_CLIENT_EXCEPTION, 27 | // tx 28 | WEAVER_CLIENT_ABORT, 29 | WEAVER_CLIENT_ACTIVETX, 30 | WEAVER_CLIENT_NOACTIVETX, 31 | WEAVER_CLIENT_NOAUXINDEX, 32 | // node prog 33 | WEAVER_CLIENT_NOTFOUND, 34 | WEAVER_CLIENT_BADPROGTYPE, 35 | WEAVER_CLIENT_DLOPENERROR, 36 | // msg 37 | WEAVER_CLIENT_DISRUPTED, 38 | WEAVER_CLIENT_INTERNALMSGERROR 39 | }; 40 | 41 | const char* 42 | weaver_client_returncode_to_string(enum weaver_client_returncode code); 43 | 44 | #ifdef __cplusplus 45 | } 46 | #endif 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /client/weaver_returncode.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Implement returncode_to_string 4 | * 5 | * Author: Ayush Dubey, dubey@cs.cornell.edu 6 | * 7 | * Copyright (C) 2014, Cornell University, see the LICENSE file 8 | * for licensing agreement 9 | * =============================================================== 10 | */ 11 | 12 | #include "client/weaver/weaver_returncode.h" 13 | 14 | #ifdef __cplusplus 15 | extern "C" 16 | { 17 | #endif 18 | 19 | const char* 20 | weaver_client_returncode_to_string(enum weaver_client_returncode code) 21 | { 22 | switch (code) { 23 | case WEAVER_CLIENT_SUCCESS: 24 | return "WEAVER_CLIENT_SUCCESS"; 25 | case WEAVER_CLIENT_INITERROR: 26 | return "WEAVER_CLIENT_INITERROR"; 27 | case WEAVER_CLIENT_LOGICALERROR: 28 | return "WEAVER_CLIENT_LOGICALERROR"; 29 | case WEAVER_CLIENT_ABORT: 30 | return "WEAVER_CLIENT_ABORT"; 31 | case WEAVER_CLIENT_ACTIVETX: 32 | return "WEAVER_CLIENT_ACTIVETX"; 33 | case WEAVER_CLIENT_NOACTIVETX: 34 | return "WEAVER_CLIENT_NOACTIVETX"; 35 | case WEAVER_CLIENT_NOAUXINDEX: 36 | return "WEAVER_CLIENT_NOAUXINDEX"; 37 | case WEAVER_CLIENT_NOTFOUND: 38 | return "WEAVER_CLIENT_NOTFOUND"; 39 | case WEAVER_CLIENT_BADPROGTYPE: 40 | return "WEAVER_CLIENT_BADPROGTYPE"; 41 | case WEAVER_CLIENT_DLOPENERROR: 42 | return "WEAVER_CLIENT_DLOPENERROR"; 43 | case WEAVER_CLIENT_DISRUPTED: 44 | return "WEAVER_CLIENT_DISRUPTED"; 45 | case WEAVER_CLIENT_INTERNALMSGERROR: 46 | return "WEAVER_CLIENT_INTERNALMSGERROR"; 47 | default: 48 | return "unknown weaver returncode"; 49 | } 50 | } 51 | 52 | #ifdef __cplusplus 53 | } 54 | #endif 55 | -------------------------------------------------------------------------------- /common/.gitignore: -------------------------------------------------------------------------------- 1 | shards 2 | shards.local 3 | shards.cluster 4 | -------------------------------------------------------------------------------- /common/MurmurHash3.h: -------------------------------------------------------------------------------- 1 | //----------------------------------------------------------------------------- 2 | // MurmurHash3 was written by Austin Appleby, and is placed in the public 3 | // domain. The author hereby disclaims copyright to this source code. 4 | 5 | #ifndef _MURMURHASH3_H_ 6 | #define _MURMURHASH3_H_ 7 | 8 | //----------------------------------------------------------------------------- 9 | // Platform-specific functions and macros 10 | 11 | // Microsoft Visual Studio 12 | 13 | #if defined(_MSC_VER) && (_MSC_VER < 1600) 14 | 15 | typedef unsigned char uint8_t; 16 | typedef unsigned int uint32_t; 17 | typedef unsigned __int64 uint64_t; 18 | 19 | // Other compilers 20 | 21 | #else // defined(_MSC_VER) 22 | 23 | #include 24 | 25 | #endif // !defined(_MSC_VER) 26 | 27 | //----------------------------------------------------------------------------- 28 | 29 | void MurmurHash3_x86_32 ( const void * key, int len, uint32_t seed, void * out ); 30 | 31 | void MurmurHash3_x86_128 ( const void * key, int len, uint32_t seed, void * out ); 32 | 33 | void MurmurHash3_x64_128 ( const void * key, int len, uint32_t seed, void * out ); 34 | 35 | //----------------------------------------------------------------------------- 36 | 37 | #endif // _MURMURHASH3_H_ 38 | -------------------------------------------------------------------------------- /common/bool_vector.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Helper methods for std::vector with some 4 | * bitset functionality. 5 | * 6 | * Created: 2014-06-25 12:02:54 7 | * 8 | * Author: Ayush Dubey, dubey@cs.cornell.edu 9 | * 10 | * Copyright (C) 2013, Cornell University, see the LICENSE file 11 | * for licensing agreement 12 | * =============================================================== 13 | */ 14 | 15 | #ifndef weaver_common_bool_vector_h_ 16 | #define weaver_common_bool_vector_h_ 17 | 18 | #include 19 | 20 | namespace weaver_util 21 | { 22 | inline bool 23 | all(const std::vector &v) 24 | { 25 | for (size_t i = 0; i < v.size(); i++) { 26 | if (!v[i]) { 27 | return false; 28 | } 29 | } 30 | return true; 31 | } 32 | 33 | inline bool 34 | any(const std::vector &v) 35 | { 36 | for (size_t i = 0; i < v.size(); i++) { 37 | if (v[i]) { 38 | return true; 39 | } 40 | } 41 | return false; 42 | } 43 | 44 | inline bool 45 | none(const std::vector &v) 46 | { 47 | for (size_t i = 0; i < v.size(); i++) { 48 | if (v[i]) { 49 | return false; 50 | } 51 | } 52 | return true; 53 | } 54 | 55 | inline void 56 | set_bool_vec(std::vector &vec, bool val) 57 | { 58 | for (size_t i = 0; i < vec.size(); i++) { 59 | vec[i] = val; 60 | } 61 | } 62 | 63 | inline void 64 | set_all(std::vector &v) 65 | { 66 | set_bool_vec(v, true); 67 | } 68 | 69 | inline void 70 | reset_all(std::vector &v) 71 | { 72 | set_bool_vec(v, false); 73 | } 74 | } 75 | 76 | #endif 77 | -------------------------------------------------------------------------------- /common/cache_constants.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Weaver cache constants. 4 | * 5 | * Created: 2014-06-19 13:22:42 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #ifndef weaver_common_cache_constants_h_ 15 | #define weaver_common_cache_constants_h_ 16 | 17 | extern uint16_t MaxCacheEntries; 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /common/clock.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Timing-related functions implementation. 4 | * 5 | * Created: 2014-05-29 12:22:12 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #include 15 | #ifdef __MACH__ 16 | #include 17 | #include 18 | #endif 19 | 20 | #include "common/clock.h" 21 | 22 | using wclock::weaver_timer; 23 | 24 | void 25 | weaver_timer :: get_clock() 26 | { 27 | #ifdef __MACH__ // OS X does not have clock_gettime, use clock_get_time 28 | clock_serv_t cclock; 29 | mach_timespec_t mts; 30 | host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock); 31 | clock_get_time(cclock, &mts); 32 | mach_port_deallocate(mach_task_self(), cclock); 33 | ts.tv_sec = mts.tv_sec; 34 | ts.tv_nsec = mts.tv_nsec; 35 | #else 36 | clock_gettime(CLOCK_MONOTONIC, &ts); 37 | #endif 38 | } 39 | 40 | uint64_t 41 | weaver_timer :: get_time_elapsed() 42 | { 43 | uint64_t ret = 0; 44 | get_clock(); 45 | ret += ts.tv_sec*GIGA + ts.tv_nsec; 46 | return ret; 47 | } 48 | 49 | uint64_t 50 | weaver_timer :: get_time_elapsed_millis() 51 | { 52 | uint64_t ret = get_time_elapsed() / MEGA; 53 | return ret; 54 | } 55 | 56 | uint64_t 57 | weaver_timer :: get_real_time() 58 | { 59 | uint64_t ret = 0; 60 | clock_gettime(CLOCK_REALTIME, &ts); 61 | ret += ts.tv_sec*GIGA + ts.tv_nsec; 62 | return ret; 63 | } 64 | 65 | uint64_t 66 | weaver_timer :: get_real_time_millis() 67 | { 68 | uint64_t ret = get_real_time() / MEGA; 69 | return ret; 70 | } 71 | -------------------------------------------------------------------------------- /common/clock.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Get current clock value, for both Posix and 4 | * Mac 5 | * 6 | * Created: 06/21/13 14:43:40 7 | * 8 | * Author: Ayush Dubey, dubey@cs.cornell.edu 9 | * 10 | * Copyright (C) 2013, Cornell University, see the LICENSE file 11 | * for licensing agreement 12 | * =============================================================== 13 | */ 14 | 15 | #ifndef weaver_common_clock_h_ 16 | #define weaver_common_clock_h_ 17 | 18 | #include 19 | 20 | #include "common/weaver_constants.h" 21 | 22 | namespace wclock 23 | { 24 | class weaver_timer 25 | { 26 | public: 27 | timespec ts; 28 | void get_clock(); 29 | weaver_timer() { get_clock(); } 30 | double get_secs() { return ts.tv_sec + ((double)ts.tv_nsec)/GIGA; } 31 | uint64_t get_nanosecs() { return ts.tv_nsec + ((uint64_t)ts.tv_sec)*GIGA; } 32 | // return nanosecs elapsed since some fixed time 33 | uint64_t get_time_elapsed(); 34 | // get_time_elapsed / 1000000 35 | uint64_t get_time_elapsed_millis(); 36 | uint64_t get_real_time(); 37 | uint64_t get_real_time_millis(); 38 | weaver_timer& operator-=(const weaver_timer &rhs); 39 | }; 40 | 41 | inline weaver_timer 42 | operator-(weaver_timer lhs, const weaver_timer &rhs) 43 | { 44 | if ((lhs.ts.tv_nsec-rhs.ts.tv_nsec)<0) { 45 | lhs.ts.tv_sec = lhs.ts.tv_sec - rhs.ts.tv_sec - 1; 46 | lhs.ts.tv_nsec = GIGA + lhs.ts.tv_nsec - rhs.ts.tv_nsec; 47 | } else { 48 | lhs.ts.tv_sec -= rhs.ts.tv_sec; 49 | lhs.ts.tv_nsec -= rhs.ts.tv_nsec; 50 | } 51 | return lhs; 52 | } 53 | 54 | inline weaver_timer& 55 | weaver_timer :: operator-=(const weaver_timer &rhs) 56 | { 57 | *this = *this - rhs; 58 | return *this; 59 | } 60 | } 61 | #endif 62 | -------------------------------------------------------------------------------- /common/enum_serialization.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: serialization for Weaver enums 4 | * 5 | * Author: Ayush Dubey, dubey@cs.cornell.edu 6 | * 7 | * Copyright (C) 2015, Cornell University, see the LICENSE file 8 | * for licensing agreement 9 | * =============================================================== 10 | */ 11 | 12 | #ifndef weaver_common_enum_serialization_h_ 13 | #define weaver_common_enum_serialization_h_ 14 | 15 | #include 16 | 17 | #include "common/transaction.h" 18 | #include "common/property_predicate.h" 19 | #include "node_prog/node_prog_type.h" 20 | 21 | namespace message 22 | { 23 | uint64_t size(void*, const enum transaction::update_type&); 24 | uint64_t size(void*, const enum transaction::tx_type&); 25 | uint64_t size(void*, const enum predicate::relation&); 26 | uint64_t size(void*, const enum node_prog::prog_type&); 27 | void pack_buffer(e::packer &packer, void*, const enum transaction::update_type &t); 28 | void pack_buffer(e::packer &packer, void*, const enum transaction::tx_type &t); 29 | void pack_buffer(e::packer &packer, void*, const enum predicate::relation &t); 30 | void pack_buffer(e::packer &packer, void*, const enum node_prog::prog_type &t); 31 | void unpack_buffer(e::unpacker &unpacker, void*, enum transaction::update_type &t); 32 | void unpack_buffer(e::unpacker &unpacker, void*, enum transaction::tx_type &t); 33 | void unpack_buffer(e::unpacker &unpacker, void*, enum predicate::relation &t); 34 | void unpack_buffer(e::unpacker &unpacker, void*, enum node_prog::prog_type &t); 35 | } 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /common/ids.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Macros for ID packers and unpackers. 4 | * 5 | * Created: 2014-02-08 17:26:46 6 | * 7 | * Author: Robert Escriva, escriva@cs.cornell.edu 8 | * Ayush Dubey, dubey@cs.cornell.edu 9 | * 10 | * Copyright (C) 2013, Cornell University, see the LICENSE file 11 | * for licensing agreement 12 | * =============================================================== 13 | */ 14 | 15 | // Most of the following code has been 'borrowed' from 16 | // Robert Escriva's HyperDex coordinator. 17 | // see https://github.com/rescrv/HyperDex for the original code. 18 | 19 | // Weaver 20 | #include "common/ids.h" 21 | 22 | #define CREATE_ID(TYPE) \ 23 | std::ostream& \ 24 | operator << (std::ostream& lhs, const TYPE ## _id& rhs) \ 25 | { \ 26 | return lhs << #TYPE "(" << rhs.get() << ")"; \ 27 | } \ 28 | e::packer \ 29 | operator << (e::packer pa, const TYPE ## _id& rhs) \ 30 | { \ 31 | return pa << rhs.get(); \ 32 | } \ 33 | e::unpacker \ 34 | operator >> (e::unpacker up, TYPE ## _id& rhs) \ 35 | { \ 36 | uint64_t id; \ 37 | up = up >> id; \ 38 | rhs = TYPE ## _id(id); \ 39 | return up; \ 40 | } 41 | 42 | CREATE_ID(server) 43 | 44 | #undef CREATE_ID 45 | -------------------------------------------------------------------------------- /common/ids.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Macros for generating ID classes, which are 4 | * basically wrappers around a uint64_t. 5 | * 6 | * Created: 2014-02-08 17:26:46 7 | * 8 | * Author: Robert Escriva, escriva@cs.cornell.edu 9 | * Ayush Dubey, dubey@cs.cornell.edu 10 | * 11 | * Copyright (C) 2013, Cornell University, see the LICENSE file 12 | * for licensing agreement 13 | * =============================================================== 14 | */ 15 | 16 | // Most of the following code has been 'borrowed' from 17 | // Robert Escriva's HyperDex coordinator. 18 | // see https://github.com/rescrv/HyperDex for the original code. 19 | 20 | #ifndef weaver_common_ids_h_ 21 | #define weaver_common_ids_h_ 22 | 23 | // C 24 | #include 25 | 26 | // C++ 27 | #include 28 | 29 | // e 30 | #include 31 | #include 32 | 33 | // An ID is a simple wrapper around uint64_t in order to prevent devs from 34 | // accidently using one type of ID as another. 35 | 36 | #define OPERATOR(TYPE, OP) \ 37 | inline bool \ 38 | operator OP (const TYPE ## _id& lhs, const TYPE ## _id& rhs) \ 39 | { \ 40 | return lhs.get() OP rhs.get(); \ 41 | } 42 | #define CREATE_ID(TYPE) \ 43 | class TYPE ## _id \ 44 | { \ 45 | public: \ 46 | TYPE ## _id() : m_id(0) {} \ 47 | explicit TYPE ## _id(uint64_t id) : m_id(id) {} \ 48 | public: \ 49 | uint64_t get() const { return m_id; } \ 50 | private: \ 51 | uint64_t m_id; \ 52 | }; \ 53 | std::ostream& \ 54 | operator << (std::ostream& lhs, const TYPE ## _id& rhs); \ 55 | inline size_t \ 56 | pack_size(const TYPE ## _id&) \ 57 | { \ 58 | uint64_t i = 0; \ 59 | return e::pack_size(i); \ 60 | } \ 61 | e::packer \ 62 | operator << (e::packer pa, const TYPE ## _id& rhs); \ 63 | e::unpacker \ 64 | operator >> (e::unpacker up, TYPE ## _id& rhs); \ 65 | OPERATOR(TYPE, <) \ 66 | OPERATOR(TYPE, <=) \ 67 | OPERATOR(TYPE, ==) \ 68 | OPERATOR(TYPE, !=) \ 69 | OPERATOR(TYPE, >=) \ 70 | OPERATOR(TYPE, >) 71 | 72 | CREATE_ID(server) 73 | 74 | #undef OPERATOR 75 | #undef CREATE_ID 76 | 77 | #endif 78 | -------------------------------------------------------------------------------- /common/message_constants.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Constant values related to messaging layer. 4 | * 5 | * Created: 2014-06-19 12:56:05 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #ifndef weaver_common_message_constants_h_ 15 | #define weaver_common_message_constants_h_ 16 | 17 | #define ID_INCR (1ULL << 32ULL) 18 | #define WEAVER_TO_BUSYBEE(x) (x+ID_INCR) 19 | #define BUSYBEE_TO_WEAVER(x) (x-ID_INCR) 20 | 21 | #endif 22 | -------------------------------------------------------------------------------- /common/prog_write_and_dlopen.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Implement write_and_dlopen 4 | * 5 | * Author: Ayush Dubey, dubey@cs.cornell.edu 6 | * 7 | * Copyright (C) 2015, Cornell University, see the LICENSE file 8 | * for licensing agreement 9 | * =============================================================== 10 | */ 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | #define weaver_debug_ 17 | #include 18 | #include "common/weaver_constants.h" 19 | #include "common/prog_write_and_dlopen.h" 20 | 21 | std::shared_ptr 22 | write_and_dlopen(std::vector &buf, const std::string &prog_handle) 23 | { 24 | std::string dir_unexp = "~/weaver_runtime"; 25 | wordexp_t dir_exp; 26 | wordexp(dir_unexp.c_str(), &dir_exp, 0); 27 | std::string dir = dir_exp.we_wordv[0]; 28 | mkdir(dir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); 29 | 30 | std::string so_file_name = dir + "/" + prog_handle; 31 | WDEBUG << "so file name = " << so_file_name << std::endl; 32 | 33 | FILE *test_file = fopen(so_file_name.c_str(), "r"); 34 | bool created; 35 | if (test_file) { 36 | fclose(test_file); 37 | created = false; 38 | } else { 39 | std::ofstream write_so; 40 | write_so.open(so_file_name, std::ofstream::out | std::ofstream::binary); 41 | write_so.write((const char*)&buf[0], buf.size()); 42 | write_so.close(); 43 | created = true; 44 | } 45 | 46 | void *prog_ptr = dlopen(so_file_name.c_str(), RTLD_NOW); 47 | 48 | if (created) { 49 | remove(so_file_name.c_str()); 50 | } 51 | 52 | if (prog_ptr == NULL) { 53 | WDEBUG << "dlopen error: " << dlerror() << std::endl; 54 | WDEBUG << "failed registering node prog" << std::endl; 55 | return nullptr; 56 | } else { 57 | auto prog_table = std::make_shared(prog_ptr); 58 | return prog_table; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /common/prog_write_and_dlopen.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Write a vector of bytes that represents an so 4 | * file to a tmp file, and then dlopen that file. 5 | * 6 | * Author: Ayush Dubey, dubey@cs.cornell.edu 7 | * 8 | * Copyright (C) 2015, Cornell University, see the LICENSE file 9 | * for licensing agreement 10 | * =============================================================== 11 | */ 12 | 13 | #include "node_prog/dynamic_prog_table.h" 14 | 15 | std::shared_ptr 16 | write_and_dlopen(std::vector &buf, const std::string &prog_handle); 17 | -------------------------------------------------------------------------------- /common/property_predicate.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Implement property predicate. 4 | * 5 | * Author: Ayush Dubey, dubey@cs.cornell.edu 6 | * 7 | * Copyright (C) 2014, Cornell University, see the LICENSE file 8 | * for licensing agreement 9 | * =============================================================== 10 | */ 11 | 12 | #include 13 | 14 | #define weaver_debug_ 15 | #include "common/weaver_constants.h" 16 | #include "common/property_predicate.h" 17 | 18 | using predicate::prop_predicate; 19 | 20 | bool 21 | prop_predicate :: check(const node_prog::property &prop) const 22 | { 23 | switch (rel) { 24 | case EQUALS: 25 | return key == prop.get_key() && prop.get_value() == value; 26 | break; 27 | 28 | case LESS: 29 | return key == prop.get_key() && prop.get_value() < value; 30 | break; 31 | 32 | case GREATER: 33 | return key == prop.get_key() && prop.get_value() > value; 34 | break; 35 | 36 | case LESS_EQUAL: 37 | return key == prop.get_key() && prop.get_value() <= value; 38 | break; 39 | 40 | case GREATER_EQUAL: 41 | return key == prop.get_key() && prop.get_value() >= value; 42 | break; 43 | 44 | case STARTS_WITH: 45 | return key == prop.get_key() 46 | && prop.get_value().size() >= value.size() 47 | && prop.get_value().compare(0, value.size(), value) == 0; 48 | break; 49 | 50 | case ENDS_WITH: 51 | return key == prop.get_key() 52 | && prop.get_value().size() >= value.size() 53 | && prop.get_value().compare(prop.get_value().size()-value.size(), value.size(), value) == 0; 54 | break; 55 | 56 | case CONTAINS: 57 | return key == prop.get_key() 58 | && prop.get_value().find(value) != std::string::npos; 59 | break; 60 | 61 | default: 62 | WDEBUG << "bad rel " << rel << std::endl; 63 | return false; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /common/property_predicate.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: A predicate based on node/edge property. 4 | * 5 | * Author: Ayush Dubey, dubey@cs.cornell.edu 6 | * 7 | * Copyright (C) 2014, Cornell University, see the LICENSE file 8 | * for licensing agreement 9 | * =============================================================== 10 | */ 11 | 12 | #ifndef weaver_common_property_predicate_h_ 13 | #define weaver_common_property_predicate_h_ 14 | 15 | #include 16 | 17 | #include "node_prog/property.h" 18 | 19 | namespace predicate 20 | { 21 | enum relation 22 | { 23 | EQUALS, 24 | LESS, 25 | GREATER, 26 | LESS_EQUAL, 27 | GREATER_EQUAL, 28 | STARTS_WITH, 29 | ENDS_WITH, 30 | CONTAINS 31 | }; 32 | 33 | struct prop_predicate 34 | { 35 | std::string key; 36 | std::string value; 37 | relation rel; 38 | 39 | bool check(const node_prog::property &prop) const; 40 | }; 41 | } 42 | 43 | #endif 44 | -------------------------------------------------------------------------------- /common/server.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Server state for cluster configuration. 4 | * 5 | * Created: 2014-02-08 17:26:46 6 | * 7 | * Author: Robert Escriva, escriva@cs.cornell.edu 8 | * Ayush Dubey, dubey@cs.cornell.edu 9 | * 10 | * Copyright (C) 2013, Cornell University, see the LICENSE file 11 | * for licensing agreement 12 | * =============================================================== 13 | */ 14 | 15 | // Most of the following code has been 'borrowed' from 16 | // Robert Escriva's HyperDex coordinator. 17 | // see https://github.com/rescrv/HyperDex for the original code. 18 | 19 | #ifndef weaver_common_server_h_ 20 | #define weaver_common_server_h_ 21 | 22 | // po6 23 | #include 24 | 25 | // e 26 | #include 27 | 28 | // Weaver 29 | #include "common/ids.h" 30 | 31 | class server 32 | { 33 | public: 34 | enum state_t 35 | { 36 | ASSIGNED = 1, 37 | NOT_AVAILABLE = 2, 38 | AVAILABLE = 3, 39 | SHUTDOWN = 4, 40 | KILLED = 5 41 | }; 42 | enum type_t 43 | { 44 | UNDEF = 0, 45 | SHARD = 1, 46 | VT = 2, 47 | BACKUP_SHARD = 3, 48 | BACKUP_VT = 4 49 | }; 50 | static const char* to_string(state_t state); 51 | static const char* to_string(type_t type); 52 | 53 | public: 54 | server(); 55 | explicit server(const server_id&); 56 | bool operator==(const server &other) const; 57 | bool operator!=(const server &other) const; 58 | 59 | public: 60 | state_t state; 61 | server_id id; // random int 62 | // increasing counter signifying order in which this server was registered 63 | uint64_t weaver_id; 64 | // if virtual id of a server is v, it is the v^{th} vt or shard that was registered 65 | // to send a message to a vt with virtual id v, you send(v, ...) 66 | // to send a message to a shard with virtual id v, you send to (v+NumVts, ...) 67 | uint64_t virtual_id; 68 | type_t type; 69 | po6::net::location bind_to; 70 | }; 71 | 72 | bool 73 | operator < (const server& lhs, const server& rhs); 74 | 75 | e::packer 76 | operator << (e::packer lhs, const server& rhs); 77 | e::unpacker 78 | operator >> (e::unpacker lhs, server& rhs); 79 | size_t 80 | pack_size(const server& p); 81 | 82 | #endif 83 | -------------------------------------------------------------------------------- /common/server_manager_returncode.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Return codes from server manager to shards 4 | * and timestampers. 5 | * 6 | * Created: 2014-02-09 17:11:00 7 | * 8 | * Author: Robert Escriva, escriva@cs.cornell.edu 9 | * Ayush Dubey, dubey@cs.cornell.edu 10 | * 11 | * Copyright (C) 2013, Cornell University, see the LICENSE file 12 | * for licensing agreement 13 | * =============================================================== 14 | */ 15 | 16 | // Most of the following code has been 'borrowed' from 17 | // Robert Escriva's HyperDex coordinator. 18 | // see https://github.com/rescrv/HyperDex for the original code. 19 | 20 | #ifndef weaver_common_server_manager_returncode_h_ 21 | #define weaver_common_server_manager_returncode_h_ 22 | 23 | #include 24 | 25 | #define XSTR(x) #x 26 | #define STR(x) XSTR(x) 27 | #define STRINGIFY(x) case (x): lhs << STR(x); break 28 | #define CSTRINGIFY(x) case (x): return STR(x); 29 | 30 | // occupies [8832, 8960) 31 | // these are hardcoded as byte strings in coordinator/coordinator.cc 32 | // keep them in sync 33 | enum server_manager_returncode 34 | { 35 | COORD_SUCCESS = 8832, 36 | COORD_MALFORMED = 8833, 37 | COORD_DUPLICATE = 8834, 38 | COORD_NOT_FOUND = 8835, 39 | COORD_UNINITIALIZED = 8837, 40 | COORD_NO_CAN_DO = 8839 41 | }; 42 | 43 | //std::ostream& 44 | //operator << (std::ostream& lhs, server_manager_returncode rhs) 45 | //{ 46 | // switch (rhs) 47 | // { 48 | // STRINGIFY(COORD_SUCCESS); 49 | // STRINGIFY(COORD_MALFORMED); 50 | // STRINGIFY(COORD_DUPLICATE); 51 | // STRINGIFY(COORD_NOT_FOUND); 52 | // STRINGIFY(COORD_UNINITIALIZED); 53 | // STRINGIFY(COORD_NO_CAN_DO); 54 | // default: 55 | // lhs << "unknown server_manager_returncode"; 56 | // break; 57 | // } 58 | // 59 | // return lhs; 60 | //} 61 | 62 | #endif 63 | -------------------------------------------------------------------------------- /common/transaction.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Implementation of transaction:: methods 4 | * 5 | * Created: 2014-09-18 15:28:18 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #include "common/transaction.h" 15 | 16 | using transaction::pending_tx; 17 | 18 | pending_tx :: pending_tx(tx_type t) 19 | : type(t) 20 | , vt_seq(42) 21 | , nop(nullptr) 22 | { } 23 | 24 | pending_tx :: ~pending_tx() 25 | { } 26 | 27 | std::shared_ptr 28 | pending_tx :: copy_fail_transaction() 29 | { 30 | std::shared_ptr fail_tx = std::make_shared(FAIL); 31 | fail_tx->id = id; 32 | fail_tx->timestamp = timestamp; 33 | fail_tx->vt_seq = vt_seq; 34 | return fail_tx; 35 | } 36 | -------------------------------------------------------------------------------- /common/transaction.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Data structures for unprocessed transactions. 4 | * 5 | * Created: 08/31/2013 03:17:04 PM 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #ifndef weaver_common_transaction_h_ 15 | #define weaver_common_transaction_h_ 16 | 17 | #include 18 | #include 19 | #include 20 | 21 | #include "common/vclock.h" 22 | 23 | namespace transaction 24 | { 25 | enum update_type 26 | { 27 | NODE_CREATE_REQ, 28 | EDGE_CREATE_REQ, 29 | NODE_DELETE_REQ, 30 | EDGE_DELETE_REQ, 31 | NODE_SET_PROPERTY, 32 | EDGE_SET_PROPERTY, 33 | ADD_AUX_INDEX 34 | }; 35 | 36 | // store state for update received from client but not yet completed 37 | struct pending_update 38 | { 39 | update_type type; 40 | std::string handle, handle1, handle2, alias1, alias2; 41 | uint64_t loc1, loc2, sender; 42 | std::unique_ptr key, value; 43 | }; 44 | 45 | struct nop_data 46 | { 47 | vc::vclock_t max_done_clk; 48 | uint64_t outstanding_progs; 49 | std::vector shard_node_count; 50 | std::vector done_txs; 51 | }; 52 | 53 | using tx_list_t = std::vector>; 54 | 55 | enum tx_type 56 | { 57 | // internal 58 | FAIL, 59 | EPOCH_CHANGE, 60 | // to shards 61 | NOP, 62 | UPDATE 63 | }; 64 | 65 | struct pending_tx 66 | { 67 | tx_type type; 68 | uint64_t id; // unique tx id, assigned by client 69 | vc::vclock timestamp; // vector timestamp 70 | uint64_t vt_seq; // tx seq number at the timestamper 71 | uint64_t qts; // queue timestamp 72 | std::vector shard_write; // which shards are involved in the write 73 | 74 | tx_list_t writes; // if this is a write tx 75 | uint64_t sender; // client to which we need to reply for write tx 76 | 77 | std::shared_ptr nop; // if this is a nop 78 | 79 | uint64_t new_epoch; // if this is an epoch change 80 | 81 | pending_tx(tx_type t); 82 | ~pending_tx(); 83 | 84 | std::shared_ptr copy_fail_transaction(); 85 | }; 86 | 87 | } 88 | 89 | #endif 90 | -------------------------------------------------------------------------------- /common/vclock.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Implementation of vclock 4 | * 5 | * Created: 2014-06-19 13:14:44 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #include "common/vclock.h" 15 | #include "common/config_constants.h" 16 | 17 | using vc::vclock; 18 | 19 | vclock :: vclock(uint64_t vtid, uint64_t clk_init) 20 | : vt_id(vtid) 21 | , clock(std::vector(ClkSz, clk_init)) 22 | { 23 | assert(vt_id < NumVts || vt_id == UINT64_MAX); 24 | } 25 | 26 | vclock :: vclock(uint64_t vtid, vclock_t &vclk) 27 | : vt_id(vtid) 28 | , clock(vclk) 29 | { 30 | assert(vt_id < NumVts || vt_id == UINT64_MAX); 31 | } 32 | 33 | void 34 | vclock :: new_epoch(uint64_t new_epoch) 35 | { 36 | assert(clock[0] < new_epoch); 37 | clock = vclock_t(ClkSz, 0); 38 | clock[0] = new_epoch; 39 | } 40 | 41 | void 42 | vclock :: update_clock(vc::vclock &other) 43 | { 44 | uint64_t vtid = other.vt_id; 45 | assert(vtid < NumVts); 46 | if (clock[0] == other.clock[0] && clock[vtid+1] < other.clock[vtid+1]) { 47 | clock[vtid+1] = other.clock[vtid+1]; 48 | } 49 | } 50 | 51 | bool 52 | vclock :: operator==(const vclock &other) const 53 | { 54 | return vt_id == other.vt_id && clock == other.clock; 55 | } 56 | 57 | bool 58 | vclock :: operator!=(const vclock &other) const 59 | { 60 | return !(*this==other); 61 | } 62 | -------------------------------------------------------------------------------- /common/vclock.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Vector of event counters, one for each vector 4 | * timestamper. Basically a wrapper class around 5 | * a vector of ints. 6 | * 7 | * Created: 01/15/2013 06:23:23 PM 8 | * 9 | * Author: Ayush Dubey, dubey@cs.cornell.edu 10 | * 11 | * Copyright (C) 2013, Cornell University, see the LICENSE file 12 | * for licensing agreement 13 | * =============================================================== 14 | */ 15 | 16 | #ifndef weaver_common_vclock_h_ 17 | #define weaver_common_vclock_h_ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "common/utils.h" 25 | 26 | namespace vc 27 | { 28 | typedef std::vector vclock_t; 29 | typedef std::vector qtimestamp_t; 30 | 31 | class vclock 32 | { 33 | public: 34 | uint64_t vt_id; 35 | vclock_t clock; 36 | 37 | vclock() : vt_id(UINT64_MAX) { } 38 | vclock(uint64_t vt_id, uint64_t clk_init); 39 | vclock(uint64_t vt_id, vclock_t &vclk); 40 | void new_epoch(uint64_t epoch_num); 41 | uint64_t get_clock() const { return clock[vt_id+1]; } 42 | uint64_t get_epoch() const { return clock[0]; } 43 | void increment_clock() { clock[vt_id+1]++; } 44 | void update_clock(vc::vclock &other); 45 | 46 | bool operator==(const vclock &rhs) const; 47 | bool operator!=(const vclock &rhs) const; 48 | }; 49 | 50 | using vclock_ptr_t = std::shared_ptr; 51 | } 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /common/weaver_constants.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Constant values used across the project 4 | * 5 | * Created: 01/15/2013 03:01:04 PM 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #include 15 | #include 16 | #include 17 | 18 | // debugging 19 | #ifndef WDEBUG 20 | #ifdef weaver_debug_ 21 | #define WDEBUG LOG(INFO) 22 | #else 23 | #define WDEBUG if (false) std::cerr << __FILE__ << ":" << __LINE__ << " " 24 | #endif 25 | #endif 26 | 27 | #ifndef weaver_common_constants_h_ 28 | #define weaver_common_constants_h_ 29 | 30 | // if benchmarking 31 | //#define weaver_benchmark_ 32 | 33 | // unused expression for no warnings 34 | #define UNUSED(exp) do { (void)(exp); } while (0) 35 | 36 | #define NANO (1000000000ULL) 37 | #define GIGA (1000000000ULL) 38 | #define MEGA (1000000UL) 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /conf/weaver.yaml: -------------------------------------------------------------------------------- 1 | # Typical Weaver configuration file. 2 | # Copyright (C) 2014 Ayush Dubey 3 | 4 | # Number of vector timestampers. 5 | # Default: 1 6 | num_vts : 1 7 | 8 | # Number of cache entries per node for node programs. 0 means no caching. 9 | # Default: 0 10 | max_cache_entries : 0 11 | 12 | # HyperDex coordinator ip addrs and corresponding ports. 13 | # Used for starting up cluster and for connecting to HyperDex. 14 | hyperdex_coord: 15 | - 127.0.0.1 : 7982 16 | 17 | # HyperDex daemons ip addrs and corresponding ports. 18 | # Used for starting up cluster. 19 | hyperdex_daemons: 20 | - 127.0.0.1 : 8012 21 | 22 | # Kronos replicated state machine ip addrs and corresponding ports. 23 | # Used for starting up cluster and connecting to Kronos. 24 | kronos: 25 | - 127.0.0.1 : 1992 26 | 27 | # Weaver coordinator ip addrs and ports. 28 | weaver_coord: 29 | - 127.0.0.1 : 2002 30 | 31 | # Boolean aux_index controls whether Weaver maintains auxiliary indexing. 32 | # Auxiliary indexing can be used to access edges and attach arbitrary handles to nodes. 33 | # Default: false 34 | aux_index: true 35 | 36 | # BulkLoadPropertyValueDelimiter is a character used to delimit property lists while bulk loading graphml graphs. 37 | # Default: '\0' 38 | #bulk_load_property_value_delimiter: "," 39 | 40 | # BulkLoadNodeAliasKey is the key of node property that corresponds to the node aliases while bulk loading graphml graphs. 41 | # Default: "" 42 | bulk_load_node_alias_key: "alias" 43 | 44 | # BulkLoadEdgeHandlePrefix is the prefix-string attached to edge handles during bulk loading graphs that do not specify edge handles. 45 | # Default: "e" 46 | bulk_load_edge_handle_prefix: "" 47 | 48 | # nodes per map in memory during bulk load 49 | nodes_per_map: 100000 50 | 51 | # max memory to be used as ratio of total system memory 52 | max_memory: 0.75 53 | 54 | # number of nanoseconds between successive read nops 55 | # default: 100000 56 | rd_nop_period: 10000 57 | 58 | # number of nanoseconds between successive write nops (in expectation) 59 | # default: 10000000 60 | wr_nop_period: 1000000 61 | 62 | # number of nanoseconds between vt gossip 63 | # default: 10000 64 | clk_gossip_period: 10000 65 | -------------------------------------------------------------------------------- /coordinator/.gitignore: -------------------------------------------------------------------------------- 1 | timestamper 2 | -------------------------------------------------------------------------------- /coordinator/blocked_prog.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: DS for storing node prog state while shard is 4 | * restoring graph data. 5 | * 6 | * Created: 2014-11-11 09:35:38 7 | * 8 | * Author: Ayush Dubey, dubey@cs.cornell.edu 9 | * 10 | * Copyright (C) 2014, Cornell University, see the LICENSE file 11 | * for licensing agreement 12 | * =============================================================== 13 | */ 14 | 15 | #ifndef weaver_coordinator_blocked_prog_h_ 16 | #define weaver_coordinator_blocked_prog_h_ 17 | 18 | namespace coordinator 19 | { 20 | struct blocked_prog 21 | { 22 | uint64_t client; 23 | std::unique_ptr msg; 24 | 25 | blocked_prog(uint64_t c, std::unique_ptr m) 26 | : client(c) 27 | , msg(std::move(m)) 28 | { } 29 | }; 30 | } 31 | #endif 32 | -------------------------------------------------------------------------------- /coordinator/current_prog.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Metadata for a currently executing node prog. 4 | * 5 | * Created: 2014-02-24 14:07:52 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013-2014, Cornell University, see the LICENSE 10 | * file for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #ifndef weaver_coordinator_current_prog_h_ 15 | #define weaver_coordinator_current_prog_h_ 16 | 17 | namespace coordinator 18 | { 19 | struct current_prog 20 | { 21 | uint64_t req_id, client; 22 | std::unique_ptr vclk; 23 | 24 | current_prog(uint64_t rid, uint64_t cl, const vc::vclock &vc) 25 | : req_id(rid) 26 | , client(cl) 27 | , vclk(new vc::vclock(vc)) 28 | { } 29 | 30 | current_prog() : req_id(UINT64_MAX), client(UINT64_MAX) { } 31 | }; 32 | } 33 | 34 | #endif 35 | -------------------------------------------------------------------------------- /coordinator/register_node_prog_state.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: State saved for registering node prog 4 | * 5 | * Author: Ayush Dubey, dubey@cs.cornell.edu 6 | * 7 | * Copyright (C) 2015, Cornell University, see the LICENSE file 8 | * for licensing agreement 9 | * =============================================================== 10 | */ 11 | 12 | #ifndef weaver_coordinator_register_node_prog_state_h_ 13 | #define weaver_coordinator_register_node_prog_state_h_ 14 | 15 | namespace coordinator 16 | { 17 | struct register_node_prog_state 18 | { 19 | uint64_t client; 20 | std::unordered_set shards; 21 | bool success; 22 | 23 | register_node_prog_state() : success(true) { } 24 | }; 25 | } 26 | #endif 27 | -------------------------------------------------------------------------------- /coordinator/symtable.c: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Replicant state machine definition. 4 | * 5 | * Created: 2014-02-10 12:06:06 6 | * 7 | * Author: Robert Escriva, escriva@cs.cornell.edu 8 | * Ayush Dubey, dubey@cs.cornell.edu 9 | * 10 | * Copyright (C) 2013, Cornell University, see the LICENSE file 11 | * for licensing agreement 12 | * =============================================================== 13 | */ 14 | 15 | // Most of the following code has been 'borrowed' from 16 | // Robert Escriva's HyperDex coordinator. 17 | // see https://github.com/rescrv/HyperDex for the original code. 18 | 19 | /* Replicant */ 20 | #include 21 | 22 | /* Weaver */ 23 | #include "coordinator/transitions.h" 24 | 25 | #pragma GCC diagnostic push 26 | #pragma GCC diagnostic ignored "-pedantic" 27 | 28 | struct state_machine rsm = { 29 | weaver_server_manager_create, 30 | weaver_server_manager_recreate, 31 | weaver_server_manager_snapshot, 32 | {{"config_get", weaver_server_manager_config_get}, 33 | {"config_ack", weaver_server_manager_config_ack}, 34 | {"config_stable", weaver_server_manager_config_stable}, 35 | {"server_register", weaver_server_manager_server_register}, 36 | {"server_online", weaver_server_manager_server_online}, 37 | {"server_offline", weaver_server_manager_server_offline}, 38 | {"server_shutdown", weaver_server_manager_server_shutdown}, 39 | {"server_kill", weaver_server_manager_server_kill}, 40 | {"server_forget", weaver_server_manager_server_forget}, 41 | {"server_suspect", weaver_server_manager_server_suspect}, 42 | {"report_disconnect", weaver_server_manager_server_suspect}, 43 | {"debug_dump", weaver_server_manager_debug_dump}, 44 | {"init", weaver_server_manager_init}, 45 | {NULL, NULL}} 46 | }; 47 | 48 | #pragma GCC diagnostic pop 49 | -------------------------------------------------------------------------------- /coordinator/transitions.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Replicant shim methods interface. 4 | * 5 | * Created: 2014-02-10 12:10:01 6 | * 7 | * Author: Robert Escriva, escriva@cs.cornell.edu 8 | * Ayush Dubey, dubey@cs.cornell.edu 9 | * 10 | * Copyright (C) 2013, Cornell University, see the LICENSE file 11 | * for licensing agreement 12 | * =============================================================== 13 | */ 14 | 15 | // Most of the following code has been 'borrowed' from 16 | // Robert Escriva's HyperDex coordinator. 17 | // see https://github.com/rescrv/HyperDex for the original code. 18 | 19 | #ifndef weaver_server_manager_transitions_h_ 20 | #define weaver_server_manager_transitions_h_ 21 | #ifdef __cplusplus 22 | extern "C" 23 | { 24 | #endif /* __cplusplus */ 25 | 26 | /* Replicant */ 27 | #include 28 | 29 | void* 30 | weaver_server_manager_create(struct rsm_context* ctx); 31 | 32 | void* 33 | weaver_server_manager_recreate(struct rsm_context* ctx, 34 | const char* data, size_t data_sz); 35 | 36 | int 37 | weaver_server_manager_snapshot(struct rsm_context* ctx, 38 | void* obj, char** data, size_t* sz); 39 | 40 | #define TRANSITION(X) void \ 41 | weaver_server_manager_ ## X(struct rsm_context* ctx, \ 42 | void* obj, const char* data, size_t data_sz) 43 | 44 | TRANSITION(init); 45 | 46 | TRANSITION(config_get); 47 | TRANSITION(config_ack); 48 | TRANSITION(config_stable); 49 | 50 | TRANSITION(server_register); 51 | TRANSITION(server_online); 52 | TRANSITION(server_offline); 53 | TRANSITION(server_shutdown); 54 | TRANSITION(server_kill); 55 | TRANSITION(server_forget); 56 | TRANSITION(server_suspect); 57 | TRANSITION(report_disconnect); 58 | 59 | TRANSITION(debug_dump); 60 | 61 | #ifdef __cplusplus 62 | } /* extern "C" */ 63 | #endif /* __cplusplus */ 64 | #endif /* weaver_server_manager_transitions_h_ */ 65 | -------------------------------------------------------------------------------- /coordinator/util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Server manager util methods. 4 | * 5 | * Created: 2014-02-09 17:06:10 6 | * 7 | * Author: Robert Escriva, escriva@cs.cornell.edu 8 | * Ayush Dubey, dubey@cs.cornell.edu 9 | * 10 | * Copyright (C) 2013, Cornell University, see the LICENSE file 11 | * for licensing agreement 12 | * =============================================================== 13 | */ 14 | 15 | // Most of the following code has been 'borrowed' from 16 | // Robert Escriva's HyperDex coordinator. 17 | // see https://github.com/rescrv/HyperDex for the original code. 18 | 19 | #include "common/server_manager_returncode.h" 20 | 21 | static inline void 22 | generate_response(rsm_context* ctx, server_manager_returncode x) 23 | { 24 | const char* ptr = nullptr; 25 | 26 | switch (x) 27 | { 28 | case COORD_SUCCESS: 29 | ptr = "\x22\x80"; 30 | break; 31 | case COORD_MALFORMED: 32 | ptr = "\x22\x81"; 33 | break; 34 | case COORD_DUPLICATE: 35 | ptr = "\x22\x82"; 36 | break; 37 | case COORD_NOT_FOUND: 38 | ptr = "\x22\x83"; 39 | break; 40 | case COORD_UNINITIALIZED: 41 | ptr = "\x22\x85"; 42 | break; 43 | case COORD_NO_CAN_DO: 44 | ptr = "\x22\x87"; 45 | break; 46 | default: 47 | ptr = "\xff\xff"; 48 | break; 49 | } 50 | 51 | rsm_set_output(ctx, ptr, 2); 52 | } 53 | 54 | #define INVARIANT_BROKEN(X) \ 55 | fprintf(log, "invariant broken at " __FILE__ ":%d: %s\n", __LINE__, X "\n") 56 | -------------------------------------------------------------------------------- /coordinator/vt_constants.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Constants for timestamper. 4 | * 5 | * Created: 09/21/2014 11:29:23 AM 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #ifndef weaver_coordinator_vt_constants_h_ 15 | #define weaver_coordinator_vt_constants_h_ 16 | 17 | #define NUM_VT_THREADS 8 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /db/.gitignore: -------------------------------------------------------------------------------- 1 | shard 2 | -------------------------------------------------------------------------------- /db/async_nodeprog_state.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Node prog state when an async get node call 4 | * delays execution of the node program 5 | * 6 | * Author: Ayush Dubey, dubey@cs.cornell.edu 7 | * 8 | * Copyright (C) 2015, Cornell University, see the LICENSE file 9 | * for licensing agreement 10 | * =============================================================== 11 | */ 12 | 13 | #ifndef weaver_db_async_nodeprog_state_h_ 14 | #define weaver_db_async_nodeprog_state_h_ 15 | 16 | #include "common/vclock.h" 17 | #include "db/node.h" 18 | 19 | namespace db 20 | { 21 | struct async_nodeprog_state 22 | { 23 | std::string type; 24 | vc::vclock clk; 25 | std::shared_ptr state; 26 | db::node *n; 27 | }; 28 | } 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /db/cache_entry.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Graph query cachine classes 4 | * 5 | * Created: Tuesday 18 November 2013 02:28:29 EDT 6 | * 7 | * Author: Gregory D. Hill, gdh39@cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #ifndef weaver_db_cache_entry_h_ 15 | #define weaver_db_cache_entry_h_ 16 | 17 | #include "db/remote_node.h" 18 | #include "node_prog/base_classes.h" 19 | 20 | namespace db 21 | { 22 | struct cache_entry 23 | { 24 | std::shared_ptr val; 25 | std::shared_ptr clk; 26 | std::shared_ptr> watch_set; 27 | 28 | cache_entry() { } 29 | 30 | cache_entry(std::shared_ptr v, 31 | std::shared_ptr c, 32 | std::shared_ptr> ws) 33 | : val(v) 34 | , clk(c) 35 | , watch_set(ws) 36 | { } 37 | }; 38 | } 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /db/deferred_write.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: DS for deferred write for a migrated node. 4 | * 5 | * Created: 09/16/2013 03:55:25 PM 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #ifndef weaver_db_deferred_write_h_ 15 | #define weaver_db_deferred_write_h_ 16 | 17 | namespace db 18 | { 19 | // state for a deferred write for a migrated node 20 | struct deferred_write 21 | { 22 | transaction::update_type type; 23 | vclock_ptr_t vclk; 24 | node_handle_t remote_node, alias; 25 | uint64_t remote_loc; 26 | edge_handle_t edge_handle; 27 | std::unique_ptr key, value; 28 | 29 | inline 30 | deferred_write(transaction::update_type t, vclock_ptr_t vc) : type(t), vclk(vc) { } 31 | }; 32 | 33 | typedef std::vector def_write_lst; 34 | } 35 | 36 | #endif 37 | -------------------------------------------------------------------------------- /db/del_obj.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: DS for storing a deleted object to be 4 | * permanently removed later on. 5 | * 6 | * Created: 2014-01-20 15:10:53 7 | * 8 | * Author: Ayush Dubey, dubey@cs.cornell.edu 9 | * 10 | * Copyright (C) 2013, Cornell University, see the LICENSE file 11 | * for licensing agreement 12 | * =============================================================== 13 | */ 14 | 15 | #ifndef weaver_db_del_obj_h_ 16 | #define weaver_db_del_obj_h_ 17 | 18 | namespace db 19 | { 20 | struct del_obj 21 | { 22 | transaction::update_type type; 23 | vclock_ptr_t tdel; 24 | vclock_ptr_t version; // creat time of the node 25 | node_handle_t node; 26 | edge_handle_t edge; 27 | std::vector no_outstanding_progs; 28 | 29 | inline 30 | del_obj(transaction::update_type t, const vclock_ptr_t &td, const vclock_ptr_t &ver, const node_handle_t &n, const edge_handle_t &e) 31 | : type(t) 32 | , tdel(td) 33 | , version(ver) 34 | , node(n) 35 | , edge(e) 36 | , no_outstanding_progs(NumVts, false) 37 | { } 38 | }; 39 | 40 | // for permanent deletion priority queue 41 | struct perm_del_compare 42 | : std::binary_function 43 | { 44 | bool operator()(const del_obj* const &dw1, const del_obj* const &dw2) 45 | { 46 | assert(dw1->tdel->clock.size() == ClkSz); 47 | assert(dw2->tdel->clock.size() == ClkSz); 48 | for (uint64_t i = 0; i < ClkSz; i++) { 49 | if (dw1->tdel->clock[i] <= dw2->tdel->clock[i]) { 50 | return false; 51 | } 52 | } 53 | return true; 54 | } 55 | }; 56 | } 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /db/edge.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Graph edge class 4 | * 5 | * Created: Tuesday 16 October 2012 02:28:29 EDT 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #ifndef weaver_db_edge_h_ 15 | #define weaver_db_edge_h_ 16 | 17 | #include 18 | #include 19 | 20 | #include "node_prog/edge.h" 21 | #include "node_prog/prop_list.h" 22 | #include "node_prog/property.h" 23 | #include "db/remote_node.h" 24 | #include "db/element.h" 25 | #include "client/datastructures.h" 26 | 27 | namespace db 28 | { 29 | class edge : public node_prog::edge 30 | { 31 | public: 32 | edge(); 33 | edge(const edge_handle_t &handle, vclock_ptr_t &vclk, uint64_t remote_loc, const node_handle_t &remote_handle); 34 | edge(const edge_handle_t &handle, vclock_ptr_t &vclk, remote_node &rn); 35 | ~edge(); 36 | 37 | public: 38 | element base; 39 | remote_node nbr; // out-neighbor for this edge 40 | #ifdef WEAVER_CLDG 41 | uint32_t msg_count; // number of messages sent on this link 42 | #endif 43 | #ifdef WEAVER_NEW_CLDG 44 | uint32_t msg_count; // number of messages sent on this link 45 | #endif 46 | uint64_t edge_id; // for new HD schema 47 | void traverse(); // indicate that this edge was traversed; useful for migration statistics 48 | 49 | const remote_node& get_neighbor() { return nbr; } 50 | node_prog::prop_list get_properties(); 51 | std::string get_property(const std::string &key); 52 | bool has_property(std::pair &p); 53 | bool has_all_properties(std::vector> &props); 54 | bool has_all_predicates(std::vector &preds); 55 | const edge_handle_t& get_handle() const { return base.get_handle(); } 56 | void get_client_edge(const std::string &node, cl::edge &e); 57 | 58 | static edge empty_edge; 59 | }; 60 | } 61 | 62 | #endif 63 | -------------------------------------------------------------------------------- /db/message_wrapper.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Wrapper around common::message, for a queued 4 | * request. 5 | * 6 | * Created: 2014-02-20 18:56:42 7 | * 8 | * Author: Ayush Dubey, dubey@cs.cornell.edu 9 | * 10 | * Copyright (C) 2013-2014, Cornell University, see the LICENSE 11 | * file for licensing agreement 12 | * =============================================================== 13 | */ 14 | 15 | #ifndef weaver_db_message_wrapper_h_ 16 | #define weaver_db_message_wrapper_h_ 17 | 18 | #include "common/message.h" 19 | #include "common/event_order.h" 20 | 21 | namespace db 22 | { 23 | class message_wrapper 24 | { 25 | public: 26 | message_wrapper(enum message::msg_type mt, std::unique_ptr m) 27 | : type(mt) 28 | , msg(std::move(m)) 29 | { } 30 | 31 | public: 32 | enum message::msg_type type; 33 | std::unique_ptr msg; 34 | order::oracle *time_oracle; 35 | }; 36 | } 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /db/node_entry.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: An entry in the node map (~page table entry) 4 | * 5 | * Author: Ayush Dubey, dubey@cs.cornell.edu 6 | * 7 | * Copyright (C) 2014, Cornell University, see the LICENSE file 8 | * for licensing agreement 9 | * =============================================================== 10 | */ 11 | 12 | #ifndef weaver_db_node_entry_h_ 13 | #define weaver_db_node_entry_h_ 14 | 15 | #include 16 | 17 | #include "db/node.h" 18 | 19 | namespace db 20 | { 21 | struct node_entry 22 | { 23 | bool present, used; 24 | std::vector nodes; 25 | std::shared_ptr prev, next; 26 | 27 | node_entry() : present(false), used(true) { } 28 | 29 | node_entry(node *n) 30 | : present(true) 31 | , used(true) 32 | , nodes(1, n) 33 | { } 34 | }; 35 | } 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /db/node_prog_running_state.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Node program running state 4 | * 5 | * Created: Sunday 17 March 2013 11:00:03 EDT 6 | * 7 | * Author: Ayush Dubey, Greg Hill 8 | * dubey@cs.cornell.edu, gdh39@cornell.edu 9 | * 10 | * Copyright (C) 2013, Cornell University, see the LICENSE file 11 | * for licensing agreement 12 | * ================================================================ 13 | */ 14 | 15 | #ifndef weaver_db_node_prog_running_state_ 16 | #define weaver_db_node_prog_running_state_ 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include "common/vclock.h" 23 | #include "node_prog/base_classes.h" 24 | 25 | namespace db 26 | { 27 | struct node_prog_running_state 28 | { 29 | node_prog_running_state() : m_handle(nullptr) { } 30 | 31 | std::string m_type; 32 | void *m_handle; 33 | uint64_t vt_id; 34 | std::shared_ptr req_vclock; 35 | uint64_t req_id; 36 | uint64_t vt_prog_ptr; 37 | std::deque> start_node_params; 38 | //std::unique_ptr> cache_value; 39 | std::unordered_map>> batched_node_progs; 40 | std::vector> nodes_that_created_state; 41 | }; 42 | } 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /db/property.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: db::property implementation. 4 | * 5 | * Created: 2014-05-30 17:24:08 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #include "db/property.h" 15 | 16 | using db::property; 17 | using db::property_key_hasher; 18 | 19 | property :: property() 20 | { } 21 | 22 | property :: property(const std::string &k, const std::string &v) 23 | : node_prog::property(k, v) 24 | { } 25 | 26 | property :: property(const std::string &k, const std::string &v, const vclock_ptr_t &creat) 27 | : node_prog::property(k, v) 28 | , creat_time(creat) 29 | { } 30 | 31 | property :: property(const property &other) 32 | : node_prog::property(other.key, other.value) 33 | , creat_time(other.creat_time) 34 | { 35 | if (other.del_time) { 36 | del_time = other.del_time; 37 | } 38 | } 39 | 40 | bool 41 | property :: operator==(property const &other) const 42 | { 43 | return (key == other.key) && (value == other.value); 44 | } 45 | 46 | const vclock_ptr_t& 47 | property :: get_creat_time() const 48 | { 49 | return creat_time; 50 | } 51 | 52 | const vclock_ptr_t& 53 | property :: get_del_time() const 54 | { 55 | return del_time; 56 | } 57 | 58 | bool 59 | property :: is_deleted() const 60 | { 61 | return (del_time != nullptr); 62 | } 63 | 64 | void 65 | property :: update_del_time(const vclock_ptr_t &tdel) 66 | { 67 | del_time = tdel; 68 | } 69 | 70 | void 71 | property :: update_creat_time(const vclock_ptr_t &tcreat) 72 | { 73 | creat_time = tcreat; 74 | } 75 | 76 | size_t 77 | property_key_hasher :: operator()(const property &p) const 78 | { 79 | return weaver_util::murmur_hasher()(p.key); 80 | } 81 | -------------------------------------------------------------------------------- /db/property.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Each graph element (node or edge) can have 4 | * properties, which are key-value pairs 5 | * 6 | * Created: Friday 12 October 2012 01:28:02 EDT 7 | * 8 | * Author: Ayush Dubey, dubey@cs.cornell.edu 9 | * 10 | * Copyright (C) 2013, Cornell University, see the LICENSE file 11 | * for licensing agreement 12 | * =============================================================== 13 | */ 14 | 15 | #ifndef weaver_db_property_h_ 16 | #define weaver_db_property_h_ 17 | 18 | #include 19 | #include 20 | 21 | #include "common/weaver_constants.h" 22 | #include "common/vclock.h" 23 | 24 | #include "node_prog/property.h" 25 | 26 | using vc::vclock_ptr_t; 27 | 28 | namespace db 29 | { 30 | class property : public node_prog::property 31 | { 32 | private: 33 | vclock_ptr_t creat_time; 34 | vclock_ptr_t del_time; 35 | 36 | public: 37 | property(); 38 | property(const std::string&, const std::string&); 39 | property(const std::string&, const std::string&, const vclock_ptr_t&); 40 | property(const property &other); 41 | 42 | bool operator==(property const &p2) const; 43 | 44 | const vclock_ptr_t& get_creat_time() const; 45 | const vclock_ptr_t& get_del_time() const; 46 | bool is_deleted() const; 47 | void update_creat_time(const vclock_ptr_t&); 48 | void update_del_time(const vclock_ptr_t&); 49 | }; 50 | 51 | class property_key_hasher 52 | { 53 | size_t operator() (const property &p) const; 54 | }; 55 | } 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /db/queued_request.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Data structure to hold a request (write tx or 4 | * node program) which cannot be executed on 5 | * receipt due to ordering constraints. 6 | * 7 | * Created: 2014-02-20 16:35:42 8 | * 9 | * Author: Ayush Dubey, dubey@cs.cornell.edu 10 | * 11 | * Copyright (C) 2013-2014, Cornell University, see the LICENSE 12 | * file for licensing agreement 13 | * =============================================================== 14 | */ 15 | 16 | #ifndef weaver_db_queued_request_h_ 17 | #define weaver_db_queued_request_h_ 18 | 19 | #include "common/vclock.h" 20 | #include "db/message_wrapper.h" 21 | 22 | namespace db 23 | { 24 | enum qreq_type 25 | { 26 | NOP, 27 | TX, 28 | NODE_PROG, 29 | OTHER 30 | }; 31 | 32 | class queued_request 33 | { 34 | public: 35 | queued_request(uint64_t prio, 36 | vc::vclock vclk, 37 | void (*f)(uint64_t, message_wrapper*), 38 | message_wrapper *a, 39 | qreq_type t) 40 | : priority(prio) 41 | , vclock(vclk) 42 | , func(f) 43 | , arg(a) 44 | , type(t) 45 | { } 46 | 47 | public: 48 | uint64_t priority; 49 | vc::vclock vclock; 50 | void (*func)(uint64_t, message_wrapper*); 51 | message_wrapper *arg; 52 | qreq_type type; 53 | }; 54 | 55 | // for work queues 56 | struct work_thread_compare 57 | : std::binary_function 58 | { 59 | bool operator()(const queued_request* const &r1, const queued_request* const &r2) 60 | { 61 | return (r1->priority) > (r2->priority); 62 | } 63 | }; 64 | 65 | } 66 | 67 | #endif 68 | -------------------------------------------------------------------------------- /db/remote_node.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Remote node pointer for edges. 4 | * 5 | * Created: 03/01/2013 11:29:16 AM 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #include "common/types.h" 15 | 16 | #ifndef weaver_db_remote_node_h_ 17 | #define weaver_db_remote_node_h_ 18 | 19 | namespace db 20 | { 21 | class remote_node 22 | { 23 | public: 24 | remote_node() { } 25 | remote_node(uint64_t l, const node_handle_t &h) : loc(l), handle(h) { } 26 | 27 | public: 28 | uint64_t loc; 29 | node_handle_t handle; 30 | bool operator==(const db::remote_node &t) const { return (handle == t.handle) && (loc == t.loc); } 31 | bool operator!=(const db::remote_node &t) const { return (handle != t.handle) || (loc != t.loc); } 32 | }; 33 | 34 | static db::remote_node coordinator(0, node_handle_t("")); 35 | } 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /db/shard_constants.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Constants for shards. 4 | * 5 | * Created: 2014-06-19 13:01:18 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #ifndef weaver_db_shard_constants_h_ 15 | #define weaver_db_shard_constants_h_ 16 | 17 | #define NUM_SHARD_THREADS (sysconf(_SC_NPROCESSORS_ONLN )) 18 | //#define NUM_SHARD_THREADS 1 19 | //#define NUM_SHARD_THREADS 128 20 | 21 | #define NUM_NODE_MAPS 64 22 | #define SHARD_MSGRECV_TIMEOUT 0 // busybee recv timeout (ms) for shard worker threads 23 | 24 | #define BATCH_MSG_SIZE 10 // 1 == no batching 25 | 26 | // migration 27 | //#define WEAVER_CLDG // defined if communication-based LDG, undef otherwise 28 | //#define WEAVER_NEW_CLDG // defined if communication-based LDG, undef otherwise 29 | 30 | // async node recovery 31 | //#define weaver_async_node_recovery_ 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /db/types.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Shard data types. 4 | * 5 | * Author: Ayush Dubey, dubey@cs.cornell.edu 6 | * 7 | * Copyright (C) 2014, Cornell University, see the LICENSE file 8 | * for licensing agreement 9 | * =============================================================== 10 | */ 11 | 12 | #ifndef weaver_db_types_h 13 | #define weaver_db_types_h 14 | 15 | #include 16 | #include 17 | 18 | #include "common/utils.h" 19 | #include "common/vclock.h" 20 | 21 | namespace db 22 | { 23 | template using data_map = google::sparse_hash_map, 26 | weaver_util::eqstr>; 27 | 28 | using string_set = google::sparse_hash_set, 30 | weaver_util::eqstr>; 31 | } 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /db/utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Commonly used db functionality 4 | * 5 | * Author: Ayush Dubey, dubey@cs.cornell.edu 6 | * 7 | * Copyright (C) 2015, Cornell University, see the LICENSE file 8 | * for licensing agreement 9 | * =============================================================== 10 | */ 11 | 12 | #ifndef weaver_db_utils_h 13 | #define weaver_db_utils_h 14 | 15 | #include "common/types.h" 16 | #include "db/shard_constants.h" 17 | 18 | inline uint64_t 19 | get_map_idx(const node_handle_t &nh) 20 | { 21 | return (weaver_util::murmur_hasher()(hash_node_handle(nh)) % NUM_NODE_MAPS); 22 | } 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /m4/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dubey/weaver/56a42fd2d0bbb14867ba792ca5461d16310a7387/m4/.gitignore -------------------------------------------------------------------------------- /node_prog/base_classes.h: -------------------------------------------------------------------------------- 1 | #ifndef weaver_node_prog_base_classes_h_ 2 | #define weaver_node_prog_base_classes_h_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "common/types.h" 10 | 11 | namespace db 12 | { 13 | class remote_node; 14 | } 15 | 16 | namespace node_prog 17 | { 18 | class node; 19 | 20 | enum search_type 21 | { 22 | BREADTH_FIRST, 23 | DEPTH_FIRST 24 | }; 25 | 26 | class Packable 27 | { 28 | public: 29 | virtual uint64_t size(void*) const = 0; 30 | virtual void pack(e::packer& packer, void*) const = 0; 31 | virtual void unpack(e::unpacker& unpacker, void*) = 0; 32 | }; 33 | 34 | class Deletable 35 | { 36 | public: 37 | virtual ~Deletable() = 0; 38 | }; 39 | 40 | inline Deletable::~Deletable() 41 | { 42 | /* destructor must be defined */ 43 | } 44 | 45 | class Node_Parameters_Base : public virtual Packable 46 | { 47 | virtual bool search_cache() = 0; 48 | virtual cache_key_t cache_key() = 0; 49 | }; 50 | 51 | class Node_State_Base : public virtual Packable, public virtual Deletable 52 | { 53 | public: 54 | virtual ~Node_State_Base() { } // << important 55 | std::unordered_set contexts_found; 56 | }; 57 | 58 | class Cache_Value_Base : public virtual Packable, public virtual Deletable 59 | { 60 | }; 61 | 62 | typedef std::shared_ptr np_param_ptr_t; 63 | typedef std::shared_ptr np_state_ptr_t; 64 | 65 | typedef np_param_ptr_t (*param_ctor_func_t)(); 66 | typedef uint64_t (*param_size_func_t)(const Node_Parameters_Base&, void*); 67 | typedef void (*param_pack_func_t)(const Node_Parameters_Base&, e::packer&, void*); 68 | typedef void (*param_unpack_func_t)(Node_Parameters_Base&, e::unpacker&, void*); 69 | 70 | typedef np_state_ptr_t (*state_ctor_func_t)(); 71 | typedef uint64_t (*state_size_func_t)(const Node_State_Base&, void*); 72 | typedef void (*state_pack_func_t)(const Node_State_Base&, e::packer&, void*); 73 | typedef void (*state_unpack_func_t)(Node_State_Base&, e::unpacker&, void*); 74 | 75 | typedef std::pair>> (*prog_ptr_t)(node_prog::node &n, 76 | db::remote_node &rn, 77 | np_param_ptr_t, 78 | std::function state_getter); 79 | } 80 | 81 | #endif 82 | -------------------------------------------------------------------------------- /node_prog/clustering_program.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Dijkstra shortest path program. 4 | * 5 | * Created: Sunday 21 April 2013 11:00:03 EDT 6 | * 7 | * Author: Ayush Dubey, Greg Hill 8 | * dubey@cs.cornell.edu, gdh39@cornell.edu 9 | * 10 | * Copyright (C) 2013, Cornell University, see the LICENSE file 11 | * for licensing agreement 12 | * ================================================================ 13 | */ 14 | 15 | #ifndef weaver_node_prog_clustering_program_h_ 16 | #define weaver_node_prog_clustering_program_h_ 17 | 18 | #include 19 | 20 | #include "db/remote_node.h" 21 | #include "node_prog/node.h" 22 | #include "node_prog/edge.h" 23 | #include "node_prog/cache_response.h" 24 | 25 | namespace node_prog 26 | { 27 | class clustering_params : public Node_Parameters_Base 28 | { 29 | public: 30 | bool _search_cache; 31 | cache_key_t _cache_key; 32 | bool is_center; 33 | db::remote_node center; 34 | bool outgoing; 35 | std::vector neighbors; 36 | double clustering_coeff; 37 | 38 | bool search_cache() { return _search_cache; } 39 | cache_key_t cache_key() { return _cache_key; } 40 | uint64_t size() const; 41 | void pack(e::packer& packer) const; 42 | void unpack(e::unpacker& unpacker); 43 | }; 44 | 45 | struct clustering_node_state : public Node_State_Base 46 | { 47 | // map from a node id to the number of neighbors who are connected to it 48 | std::unordered_map neighbor_counts; 49 | int responses_left; 50 | 51 | ~clustering_node_state() { } 52 | uint64_t size() const; 53 | void pack(e::packer& packer) const; 54 | void unpack(e::unpacker& unpacker); 55 | }; 56 | 57 | std::pair>> 58 | clustering_node_program( 59 | node &n, 60 | db::remote_node &rn, 61 | clustering_params ¶ms, 62 | std::function get_state, 63 | std::function, 64 | std::shared_ptr>, cache_key_t)>&, 65 | cache_response*); 66 | } 67 | 68 | #endif 69 | -------------------------------------------------------------------------------- /node_prog/dynamic_prog_table.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Implement dynamic prog table 4 | * 5 | * Author: Ayush Dubey, dubey@cs.cornell.edu 6 | * 7 | * Copyright (C) 2015, Cornell University, see the LICENSE file 8 | * for licensing agreement 9 | * =============================================================== 10 | */ 11 | 12 | #include 13 | 14 | #include "node_prog/dynamic_prog_table.h" 15 | 16 | dynamic_prog_table :: dynamic_prog_table(void *prog_handle) 17 | { 18 | param_ctor = (param_ctor_func_t)dlsym(prog_handle, "param_ctor"); 19 | param_size = (param_size_func_t)dlsym(prog_handle, "param_size"); 20 | param_pack = (param_pack_func_t)dlsym(prog_handle, "param_pack"); 21 | param_unpack = (param_unpack_func_t)dlsym(prog_handle, "param_unpack"); 22 | state_ctor = (state_ctor_func_t)dlsym(prog_handle, "state_ctor"); 23 | state_size = (state_size_func_t)dlsym(prog_handle, "state_size"); 24 | state_pack = (state_pack_func_t)dlsym(prog_handle, "state_pack"); 25 | state_unpack = (state_unpack_func_t)dlsym(prog_handle, "state_unpack"); 26 | node_program = (prog_ptr_t)dlsym(prog_handle, "node_program"); 27 | 28 | m_prog_handle = prog_handle; 29 | } 30 | 31 | dynamic_prog_table :: ~dynamic_prog_table() 32 | { 33 | dlclose(m_prog_handle); 34 | } 35 | -------------------------------------------------------------------------------- /node_prog/dynamic_prog_table.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Function pointers for dynamically linked 4 | * node program 5 | * 6 | * Author: Ayush Dubey, dubey@cs.cornell.edu 7 | * 8 | * Copyright (C) 2015, Cornell University, see the LICENSE file 9 | * for licensing agreement 10 | * =============================================================== 11 | */ 12 | 13 | #ifndef weaver_common_dynamic_prog_table_h_ 14 | #define weaver_common_dynamic_prog_table_h_ 15 | 16 | #include "node_prog/base_classes.h" 17 | 18 | using node_prog::np_param_ptr_t; 19 | using node_prog::np_state_ptr_t; 20 | using node_prog::param_ctor_func_t; 21 | using node_prog::param_size_func_t; 22 | using node_prog::param_pack_func_t; 23 | using node_prog::param_unpack_func_t; 24 | using node_prog::state_ctor_func_t; 25 | using node_prog::state_size_func_t; 26 | using node_prog::state_pack_func_t; 27 | using node_prog::state_unpack_func_t; 28 | using node_prog::prog_ptr_t; 29 | 30 | struct dynamic_prog_table 31 | { 32 | param_ctor_func_t param_ctor; 33 | param_size_func_t param_size; 34 | param_pack_func_t param_pack; 35 | param_unpack_func_t param_unpack; 36 | 37 | state_ctor_func_t state_ctor; 38 | state_size_func_t state_size; 39 | state_pack_func_t state_pack; 40 | state_unpack_func_t state_unpack; 41 | 42 | prog_ptr_t node_program; 43 | 44 | void *m_prog_handle; 45 | 46 | dynamic_prog_table(void *prog_handle); 47 | ~dynamic_prog_table(); 48 | }; 49 | 50 | #endif 51 | -------------------------------------------------------------------------------- /node_prog/edge.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Copyright (C) 2013, Cornell University, see the LICENSE file 4 | * for licensing agreement 5 | * =============================================================== 6 | */ 7 | 8 | #ifndef weaver_node_prog_edge_h_ 9 | #define weaver_node_prog_edge_h_ 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #include "common/property_predicate.h" 16 | #include "db/remote_node.h" 17 | #include "node_prog/property.h" 18 | #include "client/datastructures.h" 19 | 20 | namespace node_prog 21 | { 22 | class prop_iter; 23 | class prop_list; 24 | 25 | class edge 26 | { 27 | public: 28 | virtual ~edge() { } 29 | virtual const edge_handle_t& get_handle() const = 0; 30 | virtual void traverse() = 0; 31 | virtual const db::remote_node& get_neighbor() = 0; 32 | virtual prop_list get_properties() = 0; 33 | virtual std::string get_property(const std::string &key) = 0; 34 | virtual bool has_property(std::pair &p) = 0; 35 | virtual bool has_all_properties(std::vector> &props) = 0; 36 | virtual bool has_all_predicates(std::vector &preds) = 0; 37 | virtual void get_client_edge(const std::string &node, cl::edge&) = 0; 38 | }; 39 | } 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /node_prog/edge_count_program.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Edge count program implementation. 4 | * 5 | * Created: 2014-05-30 11:36:19 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #include "common/stl_serialization.h" 15 | #include "node_prog/node_prog_type.h" 16 | #include "node_prog/edge_count_program.h" 17 | 18 | using node_prog::search_type; 19 | using node_prog::edge_count_params; 20 | using node_prog::cache_response; 21 | 22 | // params 23 | uint64_t 24 | edge_count_params :: size() const 25 | { 26 | uint64_t toRet = message::size(edges_props) 27 | + message::size(edge_count); 28 | return toRet; 29 | } 30 | 31 | void edge_count_params :: pack(e::packer& packer) const 32 | { 33 | message::pack_buffer(packer, edges_props); 34 | message::pack_buffer(packer, edge_count); 35 | } 36 | 37 | void edge_count_params :: unpack(e::unpacker& unpacker) 38 | { 39 | message::unpack_buffer(unpacker, edges_props); 40 | message::unpack_buffer(unpacker, edge_count); 41 | } 42 | 43 | // node prog code 44 | std::pair>> 45 | node_prog :: edge_count_node_program( 46 | node &n, 47 | db::remote_node &, 48 | edge_count_params ¶ms, 49 | std::function, 50 | std::function, 51 | std::shared_ptr>, cache_key_t)>&, 52 | cache_response*) 53 | { 54 | auto elist = n.get_edges(); 55 | params.edge_count = 0; 56 | for (edge &e: elist) { 57 | if (e.has_all_properties(params.edges_props)) { 58 | params.edge_count++; 59 | } 60 | } 61 | 62 | return std::make_pair(search_type::DEPTH_FIRST, std::vector> 63 | (1, std::make_pair(db::coordinator, std::move(params)))); 64 | } 65 | -------------------------------------------------------------------------------- /node_prog/edge_count_program.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Count number of edges at a node. 4 | * 5 | * Created: 2014-04-16 17:41:08 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013-2014, Cornell University, see the LICENSE 10 | * file for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #ifndef weaver_node_prog_edge_count_program_h_ 15 | #define weaver_node_prog_edge_count_program_h_ 16 | 17 | #include 18 | #include 19 | 20 | #include "db/remote_node.h" 21 | #include "node_prog/base_classes.h" 22 | #include "node_prog/node.h" 23 | #include "node_prog/cache_response.h" 24 | 25 | namespace node_prog 26 | { 27 | class edge_count_params : public virtual Node_Parameters_Base 28 | { 29 | public: 30 | std::vector> edges_props; 31 | uint64_t edge_count; 32 | 33 | public: 34 | // would never need to cache 35 | bool search_cache() { return false; } 36 | cache_key_t cache_key() { return cache_key_t(); } 37 | uint64_t size() const; 38 | void pack(e::packer& packer) const; 39 | void unpack(e::unpacker& unpacker); 40 | }; 41 | 42 | struct edge_count_state : public virtual Node_State_Base 43 | { 44 | virtual ~edge_count_state() { } 45 | virtual uint64_t size() const { return 0; } 46 | virtual void pack(e::packer&) const { } 47 | virtual void unpack(e::unpacker&) { } 48 | }; 49 | 50 | std::pair>> 51 | edge_count_node_program( 52 | node &n, 53 | db::remote_node &, 54 | edge_count_params ¶ms, 55 | std::function, 56 | std::function, 57 | std::shared_ptr>, cache_key_t)>&, 58 | cache_response*); 59 | } 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /node_prog/edge_get_program.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Get all edges from the node. Can restrict 4 | * results based on neighbor vertex and edge 5 | * handles. 6 | * 7 | * Created: 2014-04-17 11:33:57 8 | * 9 | * Author: Ayush Dubey, dubey@cs.cornell.edu 10 | * 11 | * Copyright (C) 2013-2014, Cornell University, see the LICENSE 12 | * file for licensing agreement 13 | * =============================================================== 14 | */ 15 | 16 | #ifndef weaver_node_prog_edge_get_program_h_ 17 | #define weaver_node_prog_edge_get_program_h_ 18 | 19 | #include 20 | #include 21 | 22 | #include "db/remote_node.h" 23 | #include "node_prog/base_classes.h" 24 | #include "node_prog/node.h" 25 | #include "node_prog/cache_response.h" 26 | 27 | namespace node_prog 28 | { 29 | class edge_get_params : public Node_Parameters_Base 30 | { 31 | public: 32 | // request params 33 | // take intersection of all specified predicates 34 | std::vector nbrs; // empty means all nbrs 35 | std::vector request_edges; // empty means fetch props for all edges 36 | std::vector> properties; // edges should have these properties 37 | 38 | // response params 39 | std::vector response_edges; 40 | 41 | // would never need to cache 42 | bool search_cache() { return false; } 43 | cache_key_t cache_key() { return cache_key_t(); } 44 | uint64_t size() const; 45 | void pack(e::packer& packer) const; 46 | void unpack(e::unpacker& unpacker); 47 | }; 48 | 49 | struct edge_get_state : public Node_State_Base 50 | { 51 | ~edge_get_state() { } 52 | uint64_t size() const { return 0; } 53 | void pack(e::packer&) const { } 54 | void unpack(e::unpacker&) { } 55 | }; 56 | 57 | std::pair>> 58 | edge_get_node_program( 59 | node &n, 60 | db::remote_node &, 61 | edge_get_params ¶ms, 62 | std::function, 63 | std::function, 64 | std::shared_ptr>, cache_key_t)>&, 65 | cache_response*); 66 | } 67 | 68 | #endif 69 | -------------------------------------------------------------------------------- /node_prog/edge_list.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Node prog edge list declaration. 4 | * 5 | * Created: 2014-05-29 18:48:08 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #ifndef weaver_node_prog_edge_list_h_ 15 | #define weaver_node_prog_edge_list_h_ 16 | 17 | #include 18 | #include 19 | 20 | #include "db/edge.h" 21 | #include "db/types.h" 22 | #include "common/event_order.h" 23 | #include "node_prog/edge.h" 24 | 25 | namespace node_prog 26 | { 27 | using edge_map_t = db::data_map>; 28 | class edge_map_iter : public std::iterator 29 | { 30 | db::edge *cur_edge; 31 | edge_map_t::iterator internal_cur; 32 | edge_map_t::iterator internal_end; 33 | std::shared_ptr req_time; 34 | order::oracle *time_oracle; 35 | 36 | public: 37 | edge_map_iter& operator++(); 38 | edge_map_iter(edge_map_t::iterator begin, 39 | edge_map_t::iterator end, 40 | std::shared_ptr &req_time, 41 | order::oracle *time_oracle); 42 | bool operator==(const edge_map_iter& rhs); 43 | bool operator!=(const edge_map_iter& rhs); 44 | edge& operator*(); 45 | }; 46 | 47 | class edge_list 48 | { 49 | private: 50 | edge_map_t &wrapped; 51 | std::shared_ptr &req_time; 52 | order::oracle *time_oracle; 53 | 54 | public: 55 | edge_list(edge_map_t &edge_list, 56 | std::shared_ptr &req_time, 57 | order::oracle *time_oracle); 58 | edge_map_iter begin(); 59 | edge_map_iter end(); 60 | uint64_t count(); 61 | }; 62 | } 63 | 64 | #endif 65 | -------------------------------------------------------------------------------- /node_prog/get_btc_addr.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Node program to get a BTC address and all 4 | * of its transactions 5 | * 6 | * Author: Ayush Dubey, dubey@cs.cornell.edu 7 | * 8 | * Copyright (C) 2014, Cornell University, see the LICENSE file 9 | * for licensing agreement 10 | * =============================================================== 11 | */ 12 | 13 | #ifndef weaver_node_prog_get_btc_addr_h_ 14 | #define weaver_node_prog_get_btc_addr_h_ 15 | 16 | #include 17 | 18 | #include "common/property_predicate.h" 19 | #include "db/remote_node.h" 20 | #include "node_prog/node.h" 21 | #include "node_prog/edge.h" 22 | #include "node_prog/base_classes.h" 23 | #include "node_prog/cache_response.h" 24 | 25 | namespace node_prog 26 | { 27 | struct get_btc_addr_params: public virtual Node_Parameters_Base 28 | { 29 | node_handle_t addr_handle; 30 | cl::node addr_node; 31 | std::vector txs; 32 | 33 | bool returning; 34 | db::remote_node addr_rn; 35 | 36 | get_btc_addr_params(); 37 | ~get_btc_addr_params() { } 38 | uint64_t size() const; 39 | void pack(e::packer&) const; 40 | void unpack(e::unpacker&); 41 | 42 | // no caching 43 | bool search_cache() { return false; } 44 | cache_key_t cache_key() { return cache_key_t(); } 45 | }; 46 | 47 | struct get_btc_addr_state: public virtual Node_State_Base 48 | { 49 | uint32_t outstanding_count; 50 | std::vector txs; 51 | 52 | get_btc_addr_state(); 53 | ~get_btc_addr_state() { } 54 | uint64_t size() const; 55 | void pack(e::packer&) const; 56 | void unpack(e::unpacker&); 57 | }; 58 | 59 | std::pair>> 60 | get_btc_addr_node_program(node &n, 61 | db::remote_node &rn, 62 | get_btc_addr_params ¶ms, 63 | std::function state_getter, 64 | std::function, std::shared_ptr>, cache_key_t)>&, 65 | cache_response*); 66 | } 67 | 68 | #endif 69 | -------------------------------------------------------------------------------- /node_prog/get_btc_block.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Node program to get a BTC block and all 4 | * of its transactions 5 | * 6 | * Author: Ayush Dubey, dubey@cs.cornell.edu 7 | * 8 | * Copyright (C) 2014, Cornell University, see the LICENSE file 9 | * for licensing agreement 10 | * =============================================================== 11 | */ 12 | 13 | #ifndef weaver_node_prog_get_btc_block_h_ 14 | #define weaver_node_prog_get_btc_block_h_ 15 | 16 | #include 17 | 18 | #include "common/property_predicate.h" 19 | #include "db/remote_node.h" 20 | #include "node_prog/node.h" 21 | #include "node_prog/edge.h" 22 | #include "node_prog/base_classes.h" 23 | #include "node_prog/cache_response.h" 24 | 25 | namespace node_prog 26 | { 27 | using btc_tx_t = std::pair, std::vector>; 28 | struct get_btc_block_params: public virtual Node_Parameters_Base 29 | { 30 | node_handle_t block; 31 | cl::node block_node; 32 | std::vector txs; 33 | uint32_t num_nodes_read; 34 | 35 | bool returning; 36 | db::remote_node block_rn; 37 | 38 | get_btc_block_params(); 39 | ~get_btc_block_params() { } 40 | uint64_t size() const; 41 | void pack(e::packer&) const; 42 | void unpack(e::unpacker&); 43 | 44 | // no caching 45 | bool search_cache() { return false; } 46 | cache_key_t cache_key() { return cache_key_t(); } 47 | }; 48 | 49 | struct get_btc_block_state: public virtual Node_State_Base 50 | { 51 | uint32_t outstanding_count; 52 | std::unordered_map outstanding; 53 | std::vector txs; 54 | uint32_t num_nodes_read; 55 | 56 | get_btc_block_state(); 57 | ~get_btc_block_state() { } 58 | uint64_t size() const; 59 | void pack(e::packer&) const; 60 | void unpack(e::unpacker&); 61 | }; 62 | 63 | std::pair>> 64 | get_btc_block_node_program(node &n, 65 | db::remote_node &rn, 66 | get_btc_block_params ¶ms, 67 | std::function state_getter, 68 | std::function, std::shared_ptr>, cache_key_t)>&, 69 | cache_response*); 70 | } 71 | 72 | #endif 73 | -------------------------------------------------------------------------------- /node_prog/get_btc_tx.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Get BTC tx implementation. 4 | * 5 | * Author: Ayush Dubey, dubey@cs.cornell.edu 6 | * 7 | * Copyright (C) 2015, Cornell University, see the LICENSE file 8 | * for licensing agreement 9 | * =============================================================== 10 | */ 11 | 12 | #define weaver_debug_ 13 | #include "common/stl_serialization.h" 14 | #include "node_prog/node_prog_type.h" 15 | #include "node_prog/get_btc_tx.h" 16 | #include "node_prog/parse_btc_tx.h" 17 | 18 | using node_prog::search_type; 19 | using node_prog::get_btc_tx_params; 20 | using node_prog::get_btc_tx_state; 21 | using node_prog::cache_response; 22 | using node_prog::Cache_Value_Base; 23 | 24 | // params 25 | get_btc_tx_params :: get_btc_tx_params() 26 | { } 27 | 28 | uint64_t 29 | get_btc_tx_params :: size() const 30 | { 31 | return message::size(ret_node); 32 | } 33 | 34 | void 35 | get_btc_tx_params :: pack(e::packer &packer) const 36 | { 37 | message::pack_buffer(packer, ret_node); 38 | } 39 | 40 | void 41 | get_btc_tx_params :: unpack(e::unpacker &unpacker) 42 | { 43 | message::unpack_buffer(unpacker, ret_node); 44 | } 45 | 46 | get_btc_tx_state :: get_btc_tx_state() 47 | { } 48 | 49 | uint64_t 50 | get_btc_tx_state :: size() const 51 | { 52 | return 0; 53 | } 54 | 55 | void 56 | get_btc_tx_state :: pack(e::packer &) const 57 | { } 58 | 59 | void 60 | get_btc_tx_state :: unpack(e::unpacker &) 61 | { } 62 | 63 | std::pair>> 64 | node_prog :: get_btc_tx_node_program(node_prog::node &n, 65 | db::remote_node &, 66 | get_btc_tx_params ¶ms, 67 | std::function, 68 | std::function, std::shared_ptr>, cache_key_t)>&, 69 | cache_response*) 70 | { 71 | std::vector> next; 72 | parse_btc_tx(params.ret_node, n); 73 | next.emplace_back(std::make_pair(db::coordinator, std::move(params))); 74 | 75 | return std::make_pair(search_type::BREADTH_FIRST, std::move(next)); 76 | } 77 | -------------------------------------------------------------------------------- /node_prog/get_btc_tx.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Node program to get a BTC tx 4 | * 5 | * Author: Ayush Dubey, dubey@cs.cornell.edu 6 | * 7 | * Copyright (C) 2015, Cornell University, see the LICENSE file 8 | * for licensing agreement 9 | * =============================================================== 10 | */ 11 | 12 | #ifndef weaver_node_prog_get_btc_tx_h_ 13 | #define weaver_node_prog_get_btc_tx_h_ 14 | 15 | #include 16 | 17 | #include "common/property_predicate.h" 18 | #include "db/remote_node.h" 19 | #include "node_prog/node.h" 20 | #include "node_prog/edge.h" 21 | #include "node_prog/base_classes.h" 22 | #include "node_prog/cache_response.h" 23 | 24 | namespace node_prog 25 | { 26 | using btc_tx_t = std::pair, std::vector>; 27 | struct get_btc_tx_params: public virtual Node_Parameters_Base 28 | { 29 | cl::node ret_node; 30 | 31 | get_btc_tx_params(); 32 | ~get_btc_tx_params() { } 33 | uint64_t size() const; 34 | void pack(e::packer&) const; 35 | void unpack(e::unpacker&); 36 | 37 | // no caching 38 | bool search_cache() { return false; } 39 | cache_key_t cache_key() { return cache_key_t(); } 40 | }; 41 | 42 | struct get_btc_tx_state: public virtual Node_State_Base 43 | { 44 | get_btc_tx_state(); 45 | ~get_btc_tx_state() { } 46 | uint64_t size() const; 47 | void pack(e::packer&) const; 48 | void unpack(e::unpacker&); 49 | }; 50 | 51 | std::pair>> 52 | get_btc_tx_node_program(node &n, 53 | db::remote_node &rn, 54 | get_btc_tx_params ¶ms, 55 | std::function state_getter, 56 | std::function, std::shared_ptr>, cache_key_t)>&, 57 | cache_response*); 58 | } 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /node_prog/neural_net_infer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Node program that implements deep learning 4 | * inference using a feed forward neural network 5 | * 6 | * Author: Ayush Dubey, dubey@cs.cornell.edu 7 | * Siddhant Manocha, siddhantmanocha1994@gmail 8 | * 9 | * Copyright (C) 2015, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #ifndef weaver_node_prog_neural_net_infer_h_ 15 | #define weaver_node_prog_neural_net_infer_h_ 16 | 17 | #include "node_prog/boilerplate.h" 18 | 19 | namespace node_prog 20 | { 21 | struct nn_params: public virtual Node_Parameters_Base 22 | { 23 | // handles of first and last node 24 | std::pair network_description; 25 | // input vector or value 26 | std::vector input; 27 | // for first node, order defines which index of input to be propagated to next layer 28 | // for last node, order defines which index of the value to store input 29 | int order; 30 | // activation function 31 | std::string act_func; 32 | std::string layer_type; 33 | std::string layer_op; 34 | 35 | nn_params(); 36 | ~nn_params() { } 37 | uint64_t size(void*) const; 38 | void pack(e::packer &packer, void*) const; 39 | void unpack(e::unpacker &unpacker, void*); 40 | 41 | // no caching 42 | bool search_cache() { return false; } 43 | cache_key_t cache_key() { return cache_key_t(); } 44 | }; 45 | 46 | struct nn_state: public virtual Node_State_Base 47 | { 48 | bool visited; 49 | uint32_t in_count; 50 | std::vector value; 51 | 52 | nn_state(); 53 | ~nn_state() { } 54 | uint64_t size(void*) const; 55 | void pack(e::packer &packer, void*) const; 56 | void unpack(e::unpacker &unpacker, void*); 57 | }; 58 | 59 | extern "C" { 60 | PROG_FUNC_DECLARE; 61 | } 62 | } 63 | 64 | #endif 65 | -------------------------------------------------------------------------------- /node_prog/node.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Copyright (C) 2013, Cornell University, see the LICENSE file 4 | * for licensing agreement 5 | * =============================================================== 6 | */ 7 | 8 | #ifndef weaver_node_prog_node_h_ 9 | #define weaver_node_prog_node_h_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "common/types.h" 18 | #include "common/property_predicate.h" 19 | #include "node_prog/edge_list.h" 20 | #include "client/datastructures.h" 21 | 22 | namespace node_prog 23 | { 24 | class node 25 | { 26 | public: 27 | virtual ~node() { } 28 | virtual const node_handle_t& get_handle() const = 0; 29 | virtual bool edge_exists(const edge_handle_t&) = 0; 30 | virtual edge& get_edge(const edge_handle_t&) = 0; 31 | virtual edge_list get_edges() = 0; 32 | virtual prop_list get_properties() = 0; 33 | virtual std::string get_property(const std::string &key) = 0; 34 | virtual bool has_property(std::pair &p) = 0; 35 | virtual bool has_all_properties(std::vector> &props) = 0; 36 | virtual bool has_all_predicates(std::vector &preds) = 0; 37 | virtual bool is_alias(const node_handle_t &alias) const = 0; 38 | virtual void get_client_node(cl::node&, bool props, bool edges, bool aliases) = 0; 39 | }; 40 | } 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /node_prog/node_get_program.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Implement get node program. 4 | * 5 | * Author: Ayush Dubey, dubey@cs.cornell.edu 6 | * 7 | * Copyright (C) 2014, Cornell University, see the LICENSE file 8 | * for licensing agreement 9 | * =============================================================== 10 | */ 11 | 12 | #include "common/stl_serialization.h" 13 | #include "node_prog/node_prog_type.h" 14 | #include "node_prog/node_get_program.h" 15 | 16 | using node_prog::search_type; 17 | using node_prog::node_get_params; 18 | using node_prog::node_get_state; 19 | using node_prog::cache_response; 20 | 21 | uint64_t 22 | node_get_params :: size() const 23 | { 24 | return message::size(props) 25 | + message::size(edges) 26 | + message::size(aliases) 27 | + message::size(node); 28 | } 29 | 30 | void 31 | node_get_params :: pack(e::packer& packer) const 32 | { 33 | message::pack_buffer(packer, props); 34 | message::pack_buffer(packer, edges); 35 | message::pack_buffer(packer, aliases); 36 | message::pack_buffer(packer, node); 37 | } 38 | 39 | void 40 | node_get_params :: unpack(e::unpacker& unpacker) 41 | { 42 | message::unpack_buffer(unpacker, props); 43 | message::unpack_buffer(unpacker, edges); 44 | message::unpack_buffer(unpacker, aliases); 45 | message::unpack_buffer(unpacker, node); 46 | } 47 | 48 | std::pair>> 49 | node_prog :: node_get_node_program( 50 | node &n, 51 | db::remote_node &, 52 | node_get_params ¶ms, 53 | std::function, 54 | std::function, 55 | std::shared_ptr>, cache_key_t)>&, 56 | cache_response*) 57 | { 58 | n.get_client_node(params.node, params.props, params.edges, params.aliases); 59 | 60 | return std::make_pair(search_type::DEPTH_FIRST, std::vector> 61 | (1, std::make_pair(db::coordinator, std::move(params)))); 62 | } 63 | -------------------------------------------------------------------------------- /node_prog/node_get_program.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Get node with all client-facing data. 4 | * 5 | * Author: Ayush Dubey, dubey@cs.cornell.edu 6 | * 7 | * Copyright (C) 2014, Cornell University, see the LICENSE file 8 | * for licensing agreement 9 | * =============================================================== 10 | */ 11 | 12 | #ifndef weaver_node_prog_node_get_program_h_ 13 | #define weaver_node_prog_node_get_program_h_ 14 | 15 | #include 16 | #include 17 | 18 | #include "db/remote_node.h" 19 | #include "node_prog/base_classes.h" 20 | #include "node_prog/node.h" 21 | #include "node_prog/cache_response.h" 22 | 23 | namespace node_prog 24 | { 25 | class node_get_params: public Node_Parameters_Base 26 | { 27 | public: 28 | bool props, edges, aliases; 29 | cl::node node; 30 | 31 | // no caching needed 32 | bool search_cache() { return false; } 33 | cache_key_t cache_key() { return cache_key_t(); } 34 | uint64_t size() const; 35 | void pack(e::packer& packer) const; 36 | void unpack(e::unpacker& unpacker); 37 | }; 38 | 39 | struct node_get_state: public Node_State_Base 40 | { 41 | ~node_get_state() { } 42 | uint64_t size() const { return 0; } 43 | void pack(e::packer&) const { } 44 | void unpack(e::unpacker&) { } 45 | }; 46 | 47 | std::pair>> 48 | node_get_node_program( 49 | node &n, 50 | db::remote_node &, 51 | node_get_params ¶ms, 52 | std::function, 53 | std::function, 54 | std::shared_ptr>, cache_key_t)>&, 55 | cache_response*); 56 | } 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /node_prog/node_prog_type.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Base classes, typedefs, and necessary enum 4 | * declarations for all node programs. 5 | * 6 | * Created: Sunday 17 March 2013 11:00:03 EDT 7 | * 8 | * Author: Ayush Dubey, Greg Hill, dubey@cs.cornell.edu, gdh39@cornell.edu 9 | * 10 | * Copyright (C) 2013, Cornell University, see the LICENSE file 11 | * for licensing agreement 12 | * ================================================================ 13 | */ 14 | 15 | #ifndef weaver_node_prog_node_prog_type_h_ 16 | #define weaver_node_prog_node_prog_type_h_ 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | namespace node_prog 23 | { 24 | enum prog_type 25 | { 26 | TRAVERSE_PROPS, 27 | END 28 | }; 29 | //enum prog_type 30 | //{ 31 | // REACHABILITY, 32 | // PATHLESS_REACHABILITY, 33 | // N_HOP_REACHABILITY, 34 | // TRIANGLE_COUNT, 35 | // DIJKSTRA, 36 | // CLUSTERING, 37 | // TWO_NEIGHBORHOOD, 38 | // READ_NODE_PROPS, 39 | // READ_EDGES_PROPS, 40 | // READ_N_EDGES, 41 | // EDGE_COUNT, 42 | // EDGE_GET, 43 | // NODE_GET, 44 | // TRAVERSE_PROPS, 45 | // DISCOVER_PATHS, 46 | // CAUSE_AND_EFFECT, 47 | // N_GRAM_PATH, 48 | // GET_BTC_BLOCK, 49 | // GET_BTC_TX, 50 | // GET_BTC_ADDR, 51 | // END 52 | //}; 53 | 54 | } 55 | 56 | namespace std 57 | { 58 | // used if we want a hash table with a prog type as the key 59 | template <> 60 | struct hash 61 | { 62 | public: 63 | size_t operator()(node_prog::prog_type x) const throw() 64 | { 65 | return hash()(x); 66 | } 67 | }; 68 | } 69 | 70 | #endif 71 | -------------------------------------------------------------------------------- /node_prog/parse_btc_tx.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Implement parse_btc_tx 4 | * Author: Ayush Dubey, dubey@cs.cornell.edu 5 | * 6 | * Copyright (C) 2015, Cornell University, see the LICENSE file 7 | * for licensing agreement 8 | * =============================================================== 9 | */ 10 | 11 | #include "node_prog/parse_btc_tx.h" 12 | 13 | void 14 | node_prog::parse_btc_tx(cl::node &tx, node_prog::node &n) 15 | { 16 | std::string txout_str = "TXOUT_"; 17 | 18 | n.get_client_node(tx, true, true, true); 19 | for (auto &p: tx.out_edges) { 20 | if (p.first.compare(0, txout_str.size(), txout_str) == 0) { 21 | std::string consm_tx = "CTX_" + p.first.substr(txout_str.size()); 22 | if (n.edge_exists(consm_tx)) { 23 | node_prog::edge &ctx_edge = n.get_edge(consm_tx); 24 | cl::edge &e = p.second; 25 | 26 | std::shared_ptr consumed_prop(new cl::property()); 27 | consumed_prop->key = "consumed"; 28 | consumed_prop->value = ctx_edge.get_neighbor().handle; 29 | 30 | e.properties.emplace_back(consumed_prop); 31 | } 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /node_prog/parse_btc_tx.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Parse a node as a btc tx and return as client 4 | * node 5 | * 6 | * Author: Ayush Dubey, dubey@cs.cornell.edu 7 | * 8 | * Copyright (C) 2015, Cornell University, see the LICENSE file 9 | * for licensing agreement 10 | * =============================================================== 11 | */ 12 | 13 | #ifndef weaver_node_prog_parse_btc_tx_h_ 14 | #define weaver_node_prog_parse_btc_tx_h_ 15 | 16 | #include "node_prog/node.h" 17 | 18 | namespace node_prog 19 | { 20 | void parse_btc_tx(cl::node &tx, node_prog::node &n); 21 | } 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /node_prog/pathless_reach_program.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Reachability program. 4 | * 5 | * Created: Sunday 23 April 2013 11:00:03 EDT 6 | * 7 | * Author: Ayush Dubey, Greg Hill 8 | * dubey@cs.cornell.edu, gdh39@cornell.edu 9 | * 10 | * Copyright (C) 2013, Cornell University, see the LICENSE file 11 | * for licensing agreement 12 | * ================================================================ 13 | */ 14 | 15 | #ifndef weaver_node_prog_pathless_reach_program_h_ 16 | #define weaver_node_prog_pathless_reach_program_h_ 17 | 18 | #include 19 | #include 20 | 21 | #include "db/remote_node.h" 22 | #include "node_prog/base_classes.h" 23 | #include "node_prog/node.h" 24 | #include "node_prog/cache_response.h" 25 | 26 | namespace node_prog 27 | { 28 | class pathless_reach_params : public virtual Node_Parameters_Base 29 | { 30 | public: 31 | bool returning; // false = request, true = reply 32 | db::remote_node prev_node; 33 | node_handle_t dest; 34 | std::vector> edge_props; 35 | bool reachable; 36 | 37 | public: 38 | pathless_reach_params(); 39 | ~pathless_reach_params() { } 40 | bool search_cache(); 41 | cache_key_t cache_key() { return cache_key_t(); } 42 | uint64_t size() const; 43 | void pack(e::packer &packer) const; 44 | void unpack(e::unpacker &unpacker); 45 | }; 46 | 47 | struct pathless_reach_node_state : public virtual Node_State_Base 48 | { 49 | bool visited; 50 | db::remote_node prev_node; // previous node 51 | uint32_t out_count; // number of requests propagated 52 | bool reachable; 53 | 54 | pathless_reach_node_state(); 55 | ~pathless_reach_node_state() { } 56 | uint64_t size() const; 57 | void pack(e::packer& packer) const; 58 | void unpack(e::unpacker& unpacker); 59 | }; 60 | 61 | std::pair>> 62 | pathless_reach_node_program( 63 | node &n, 64 | db::remote_node &rn, 65 | pathless_reach_params ¶ms, 66 | std::function state_getter, 67 | std::function, 68 | std::shared_ptr>, cache_key_t)>& add_cache_func, 69 | cache_response*cache_response); 70 | } 71 | 72 | #endif 73 | -------------------------------------------------------------------------------- /node_prog/prop_list.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Node prog prop list implementation. 4 | * 5 | * Created: 2014-05-29 19:08:19 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #include "common/weaver_constants.h" 15 | #include "node_prog/property.h" 16 | #include "node_prog/prop_list.h" 17 | 18 | using node_prog::prop_iter; 19 | 20 | bool 21 | prop_iter :: check_props_iter(props_ds_t::iterator &iter) 22 | { 23 | #ifdef weaver_large_property_maps_ 24 | for (const std::shared_ptr p: iter->second) { 25 | if (time_oracle->clock_creat_before_del_after(req_time, p->get_creat_time(), p->get_del_time())) { 26 | return true; 27 | } 28 | } 29 | #else 30 | const std::shared_ptr p = *iter; 31 | if (time_oracle->clock_creat_before_del_after(req_time, p->get_creat_time(), p->get_del_time())) { 32 | return true; 33 | } 34 | #endif 35 | 36 | return false; 37 | } 38 | 39 | prop_iter& 40 | prop_iter :: operator++() 41 | { 42 | while (++internal_cur != internal_end) { 43 | if (check_props_iter(internal_cur)) { 44 | break; 45 | } 46 | } 47 | 48 | return *this; 49 | } 50 | 51 | prop_iter :: prop_iter(props_ds_t::iterator begin, 52 | props_ds_t::iterator end, 53 | vc::vclock &req_time, 54 | order::oracle *to) 55 | : internal_cur(begin) 56 | , internal_end(end) 57 | , req_time(req_time) 58 | , time_oracle(to) 59 | { 60 | if (internal_cur != internal_end) { 61 | if (!check_props_iter(internal_cur)) { 62 | ++(*this); 63 | } 64 | } 65 | } 66 | 67 | bool 68 | prop_iter :: operator!=(const prop_iter& rhs) const 69 | { 70 | return internal_cur != rhs.internal_cur || !(req_time == rhs.req_time); 71 | } 72 | 73 | std::vector> 74 | prop_iter :: operator*() 75 | { 76 | std::vector> ret; 77 | 78 | #ifdef weaver_large_property_maps_ 79 | for (const std::shared_ptr p: internal_cur->second) { 80 | if (time_oracle->clock_creat_before_del_after(req_time, p->get_creat_time(), p->get_del_time())) { 81 | ret.emplace_back(p); 82 | } 83 | } 84 | #else 85 | ret.emplace_back(*internal_cur); 86 | #endif 87 | 88 | return ret; 89 | } 90 | -------------------------------------------------------------------------------- /node_prog/prop_list.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Node prog prop list implementation. 4 | * 5 | * Created: 2014-05-29 19:08:19 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * Greg Hill, gdh39@cornell.edu 9 | * 10 | * Copyright (C) 2013, Cornell University, see the LICENSE file 11 | * for licensing agreement 12 | * =============================================================== 13 | */ 14 | 15 | #ifndef weaver_node_prog_prop_list_h_ 16 | #define weaver_node_prog_prop_list_h_ 17 | 18 | #include 19 | #include 20 | 21 | #include "common/weaver_constants.h" 22 | #include "common/event_order.h" 23 | #include "db/property.h" 24 | 25 | namespace node_prog 26 | { 27 | #ifdef weaver_large_property_maps_ 28 | typedef std::unordered_map>> props_ds_t; 29 | #else 30 | typedef std::vector> props_ds_t; 31 | #endif 32 | class prop_iter : public std::iterator 33 | { 34 | private: 35 | props_ds_t::iterator internal_cur; 36 | props_ds_t::iterator internal_end; 37 | vc::vclock &req_time; 38 | order::oracle *time_oracle; 39 | bool check_props_iter(props_ds_t::iterator&); 40 | 41 | public: 42 | prop_iter& operator++(); 43 | prop_iter(props_ds_t::iterator begin, props_ds_t::iterator end, vc::vclock& req_time, order::oracle *time_oracle); 44 | bool operator!=(const prop_iter& rhs) const; 45 | std::vector> operator*(); 46 | }; 47 | 48 | class prop_list 49 | { 50 | private: 51 | props_ds_t &wrapped; 52 | vc::vclock &req_time; 53 | order::oracle *time_oracle; 54 | 55 | public: 56 | prop_list(props_ds_t &prop_list, vc::vclock &req_time, order::oracle *to) 57 | : wrapped(prop_list), req_time(req_time), time_oracle(to) { } 58 | 59 | prop_iter begin() 60 | { 61 | return prop_iter(wrapped.begin(), wrapped.end(), req_time, time_oracle); 62 | } 63 | 64 | prop_iter end() 65 | { 66 | return prop_iter(wrapped.end(), wrapped.end(), req_time, time_oracle); 67 | } 68 | }; 69 | } 70 | 71 | #endif 72 | -------------------------------------------------------------------------------- /node_prog/property.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Each graph element (node or edge) can have 4 | * properties, which are key-value pairs 5 | * 6 | * Created: Friday 12 October 2012 01:28:02 EDT 7 | * 8 | * Author: Ayush Dubey, dubey@cs.cornell.edu 9 | * 10 | * Copyright (C) 2013, Cornell University, see the LICENSE file 11 | * for licensing agreement 12 | * =============================================================== 13 | */ 14 | 15 | #ifndef weaver_node_prog_property_h_ 16 | #define weaver_node_prog_property_h_ 17 | 18 | #include 19 | 20 | namespace node_prog 21 | { 22 | class property 23 | { 24 | public: 25 | std::string key; 26 | std::string value; 27 | 28 | property() : key(""), value("") { } 29 | property(const std::string &k, const std::string &v) : key(k), value(v) { } 30 | 31 | const std::string& get_key() const { return key; } 32 | const std::string& get_value() const { return value; } 33 | }; 34 | } 35 | 36 | #endif 37 | -------------------------------------------------------------------------------- /node_prog/read_n_edges_program.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Implementation of read-n-edges program. 4 | * 5 | * Created: 2014-05-30 12:21:42 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #include "common/stl_serialization.h" 15 | #include "node_prog/node_prog_type.h" 16 | #include "node_prog/edge.h" 17 | #include "node_prog/read_n_edges_program.h" 18 | 19 | using node_prog::search_type; 20 | using node_prog::read_n_edges_params; 21 | using node_prog::read_n_edges_state; 22 | using node_prog::cache_response; 23 | 24 | uint64_t 25 | read_n_edges_params :: size() const 26 | { 27 | uint64_t toRet = message::size(num_edges) 28 | + message::size(edges_props) 29 | + message::size(return_edges); 30 | return toRet; 31 | } 32 | 33 | void 34 | read_n_edges_params :: pack(e::packer& packer) const 35 | { 36 | message::pack_buffer(packer, num_edges); 37 | message::pack_buffer(packer, edges_props); 38 | message::pack_buffer(packer, return_edges); 39 | } 40 | 41 | void 42 | read_n_edges_params :: unpack(e::unpacker& unpacker) 43 | { 44 | message::unpack_buffer(unpacker, num_edges); 45 | message::unpack_buffer(unpacker, edges_props); 46 | message::unpack_buffer(unpacker, return_edges); 47 | } 48 | 49 | std::pair>> 50 | node_prog :: read_n_edges_node_program( 51 | node &n, 52 | db::remote_node &, 53 | read_n_edges_params ¶ms, 54 | std::function, 55 | std::function, std::shared_ptr>, cache_key_t)>&, 56 | cache_response*) 57 | { 58 | auto elist = n.get_edges(); 59 | int pushcnt = 0; 60 | for (edge &e : elist) { 61 | if (e.has_all_properties(params.edges_props)) { 62 | pushcnt++; 63 | params.return_edges.emplace_back(e.get_handle()); 64 | if (--params.num_edges == 0) { 65 | break; 66 | } 67 | } 68 | } 69 | return std::make_pair(search_type::DEPTH_FIRST, std::vector> 70 | (1, std::make_pair(db::coordinator, std::move(params)))); 71 | } 72 | -------------------------------------------------------------------------------- /node_prog/read_n_edges_program.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Program to read the 'top' n edges, used to 4 | * simulate range queries for soc. net. workload. 5 | * 6 | * Created: 2014-04-15 21:30:10 7 | * 8 | * Author: Ayush Dubey, dubey@cs.cornell.edu 9 | * 10 | * Copyright (C) 2013-2014, Cornell University, see the LICENSE 11 | * file for licensing agreement 12 | * =============================================================== 13 | */ 14 | 15 | #ifndef weaver_node_prog_read_n_edges_program_h_ 16 | #define weaver_node_prog_read_n_edges_program_h_ 17 | 18 | #include 19 | #include 20 | 21 | #include "db/remote_node.h" 22 | #include "node_prog/base_classes.h" 23 | #include "node_prog/node.h" 24 | #include "node_prog/cache_response.h" 25 | 26 | namespace node_prog 27 | { 28 | class read_n_edges_params : public Node_Parameters_Base 29 | { 30 | public: 31 | uint64_t num_edges; 32 | std::vector> edges_props; 33 | std::vector return_edges; 34 | 35 | public: 36 | // no caching needed 37 | bool search_cache() { return false; } 38 | cache_key_t cache_key() { return cache_key_t(); } 39 | uint64_t size() const; 40 | void pack(e::packer& packer) const; 41 | void unpack(e::unpacker& unpacker); 42 | }; 43 | 44 | struct read_n_edges_state : public Node_State_Base 45 | { 46 | ~read_n_edges_state() { } 47 | uint64_t size() const { return 0; } 48 | void pack(e::packer&) const { } 49 | void unpack(e::unpacker&) { } 50 | }; 51 | 52 | std::pair>> 53 | read_n_edges_node_program( 54 | node &n, 55 | db::remote_node &, 56 | read_n_edges_params ¶ms, 57 | std::function, 58 | std::function, 59 | std::shared_ptr>, cache_key_t)>&, 60 | cache_response*); 61 | } 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /node_prog/read_node_props_program.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Implementation of read-node node program. 4 | * 5 | * Created: 2014-05-30 12:37:27 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #include "common/stl_serialization.h" 15 | #include "node_prog/node_prog_type.h" 16 | #include "node_prog/node.h" 17 | #include "node_prog/edge.h" 18 | #include "node_prog/read_node_props_program.h" 19 | 20 | using node_prog::search_type; 21 | using node_prog::read_node_props_params; 22 | using node_prog::read_node_props_state; 23 | using node_prog::cache_response; 24 | 25 | uint64_t 26 | read_node_props_params :: size() const 27 | { 28 | uint64_t toRet = message::size(keys) 29 | + message::size(node_props); 30 | return toRet; 31 | } 32 | 33 | void 34 | read_node_props_params :: pack(e::packer& packer) const 35 | { 36 | message::pack_buffer(packer, keys); 37 | message::pack_buffer(packer, node_props); 38 | } 39 | 40 | void 41 | read_node_props_params :: unpack(e::unpacker& unpacker) 42 | { 43 | message::unpack_buffer(unpacker, keys); 44 | message::unpack_buffer(unpacker, node_props); 45 | } 46 | 47 | std::pair>> 48 | node_prog :: read_node_props_node_program( 49 | node &n, 50 | db::remote_node &, 51 | read_node_props_params ¶ms, 52 | std::function, 53 | std::function, 54 | std::shared_ptr>, cache_key_t)>&, 55 | cache_response*) 56 | { 57 | bool fetch_all = params.keys.empty(); 58 | for (std::vector> prop_vec : n.get_properties()) { 59 | std::string key = prop_vec[0]->get_key(); 60 | if (fetch_all || (std::find(params.keys.begin(), params.keys.end(), key) != params.keys.end())) { 61 | for (std::shared_ptr prop: prop_vec) { 62 | params.node_props.emplace_back(key, prop->get_value()); 63 | } 64 | } 65 | } 66 | 67 | return std::make_pair(search_type::DEPTH_FIRST, std::vector> 68 | (1, std::make_pair(db::coordinator, std::move(params)))); 69 | } 70 | -------------------------------------------------------------------------------- /node_prog/read_node_props_program.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Node program to read properties of a single node 4 | * 5 | * Created: Friday 17 January 2014 11:00:03 EDT 6 | * 7 | * Author: Ayush Dubey, Greg Hill 8 | * dubey@cs.cornell.edu, gdh39@cornell.edu 9 | * 10 | * Copyright (C) 2013, Cornell University, see the LICENSE file 11 | * for licensing agreement 12 | * ================================================================ 13 | */ 14 | 15 | #ifndef weaver_node_prog_read_node_props_program_h 16 | #define weaver_node_prog_read_node_props_program_h 17 | 18 | #include 19 | #include 20 | 21 | #include "db/remote_node.h" 22 | #include "node_prog/base_classes.h" 23 | #include "node_prog/node.h" 24 | #include "node_prog/cache_response.h" 25 | 26 | namespace node_prog 27 | { 28 | class read_node_props_params : public Node_Parameters_Base 29 | { 30 | public: 31 | std::vector keys; // empty vector means fetch all props 32 | std::vector> node_props; 33 | 34 | // no caching needed 35 | bool search_cache() { return false; } 36 | cache_key_t cache_key() { return cache_key_t(); } 37 | uint64_t size() const; 38 | void pack(e::packer& packer) const; 39 | void unpack(e::unpacker& unpacker); 40 | }; 41 | 42 | struct read_node_props_state : public Node_State_Base 43 | { 44 | ~read_node_props_state() { } 45 | uint64_t size() const { return 0; } 46 | void pack(e::packer&) const { } 47 | void unpack(e::unpacker&) { } 48 | }; 49 | 50 | std::pair>> 51 | read_node_props_node_program( 52 | node &n, 53 | db::remote_node &, 54 | read_node_props_params ¶ms, 55 | std::function, 56 | std::function, 57 | std::shared_ptr>, cache_key_t)>&, 58 | cache_response*); 59 | } 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /node_prog/traverse_with_props.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Program which traverses graph starting at a 4 | * user-specified node. For each traversal hop, 5 | * user specifies a list of node and edge props 6 | * must be satisfied. 7 | * 8 | * Created: 2014-06-23 11:08:56 9 | * 10 | * Author: Ayush Dubey, dubey@cs.cornell.edu 11 | * 12 | * Copyright (C) 2013, Cornell University, see the LICENSE file 13 | * for licensing agreement 14 | * =============================================================== 15 | */ 16 | 17 | #ifndef weaver_node_prog_traverse_with_props_h_ 18 | #define weaver_node_prog_traverse_with_props_h_ 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | #include "node_prog/boilerplate.h" 25 | #include "node_prog/cache_response.h" 26 | 27 | namespace node_prog 28 | { 29 | struct traverse_props_params: public virtual Node_Parameters_Base 30 | { 31 | bool returning; // false = request spreading out, true = request return 32 | db::remote_node prev_node; 33 | std::deque> node_aliases; 34 | std::deque>> node_props; 35 | std::deque>> edge_props; 36 | bool collect_nodes; 37 | bool collect_edges; 38 | std::unordered_set return_nodes; 39 | std::unordered_set return_edges; 40 | 41 | traverse_props_params(); 42 | ~traverse_props_params() { } 43 | uint64_t size(void*) const; 44 | void pack(e::packer &packer, void*) const; 45 | void unpack(e::unpacker &unpacker, void*); 46 | 47 | // no caching 48 | bool search_cache() { return false; } 49 | cache_key_t cache_key() { return cache_key_t(); } 50 | }; 51 | 52 | struct traverse_props_state: public virtual Node_State_Base 53 | { 54 | bool visited; 55 | uint32_t out_count; // number of requests propagated 56 | db::remote_node prev_node; // previous node 57 | std::unordered_set return_nodes; 58 | std::unordered_set return_edges; 59 | 60 | traverse_props_state(); 61 | ~traverse_props_state() { } 62 | uint64_t size(void*) const; 63 | void pack(e::packer& packer, void*) const ; 64 | void unpack(e::unpacker& unpacker, void*); 65 | }; 66 | 67 | extern "C" { 68 | PROG_FUNC_DECLARE; 69 | } 70 | } 71 | 72 | #endif 73 | -------------------------------------------------------------------------------- /startup_scripts/docker_kill_weaver.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # kill_weaver.sh 4 | # Copyright (C) 2014 Ayush Dubey 5 | # 6 | 7 | # get config file location 8 | if [ $# -eq 1 ] 9 | then 10 | config_file_args="-f $1" 11 | else 12 | config_file_args="" 13 | fi 14 | 15 | # get config params 16 | hyperdex_coord_ipaddr=$(weaver-parse-config -c hyperdex_coord_ipaddr $config_file_args) 17 | hyperdex_coord_port=$(weaver-parse-config -c hyperdex_coord_port $config_file_args) 18 | hyperdex_daemons_ipaddr=($(weaver-parse-config -c hyperdex_daemons_ipaddr $config_file_args)) 19 | hyperdex_daemons_port=($(weaver-parse-config -c hyperdex_daemons_port $config_file_args)) 20 | server_manager_ipaddr=($(weaver-parse-config -c server_manager_ipaddr $config_file_args)) 21 | server_manager_port=($(weaver-parse-config -c server_manager_port $config_file_args)) 22 | kronos_ipaddr=($(weaver-parse-config -c kronos_ipaddr $config_file_args)) 23 | kronos_port=($(weaver-parse-config -c kronos_port $config_file_args)) 24 | 25 | echo "Killing all replicant daemons at $hyperdex_coord_ipaddr" 26 | pidof replicant-daemon | xargs kill -9 27 | 28 | num_daemons=${#hyperdex_daemons_ipaddr[*]} 29 | for i in $(seq 1 $num_daemons); 30 | do 31 | idx=$(($i-1)) 32 | ipaddr=${hyperdex_daemons_ipaddr[$idx]} 33 | echo "Killing all HyperDex daemons at $ipaddr" 34 | pidof hyperdex-daemon | xargs kill -9 35 | done 36 | 37 | num_sm_daemons=${#server_manager_ipaddr[*]} 38 | for i in $(seq 1 $num_sm_daemons); 39 | do 40 | idx=$(($i-1)) 41 | ipaddr=${server_manager_ipaddr[$idx]} 42 | echo "Killing all replicant daemons at $ipaddr" 43 | pidof replicant-daemon | xargs kill -9 44 | done 45 | 46 | num_kronos_daemons=${#kronos_ipaddr[*]} 47 | for i in $(seq 1 $num_kronos_daemons); 48 | do 49 | idx=$(($i-1)) 50 | ipaddr=${kronos_ipaddr[$idx]} 51 | echo "Killing all replicant daemons at $ipaddr" 52 | pidof replicant-daemon | xargs kill -9 53 | done 54 | -------------------------------------------------------------------------------- /startup_scripts/ps_weaver.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # ps_weaver_support.sh 4 | # Copyright (C) 2014 Ayush Dubey 5 | # 6 | 7 | # get config file location 8 | if [ $# -eq 1 ] 9 | then 10 | config_file_args="-f $1" 11 | else 12 | config_file_args="" 13 | fi 14 | 15 | # get config params 16 | hyperdex_coord_ipaddr=$(weaver-parse-config -c hyperdex_coord_ipaddr $config_file_args) 17 | hyperdex_coord_port=$(weaver-parse-config -c hyperdex_coord_port $config_file_args) 18 | hyperdex_daemons_ipaddr=($(weaver-parse-config -c hyperdex_daemons_ipaddr $config_file_args)) 19 | hyperdex_daemons_port=($(weaver-parse-config -c hyperdex_daemons_port $config_file_args)) 20 | server_manager_ipaddr=($(weaver-parse-config -c server_manager_ipaddr $config_file_args)) 21 | server_manager_port=($(weaver-parse-config -c server_manager_port $config_file_args)) 22 | kronos_ipaddr=($(weaver-parse-config -c kronos_ipaddr $config_file_args)) 23 | kronos_port=($(weaver-parse-config -c kronos_port $config_file_args)) 24 | 25 | echo "Listing all replicant daemons at $hyperdex_coord_ipaddr" 26 | ssh $hyperdex_coord_ipaddr "ps x | grep replicant-daemon" 27 | 28 | num_daemons=${#hyperdex_daemons_ipaddr[*]} 29 | for i in $(seq 1 $num_daemons); 30 | do 31 | idx=$(($i-1)) 32 | ipaddr=${hyperdex_daemons_ipaddr[$idx]} 33 | echo "Listing all HyperDex daemons at $ipaddr" 34 | ssh $ipaddr "ps x | grep hyperdex-daemon" 35 | done 36 | 37 | num_sm_daemons=${#server_manager_ipaddr[*]} 38 | for i in $(seq 1 $num_sm_daemons); 39 | do 40 | idx=$(($i-1)) 41 | ipaddr=${server_manager_ipaddr[$idx]} 42 | echo "Listing all replicant daemons at $ipaddr" 43 | ssh $ipaddr "ps x | grep replicant-daemon" 44 | done 45 | 46 | num_kronos_daemons=${#kronos_ipaddr[*]} 47 | for i in $(seq 1 $num_kronos_daemons); 48 | do 49 | idx=$(($i-1)) 50 | ipaddr=${kronos_ipaddr[$idx]} 51 | echo "Listing all replicant daemons at $ipaddr" 52 | ssh $ipaddr "ps x | grep replicant-daemon" 53 | done 54 | -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | tests 2 | *.pyc 3 | *.log 4 | *.trs 5 | -------------------------------------------------------------------------------- /tests/cpp/basic_client.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Basic graph db test 4 | * 5 | * Created: 01/23/2013 01:20:10 PM 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #include "client/client.h" 15 | #include "node_prog/reach_program.h" 16 | 17 | bool 18 | execute_request(uint64_t n1, uint64_t n2, client &c) 19 | { 20 | bool ret; 21 | node_prog::reach_params rp; 22 | std::unique_ptr res; 23 | std::vector> initial_args; 24 | rp.mode = false; 25 | rp.reachable = false; 26 | rp.prev_node.loc = COORD_ID; 27 | rp.dest = n2; 28 | initial_args.emplace_back(std::make_pair(n1, rp)); 29 | res = c.run_node_program(node_prog::REACHABILITY, initial_args); 30 | ret = res->reachable; 31 | return ret; 32 | } 33 | 34 | void 35 | basic_client_test() 36 | { 37 | client c(CLIENT_ID); 38 | auto edge_props = std::make_shared>(); 39 | size_t nodes[10]; 40 | size_t edges[10]; 41 | int i; 42 | // create nodes 43 | for (i = 0; i < 10; i++) { 44 | nodes[i] = c.create_node(); 45 | } 46 | // request 1 47 | edges[0] = c.create_edge(nodes[0], nodes[1]); 48 | edges[1] = c.create_edge(nodes[1], nodes[2]); 49 | assert(execute_request(nodes[0], nodes[2], c)); 50 | // request 2 51 | c.delete_edge(nodes[0], edges[0]); 52 | c.delete_edge(nodes[1], edges[1]); 53 | assert(!execute_request(nodes[0], nodes[2], c)); 54 | // requests 3-5 55 | edges[0] = c.create_edge(nodes[0], nodes[2]); 56 | edges[1] = c.create_edge(nodes[1], nodes[3]); 57 | assert(execute_request(nodes[0], nodes[2], c)); 58 | assert(execute_request(nodes[1], nodes[3], c)); 59 | assert(!execute_request(nodes[3], nodes[1], c)); 60 | } 61 | -------------------------------------------------------------------------------- /tests/cpp/basic_migration_test.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Repetitively send messages between two 4 | * nodes and check that they end up on the same 5 | * shard. 6 | * 7 | * Created: 06/13/2013 01:09:44 PM 8 | * 9 | * Author: Ayush Dubey, dubey@cs.cornell.edu 10 | * 11 | * Copyright (C) 2013, Cornell University, see the LICENSE file 12 | * for licensing agreement 13 | * =============================================================== 14 | */ 15 | 16 | #include "client/client.h" 17 | 18 | inline void 19 | basic_migration_test(bool to_exit) 20 | { 21 | client c(CLIENT_ID); 22 | uint64_t n1, n2; 23 | std::vector> initial_args; 24 | n1 = c.create_node(); 25 | n2 = c.create_node(); 26 | c.create_edge(n1, n2); 27 | assert(c.get_node_loc(n1) != c.get_node_loc(n2)); 28 | initial_args.emplace_back(std::make_pair(n1, node_prog::reach_params())); 29 | initial_args.at(0).second.mode = false; 30 | initial_args.at(0).second.reachable = false; 31 | initial_args.at(0).second.prev_node.loc = COORD_ID; 32 | initial_args.at(0).second.dest = n2; 33 | for (int i = 0; i < 100000; i++) { 34 | if ((i % 100) == 0) 35 | WDEBUG << "completed " << i << " requests" << std::endl; 36 | std::unique_ptr res = c.run_node_program(node_prog::REACHABILITY, initial_args); 37 | assert(res->reachable); 38 | } 39 | uint64_t loc1 = c.get_node_loc(n1); 40 | uint64_t loc2 = c.get_node_loc(n2); 41 | assert(loc1 == loc2); 42 | c.delete_node(n1); 43 | c.delete_node(n2); 44 | if (to_exit) 45 | c.exit_weaver(); 46 | } 47 | -------------------------------------------------------------------------------- /tests/cpp/create_graph.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Simple create node and edge test. 4 | * 5 | * Created: 09/11/2013 01:48:38 PM 6 | * 7 | * Author: Ayush Dubey, dubey@cs.cornell.edu 8 | * 9 | * Copyright (C) 2013, Cornell University, see the LICENSE file 10 | * for licensing agreement 11 | * =============================================================== 12 | */ 13 | 14 | #include 15 | 16 | #include "client/client.h" 17 | 18 | void 19 | create_elems(client::client *c) 20 | { 21 | int i; 22 | size_t nodes[10]; 23 | // create nodes 24 | for (i = 0; i < 10; i++) { 25 | uint64_t tx_id = c->begin_tx(); 26 | nodes[i] = c->create_node(tx_id); 27 | WDEBUG << "Created node, handle = " << nodes[i] << std::endl; 28 | c->end_tx(tx_id); 29 | } 30 | // edges 31 | uint64_t tx_id = c->begin_tx(); 32 | for (i = 0; i < 10; i++) { 33 | c->create_edge(tx_id, nodes[i], nodes[(i+1)%10]); 34 | } 35 | c->end_tx(tx_id); 36 | for (i = 0; i < 10; i+=2) { 37 | uint64_t tx_id = c->begin_tx(); 38 | nodes[i] = c->create_node(tx_id); 39 | WDEBUG << "Created node, handle = " << nodes[i] << std::endl; 40 | nodes[i+1] = c->create_node(tx_id); 41 | WDEBUG << "Created node, handle = " << nodes[i+1] << std::endl; 42 | c->end_tx(tx_id); 43 | } 44 | } 45 | 46 | void 47 | create_graph_test() 48 | { 49 | int i; 50 | if (NUM_VTS == 1) { 51 | WDEBUG << "single vts" << std::endl; 52 | client::client c(CLIENT_ID, 0); 53 | create_elems(&c); 54 | } else { 55 | // multiple clients 56 | WDEBUG << "multiple vts, multiple clients" << std::endl; 57 | client::client* cl[50]; 58 | for (i = 0; i < 50; i++) { 59 | cl[i] = new client::client(CLIENT_ID+i, (i % 2 == 0) ? 0 : 1); 60 | } 61 | std::thread* t[50]; 62 | for (i = 0; i < 50; i++) { 63 | t[i] = new std::thread(create_elems, cl[i]); 64 | } 65 | for (i = 0; i < 50; i++) { 66 | t[i]->join(); 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /tests/cpp/hyper_stub_perf.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Test hyper stub performance. 4 | * 5 | * Author: Ayush Dubey, dubey@cs.cornell.edu 6 | * 7 | * Copyright (C) 2014, Cornell University, see the LICENSE file 8 | * for licensing agreement 9 | * =============================================================== 10 | */ 11 | 12 | #include "db/hyper_stub.h" 13 | #include "common/clock.h" 14 | #include "common/config_constants.h" 15 | 16 | DECLARE_CONFIG_CONSTANTS; 17 | 18 | int main(int argc, char *argv[]) 19 | { 20 | if (argc != 2) { 21 | std::cerr << "usage: " << argv[0] << " " << std::endl; 22 | return -1; 23 | } 24 | 25 | init_config_constants(); 26 | vclock_ptr_t dummy_clk; 27 | db::node n(argv[1], 0, dummy_clk, nullptr); 28 | db::hyper_stub hs(0,0); 29 | wclock::weaver_timer timer; 30 | 31 | uint64_t start = timer.get_real_time_millis(); 32 | if (hs.recover_node(n)) { 33 | std::cout << "node " << n.get_handle() 34 | << " has #edges=" << n.out_edges.size() 35 | << std::endl; 36 | } else { 37 | std::cerr << "error in get_node, probably " << argv[1] << " does not exist" << std::endl; 38 | } 39 | uint64_t end = timer.get_real_time_millis(); 40 | float secs = (end-start) / 1000.0; 41 | std::cout << "time taken=" << secs << " s" << std::endl; 42 | 43 | n.out_edges.clear(); 44 | return 0; 45 | } 46 | -------------------------------------------------------------------------------- /tests/cpp/test_dynamic.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Test program for dynamic linking 4 | * 5 | * Author: Ayush Dubey, dubey@cs.cornell.edu 6 | * 7 | * Copyright (C) 2015, Cornell University, see the LICENSE file 8 | * for licensing agreement 9 | * =============================================================== 10 | */ 11 | 12 | #include "client/client.h" 13 | 14 | using cl::client; 15 | using node_prog::traverse_props_params; 16 | 17 | int main() 18 | { 19 | client c("127.0.0.1", 2002, "/usr/local/etc/weaver.yaml"); 20 | 21 | std::string prog_handle; 22 | //c.register_node_prog("./.libs/libtestprog.so", prog_handle); 23 | //std::cout << "client got prog handle = " << prog_handle << std::endl; 24 | 25 | std::string empty; 26 | std::string ayush = "ayush"; 27 | std::string egs = "egs"; 28 | std::vector empty_str_vec; 29 | std::vector> empty_str_pair_vec; 30 | 31 | c.begin_tx(); 32 | c.create_node(ayush, empty_str_vec); 33 | c.create_node(egs, empty_str_vec); 34 | std::string edge_handle; 35 | c.create_edge(edge_handle, ayush, empty, egs, empty); 36 | c.end_tx(); 37 | 38 | db::remote_node coord(0, std::string()); 39 | traverse_props_params arg_param, ret_param; 40 | arg_param.returning = false; 41 | arg_param.prev_node = coord; 42 | arg_param.node_aliases.emplace_back(empty_str_vec); 43 | arg_param.node_props.emplace_back(empty_str_pair_vec); 44 | arg_param.edge_props.emplace_back(empty_str_pair_vec); 45 | arg_param.node_aliases.emplace_back(empty_str_vec); 46 | arg_param.node_props.emplace_back(empty_str_pair_vec); 47 | arg_param.collect_nodes = false; 48 | arg_param.collect_edges = false; 49 | 50 | std::vector> prog_args; 51 | prog_args.emplace_back(std::make_pair(ayush, arg_param)); 52 | c.traverse_props_program(prog_args, ret_param); 53 | 54 | bool found = false; 55 | for (const std::string &s: ret_param.return_nodes) { 56 | if ("egs" == s) { 57 | found = true; 58 | } 59 | } 60 | 61 | assert(found); 62 | 63 | return 0; 64 | } 65 | -------------------------------------------------------------------------------- /tests/cpp/triangle_counting_test.h: -------------------------------------------------------------------------------- 1 | #include "client/client.h" 2 | #include "node_prog/node_prog_type.h" 3 | #include "node_prog/triangle_program.h" 4 | #define LENGTH 4000 5 | 6 | void line_triangle_test(client::client &c) { 7 | // make line of length line_length 8 | std::cout << "line reachability makign graph" << std::endl; 9 | uint64_t tx_id = c.begin_tx(); 10 | size_t nodes[LENGTH]; 11 | for (int i = 0; i < LENGTH; i++) { 12 | nodes[i] = c.create_node(tx_id); 13 | } 14 | std::cout << "sending node transation" << std::endl; 15 | c.end_tx(tx_id); 16 | std::cout << "made nodes" << std::endl; 17 | 18 | int num_edges = 0; 19 | tx_id = c.begin_tx(); 20 | for (int i = 0; i < LENGTH-1; i++) { 21 | c.create_edge(tx_id, nodes[i], nodes[i+1]); 22 | num_edges++; 23 | c.create_edge(tx_id, nodes[i+1], nodes[i]); // temp to see if things break 24 | num_edges++; 25 | } 26 | c.end_tx(tx_id); 27 | 28 | std::vector> initial_args; 29 | node_prog::triangle_params params; 30 | initial_args.emplace_back(std::make_pair(-1, params)); 31 | std::unique_ptrres = c.run_node_program(node_prog::TRIANGLE_COUNT, initial_args); 32 | std::cout << "on " << num_edges<< " graph ... was " << res->num_edges << std::endl << std::endl; 33 | assert(res->num_edges == (2 * LENGTH)-2); 34 | 35 | test_graph * g = new test_graph(&c, time(NULL), 10000, 100000, false, false); 36 | num_edges += 100000; 37 | 38 | node_prog::triangle_params params2; 39 | initial_args.clear(); 40 | initial_args.emplace_back(std::make_pair(-1, params2)); 41 | res = c.run_node_program(node_prog::TRIANGLE_COUNT, initial_args); 42 | std::cout << "... was " << res->num_edges << std::endl << std::endl; 43 | assert(res->num_edges == num_edges); 44 | } 45 | 46 | void 47 | triangle_counting_test() 48 | { 49 | client::client c(CLIENT_ID, 0); 50 | line_triangle_test(c); 51 | } 52 | -------------------------------------------------------------------------------- /tests/cpp/unreachable_reach_program.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Issue multiple queries between two nodes 4 | * that are not reachable, forcing traversal of 5 | * entire graph. 6 | * 7 | * Created: 06/17/2013 03:36:55 PM 8 | * 9 | * Author: Ayush Dubey, dubey@cs.cornell.edu 10 | * 11 | * Copyright (C) 2013, Cornell University, see the LICENSE file 12 | * for licensing agreement 13 | * =============================================================== 14 | */ 15 | 16 | #include "client/client.h" 17 | #include "node_prog/node_prog_type.h" 18 | #include "node_prog/reach_program.h" 19 | #include "test_base.h" 20 | 21 | #define URP_REQUESTS 1000 22 | 23 | void 24 | unreachable_reach_prog(bool to_exit) 25 | { 26 | client c(CLIENT_ID); 27 | int i, num_nodes, num_edges; 28 | std::ofstream seed_file; 29 | uint64_t seed = time(NULL); 30 | WDEBUG << "seed " << seed << std::endl; 31 | seed_file.open("seed.rec"); 32 | seed_file << seed; 33 | seed_file.close(); 34 | 35 | // creating graph 36 | std::ifstream count_in; 37 | count_in.open("node_count.rec"); 38 | count_in >> num_nodes; 39 | count_in.close(); 40 | num_edges = (int)(5.5 * (double)num_nodes); // dense => lots of traversal 41 | test_graph g(&c, seed, num_nodes, num_edges, true, to_exit); 42 | 43 | // requests 44 | node_prog::reach_params rp; 45 | rp.mode = false; 46 | rp.reachable = false; 47 | rp.prev_node.loc = COORD_ID; 48 | 49 | std::ofstream file, req_time; 50 | file.open("requests.rec"); 51 | for (i = 0; i < URP_REQUESTS; i++) { 52 | int first = rand() % num_nodes; 53 | file << first << " " << num_nodes << std::endl; 54 | std::vector> initial_args; 55 | rp.dest = g.nodes[num_nodes]; 56 | initial_args.emplace_back(std::make_pair(g.nodes[first], rp)); 57 | WDEBUG << "Request " << i << ", from source " << g.nodes[first] << " to dest " << g.nodes[num_nodes] << "." << std::endl; 58 | std::unique_ptr res = c.run_node_program(node_prog::REACHABILITY, initial_args); 59 | assert(!res->reachable); 60 | } 61 | file.close(); 62 | g.end_test(); 63 | } 64 | -------------------------------------------------------------------------------- /tests/cpp/vc_ordering.h: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Test vector clock ordering, including 4 | * ambiguous cases which involve Kronos calls. 5 | * 6 | * Created: 09/10/2013 02:05:20 PM 7 | * 8 | * Author: Ayush Dubey, dubey@cs.cornell.edu 9 | * 10 | * Copyright (C) 2013, Cornell University, see the LICENSE file 11 | * for licensing agreement 12 | * =============================================================== 13 | */ 14 | 15 | #include "common/event_order.h" 16 | 17 | void 18 | vc_ordering_test() 19 | { 20 | vc::vclock clk1, clk2; 21 | clk1.vt_id = 0; 22 | clk2.vt_id = 1; 23 | for (uint64_t i = 0; i < NUM_VTS; i++) { 24 | clk1.clock.push_back(1); 25 | clk2.clock.push_back(0); 26 | } 27 | assert(order::compare_two_clocks(clk1.clock, clk2.clock) == 1); 28 | assert(order::compare_two_clocks(clk2.clock, clk1.clock) == 0); 29 | assert(order::compare_two_clocks(clk2.clock, clk2.clock) == 2); 30 | assert(order::compare_two_vts(clk1, clk2) == 1); 31 | assert(order::compare_two_vts(clk2, clk1) == 0); 32 | assert(order::compare_two_vts(clk2, clk2) == 2); 33 | clk2.clock.at(0) = 2; 34 | assert(order::compare_two_clocks(clk1.clock, clk2.clock) == -1); 35 | assert(order::compare_two_clocks(clk2.clock, clk1.clock) == -1); 36 | order::kronos_cl = new chronos_client(KRONOS_IPADDR, KRONOS_PORT); 37 | assert(NUM_VTS == KRONOS_NUM_VTS); 38 | std::cout << order::compare_two_vts(clk1, clk2) << std::endl; 39 | std::cout << order::compare_two_vts(clk2, clk1) << std::endl; 40 | assert(order::compare_two_vts(clk1, clk2) == 0); 41 | assert(order::compare_two_vts(clk2, clk1) == 1); 42 | } 43 | -------------------------------------------------------------------------------- /tests/python/.gitignore: -------------------------------------------------------------------------------- 1 | shards 2 | -------------------------------------------------------------------------------- /tests/python/benchmarks/nwx_reachability.py: -------------------------------------------------------------------------------- 1 | # =============================================================== 2 | # Description: Test performance of reachability program. 3 | # 4 | # Created: 12/17/2013 07:17:19 PM 5 | # 6 | # Author: Greg Hill, gdh39@cornell.edu 7 | # 8 | # Copyright (C) 2013, Cornell University, see the LICENSE file 9 | # for licensing agreement 10 | # =============================================================== 11 | # 12 | 13 | import random 14 | import sys 15 | import time 16 | 17 | import weaver.client as client 18 | import simple_client 19 | import test_base 20 | import random 21 | from networkx import * 22 | 23 | num_nodes = 5000 24 | num_dests = 200 25 | requests_per_dest = 1 26 | 27 | def exec_traversals(reqs, sc, caching): 28 | start = time.time() 29 | for r in reqs: 30 | assert(r[0] is not 0) 31 | assert(r[1] is not 0) 32 | sc.reachability(r[0], r[1], caching = caching) 33 | sys.stdout.write('.') 34 | sys.stdout.flush() 35 | print ' done' 36 | end = time.time() 37 | return (end-start) 38 | 39 | 40 | coord_id = 0 41 | c_list = [] 42 | c_list.append(client.Client(client._CLIENT_ID, coord_id)) 43 | c_list.append(client.Client(client._CLIENT_ID + 1, coord_id)) 44 | c_list.append(client.Client(client._CLIENT_ID + 2, coord_id)) 45 | sc = simple_client.simple_client(c_list[0]) 46 | 47 | reqs = [] 48 | random.seed(42) 49 | g = test_base.test_graph(c_list, num_nodes, 2*num_nodes, seed = 42) 50 | 51 | for _ in range(num_dests): 52 | dest = random.choice(g.nodes) 53 | for _ in range(requests_per_dest): 54 | reqs.append((random.choice(g.nodes), dest)) 55 | 56 | print "starting traversals" 57 | 58 | t = exec_traversals(reqs, sc, False) 59 | print "time taken for " + str(num_dests * requests_per_dest) + " random reachability requests over " + str(num_nodes) + " nodes was: " + str(t) 60 | 61 | t = exec_traversals(reqs, sc, True) 62 | print "time taken for " + str(num_dests * requests_per_dest) + " random reachability requests with hot caching over " + str(num_nodes) + " nodes was: " + str(t) 63 | -------------------------------------------------------------------------------- /tests/python/benchmarks/sequential_pathless_reachability_bench.py: -------------------------------------------------------------------------------- 1 | import random 2 | import sys 3 | import time 4 | 5 | import weaver.client as client 6 | 7 | num_dests = 1000 8 | requests_per_dest = 1 9 | 10 | def exec_traversals(reqs, cl): 11 | rp = client.ReachParams(caching=True) 12 | prp = client.PathlessReachParams() 13 | start = time.time() 14 | for r in reqs: 15 | s= time.time() 16 | prp.dest = r[1] 17 | prog_args = [(r[0], prp)] 18 | response = cl.run_pathless_reach_program(prog_args) 19 | if (response.reachable): 20 | print '{0:05}'.format(r[0]) + " " + '{0:05}'.format(r[1]) + " " + str(time.time()-s) 21 | else: 22 | print '{0:05}'.format(r[0]) + " " + '{0:05}'.format(r[1]) + " " + str(time.time()-s) + " " + str(response.reachable) 23 | sys.stdout.flush() 24 | print ' done' 25 | end = time.time() 26 | return (end-start) 27 | 28 | #num_nodes = 82168 # snap soc-Slashdot0902 29 | #num_nodes = 10876 # snap p2pgnutella04 30 | num_nodes = 81306 # snap twitter-combined 31 | # node handles are range(0, num_nodes) 32 | 33 | coord_id = 0 34 | c = client.Client(client._CLIENT_ID, coord_id) 35 | 36 | reqs = [] 37 | random.seed(42) 38 | for _ in range(num_dests): 39 | dest = random.randint(0, num_nodes-1) 40 | for _ in range(requests_per_dest): 41 | reqs.append((random.randint(0, num_nodes-1), dest)) 42 | 43 | print "starting traversals" 44 | t = exec_traversals(reqs, c) 45 | print "time taken for " + str(num_dests * requests_per_dest) + " random reachability requests over " + str(num_nodes) + " nodes was: " + str(t) 46 | -------------------------------------------------------------------------------- /tests/python/correctness/concurrent_clients.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # 3 | # =============================================================== 4 | # Description: Lots of clients doing concurrent requests 5 | # Warning: may require raising ulimit -n 6 | # 7 | # Author: Ayush Dubey, dubey@cs.cornell.edu 8 | # 9 | # Copyright (C) 2015, Cornell University, see the LICENSE file 10 | # for licensing agreement 11 | # =============================================================== 12 | # 13 | 14 | import sys 15 | import threading 16 | 17 | try: 18 | from weaver import client 19 | except ImportError: 20 | import client 21 | 22 | config_file='' 23 | 24 | if len(sys.argv) > 1: 25 | config_file = sys.argv[1] 26 | 27 | 28 | # exec lots of get node requests 29 | def flood_get_node(c, num_reqs, idx): 30 | for i in range(num_reqs): 31 | n = c.get_node('ayush') 32 | assert n.properties['type'] == ['user'] 33 | 34 | 35 | num_clients = 500 36 | num_requests = 100 37 | clients = [] 38 | 39 | # create clients 40 | for i in range(num_clients): 41 | clients.append(client.Client('127.0.0.1', 2002, config_file)) 42 | 43 | print 'created clients' 44 | 45 | # create node for user ayush 46 | clients[0].begin_tx() 47 | clients[0].create_node('ayush') 48 | clients[0].set_node_properties({'type': 'user', 'age': '25'}, 'ayush') 49 | clients[0].end_tx() 50 | 51 | print 'created node ayush' 52 | 53 | threads = [] 54 | for i in range(num_clients): 55 | t = threading.Thread(target=flood_get_node, args=(clients[i], num_requests, i)) 56 | t.start() 57 | threads.append(t) 58 | 59 | print 'started threads' 60 | 61 | for i in range(num_clients): 62 | threads[i].join() 63 | 64 | print 'done all threads' 65 | 66 | clients[0].begin_tx() 67 | clients[0].delete_node('ayush') 68 | clients[0].end_tx() 69 | 70 | print 'Concurrent clients successful, executed ' + str(num_clients*num_requests) + ' requests' 71 | -------------------------------------------------------------------------------- /tests/python/correctness/delete_tx.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # 3 | # =============================================================== 4 | # Description: Test deletes with multiple clients and 5 | # timestampers. 6 | # 7 | # Created: 2014-06-17 10:41:16 8 | # 9 | # Author: Ayush Dubey, dubey@cs.cornell.edu 10 | # 11 | # Copyright (C) 2013, Cornell University, see the LICENSE file 12 | # for licensing agreement 13 | # =============================================================== 14 | # 15 | 16 | import sys 17 | import time 18 | import random 19 | import threading 20 | import weaver.client as client 21 | 22 | # grid graph params 23 | num_node_groups = 1000 24 | nodes_per_group = 5 25 | num_vts = 1 26 | num_cls = 10 27 | clients = [] 28 | for i in range(num_cls): 29 | clients.append(client.Client(client._CLIENT_ID + i, i % num_vts)) 30 | 31 | # create grid graph 32 | nodes = [[0 for i in range(nodes_per_group)] for j in range(num_node_groups)] 33 | create_cl = clients[0] 34 | for i in range(num_node_groups): 35 | for j in range(nodes_per_group): 36 | create_cl.begin_tx() 37 | nodes[i][j] = create_cl.create_node() 38 | create_cl.end_tx() 39 | 40 | print 'created graph' 41 | 42 | # delete all nodes, add random delays between txs 43 | def del_edges(my_id, cl, success): 44 | assert len(success) == num_node_groups 45 | for i in range(num_node_groups): 46 | cl.begin_tx() 47 | for j in range(nodes_per_group): 48 | n = nodes[i][j] 49 | cl.delete_node(n) 50 | assert not success[i] 51 | success[i] = cl.end_tx() 52 | if success[i]: 53 | print 'Client ' + str(my_id) + ': tx ' + str(i) 54 | delay = random.random()/10 55 | time.sleep(delay) 56 | 57 | threads = [] 58 | success = [[False for i in range(num_node_groups)] for j in range(num_cls)] 59 | for i in range(num_cls): 60 | thr = threading.Thread(target=del_edges, args=(i, clients[i], success[i])) 61 | thr.start() 62 | threads.append(thr) 63 | 64 | num_finished = 0 65 | for thr in threads: 66 | thr.join() 67 | num_finished += 1 68 | print 'Thread finished #' + str(num_finished) 69 | 70 | # check that only one transaction per node succeeded 71 | for i in range(num_node_groups): 72 | num_true = 0 73 | for j in range(num_cls): 74 | if success[j][i]: 75 | num_true += 1 76 | assert num_true < 2 77 | -------------------------------------------------------------------------------- /tests/python/correctness/dijkstra_basic_test.py: -------------------------------------------------------------------------------- 1 | # 2 | # =============================================================== 3 | # Description: Line reachability program in Python. 4 | # 5 | # Created: 11/11/2013 03:17:55 PM 6 | # 7 | # Author: Ayush Dubey, dubey@cs.cornell.edu 8 | # 9 | # Copyright (C) 2013, Cornell University, see the LICENSE file 10 | # for licensing agreement 11 | # =============================================================== 12 | # 13 | 14 | import sys 15 | 16 | import weaver.client as client 17 | 18 | # create graph from file 19 | coord_id = 0 20 | c = client.Client(client._CLIENT_ID, coord_id) 21 | 22 | nodes = dict() 23 | 24 | tx_id = c.begin_tx() 25 | for i in range(6): 26 | nodes[i] = c.create_node(tx_id) 27 | 28 | edge_id = c.create_edge(tx_id, nodes[0], nodes[1]) 29 | c.set_edge_property(tx_id, nodes[0], edge_id, 'weight', '6') 30 | 31 | edge_id = c.create_edge(tx_id, nodes[0], nodes[2]) 32 | c.set_edge_property(tx_id, nodes[0], edge_id, 'weight', '5') 33 | 34 | edge_id = c.create_edge(tx_id, nodes[1], nodes[3]) 35 | c.set_edge_property(tx_id, nodes[1], edge_id, 'weight', '6') 36 | 37 | edge_id = c.create_edge(tx_id, nodes[1], nodes[4]) 38 | c.set_edge_property(tx_id, nodes[1], edge_id, 'weight', '7') 39 | 40 | edge_id = c.create_edge(tx_id, nodes[2], nodes[4]) 41 | c.set_edge_property(tx_id, nodes[2], edge_id, 'weight', '6') 42 | 43 | edge_id = c.create_edge(tx_id, nodes[3], nodes[2]) 44 | c.set_edge_property(tx_id, nodes[3], edge_id, 'weight', '6') 45 | 46 | edge_id = c.create_edge(tx_id, nodes[3], nodes[5]) 47 | c.set_edge_property(tx_id, nodes[3], edge_id, 'weight', '8') 48 | 49 | edge_id = c.create_edge(tx_id, nodes[4], nodes[5]) 50 | c.set_edge_property(tx_id, nodes[4], edge_id, 'weight', '6') 51 | 52 | c.end_tx(tx_id) 53 | 54 | dp = client.DijkstraParams(src_handle=nodes[0], dst_handle=nodes[5], edge_weight_name="weight", is_widest_path=False) 55 | prog_args = [(nodes[0], dp)] 56 | 57 | response = c.run_dijkstra_program(prog_args) 58 | print "shortest path response was cost " + str(response.cost) 59 | assert(response.cost == 17) 60 | 61 | prog_args[0][1].is_widest_path = True 62 | response = c.run_dijkstra_program(prog_args) 63 | print "widest path response was cost " + str(response.cost) 64 | assert(response.cost == 6) 65 | -------------------------------------------------------------------------------- /tests/python/correctness/empty_graph_sanity_checks.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # 3 | # =============================================================== 4 | # Description: Sanity checks -- transactions on an empty 5 | # graph should all fail. 6 | # 7 | # Created: 2013-12-17 16:56:37 8 | # 9 | # Author: Ayush Dubey, dubey@cs.cornell.edu 10 | # 11 | # Copyright (C) 2013, Cornell University, see the LICENSE file 12 | # for licensing agreement 13 | # =============================================================== 14 | # 15 | 16 | import sys 17 | 18 | try: 19 | import weaver.client as client 20 | except ImportError: 21 | import client 22 | 23 | config_file='' 24 | if len(sys.argv) > 1: 25 | config_file = sys.argv[1] 26 | 27 | coord_id = 0 28 | c = client.Client('127.0.0.1', 2002, config_file) 29 | 30 | def try_commit(c, msg): 31 | try: 32 | c.end_tx() 33 | assert False, msg 34 | except client.WeaverError: 35 | pass 36 | 37 | c.begin_tx() 38 | c.create_edge('ayush', '42') 39 | try_commit(c, 'create edge') 40 | 41 | c.begin_tx() 42 | c.delete_node('298437') 43 | try_commit(c, 'delete node') 44 | 45 | c.begin_tx() 46 | c.delete_edge('42', '298437') 47 | try_commit(c, 'delete edge') 48 | 49 | c.begin_tx() 50 | c.set_node_property('egs', 'type', 'user') 51 | try_commit(c, 'set node property') 52 | 53 | c.begin_tx() 54 | c.set_edge_property('84', '42', 'color', 'blue') 55 | try_commit(c, 'set edge property') 56 | 57 | print 'Pass empty_graph_sanity_checks.' 58 | -------------------------------------------------------------------------------- /tests/python/correctness/ft_shard_basic.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # 3 | # =============================================================== 4 | # Description: Basic fault tolerance test: 5 | # create a graph, kill some primary shards, read 6 | # from backups. 7 | # 8 | # Created: 2014-06-12 10:21:05 9 | # 10 | # Author: Ayush Dubey, dubey@cs.cornell.edu 11 | # 12 | # Copyright (C) 2013, Cornell University, see the LICENSE file 13 | # for licensing agreement 14 | # =============================================================== 15 | # 16 | 17 | import sys 18 | import time 19 | import weaver.client as client 20 | 21 | # grid graph params 22 | grid_sz = 10 # grid_sz*grid_sz nodes 23 | coord_id = 0 24 | c = client.Client('127.0.0.1', 2002) 25 | 26 | # create grid graph 27 | nodes = [[0] * grid_sz] * grid_sz 28 | edges = {} 29 | for i in range(grid_sz): 30 | for j in range(grid_sz): 31 | c.begin_tx() 32 | nodes[i][j] = c.create_node() 33 | edges[nodes[i][j]] = [] 34 | if i > 0: 35 | e = c.create_edge(nodes[i-1][j], nodes[i][j]) 36 | edges[nodes[i-1][j]].append(e) 37 | e = c.create_edge(nodes[i][j], nodes[i-1][j]) 38 | edges[nodes[i][j]].append(e) 39 | if j > 0: 40 | e = c.create_edge(nodes[i][j-1], nodes[i][j]) 41 | edges[nodes[i][j-1]].append(e) 42 | e = c.create_edge(nodes[i][j], nodes[i][j-1]) 43 | edges[nodes[i][j]].append(e) 44 | c.end_tx() 45 | 46 | # read the graph written 47 | param = client.ReadNEdgesParams() 48 | for i in range(grid_sz): 49 | for j in range(grid_sz): 50 | prog_args = [(nodes[i][j], param)] 51 | response = c.read_n_edges(prog_args) 52 | for e in response.return_edges: 53 | assert (e in edges[nodes[i][j]]) 54 | 55 | dummy = raw_input('Press a key when shards and/or timestampers are killed ') 56 | 57 | # read the graph written 58 | param = client.ReadNEdgesParams() 59 | for i in range(grid_sz): 60 | for j in range(grid_sz): 61 | prog_args = [(nodes[i][j], param)] 62 | response = c.read_n_edges(prog_args) 63 | for e in response.return_edges: 64 | assert (e in edges[nodes[i][j]]) 65 | 66 | print 'Pass ft_basic.' 67 | -------------------------------------------------------------------------------- /tests/python/correctness/line_reachability.py: -------------------------------------------------------------------------------- 1 | # 2 | # =============================================================== 3 | # Description: Line reachability program in Python. 4 | # 5 | # Created: 11/11/2013 03:17:55 PM 6 | # 7 | # Author: Ayush Dubey, dubey@cs.cornell.edu 8 | # 9 | # Copyright (C) 2013, Cornell University, see the LICENSE file 10 | # for licensing agreement 11 | # =============================================================== 12 | # 13 | 14 | import sys 15 | 16 | try: 17 | import weaver.client as client 18 | except ImportError: 19 | import client 20 | 21 | config_file='' 22 | 23 | if len(sys.argv) > 1: 24 | config_file = sys.argv[1] 25 | 26 | # creating line graph 27 | nodes = [] 28 | num_nodes = 200 29 | c = client.Client('127.0.0.1', 2002, config_file) 30 | 31 | c.begin_tx() 32 | for i in range(num_nodes): 33 | nodes.append(c.create_node()) 34 | c.end_tx() 35 | 36 | c.begin_tx() 37 | for i in range(num_nodes-1): 38 | c.create_edge(nodes[i], nodes[i+1]) 39 | c.end_tx() 40 | 41 | rp = client.ReachParams(dest=nodes[num_nodes-1]) 42 | for i in range(num_nodes): 43 | prog_args = [(nodes[i], rp)] 44 | response = c.run_reach_program(prog_args) 45 | #if i == num_nodes/10: 46 | # c.start_migration() 47 | assert response.reachable, 'From node ' + str(i) + ' to node ' + str(num_nodes-1) + ', reachable = ' + str(response.reachable) 48 | 49 | rp.dest=nodes[0] 50 | for i in range(1, num_nodes): 51 | prog_args = [(nodes[i], rp)] 52 | response = c.run_reach_program(prog_args) 53 | assert (not response.reachable), 'From node ' + str(i) + ' to node ' + str(0) + ', reachable = ' + str(response.reachable) 54 | 55 | print 'Pass line_reachability.' 56 | -------------------------------------------------------------------------------- /tests/python/correctness/multiple_del.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # 3 | # =============================================================== 4 | # Description: Create and delete multiple nodes and edges 5 | # 6 | # Author: Ayush Dubey, dubey@cs.cornell.edu 7 | # 8 | # Copyright (C) 2015, Cornell University, see the LICENSE file 9 | # for licensing agreement 10 | # =============================================================== 11 | # 12 | 13 | import sys 14 | 15 | try: 16 | import weaver.client as client 17 | except ImportError: 18 | import client 19 | 20 | config_file='' 21 | 22 | if len(sys.argv) > 1: 23 | config_file = sys.argv[1] 24 | 25 | # create client object 26 | c = client.Client('127.0.0.1', 2002, config_file) 27 | 28 | assert c.aux_index() 29 | 30 | c.begin_tx() 31 | c.create_node('ayush') 32 | c.create_node('egs') 33 | c.create_edge('ayush', 'egs', 'e1') 34 | c.create_edge('ayush', 'egs', 'e2') 35 | c.end_tx() 36 | 37 | c.begin_tx() 38 | c.delete_edge('e1') 39 | c.end_tx() 40 | 41 | c.begin_tx() 42 | c.delete_edge('e2') 43 | c.end_tx() 44 | 45 | c.begin_tx() 46 | c.create_edge('ayush', 'egs', 'e1') 47 | c.create_edge('ayush', 'egs', 'e2') 48 | c.end_tx() 49 | 50 | c.begin_tx() 51 | c.delete_edge('e1') 52 | c.delete_edge('e2') 53 | c.end_tx() 54 | 55 | c.begin_tx() 56 | c.delete_node('ayush') 57 | c.delete_node('egs') 58 | c.end_tx() 59 | -------------------------------------------------------------------------------- /tests/python/correctness/paths.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # 3 | # =============================================================== 4 | # Description: Test correctness of discover paths 5 | # 6 | # Author: Ayush Dubey, dubey@cs.cornell.edu 7 | # 8 | # Copyright (C) 2014, Cornell University, see the LICENSE file 9 | # for licensing agreement 10 | # =============================================================== 11 | # 12 | 13 | import sys 14 | 15 | try: 16 | from weaver import client 17 | except ImportError: 18 | import client 19 | 20 | config_file='' 21 | 22 | if len(sys.argv) > 1: 23 | config_file = sys.argv[1] 24 | 25 | # create client object 26 | c = client.Client('127.0.0.1', 2002, config_file) 27 | 28 | # create graph 29 | c.begin_tx() 30 | 31 | # src, dst 32 | c.create_node('src') 33 | c.create_node('dst') 34 | 35 | # layer 1 36 | c.create_node('1a') 37 | c.create_node('1b') 38 | red1 = c.create_edge('src', '1a') 39 | c.create_edge('src', '1b') 40 | 41 | # layer 2 42 | c.create_node('2a') 43 | c.create_node('2b') 44 | c.create_node('2c') 45 | c.create_node('2d') 46 | c.create_node('2e') 47 | red2 = c.create_edge('1a', '2a') 48 | c.create_edge('1a', '2b') 49 | c.create_edge('1a', '2c') 50 | c.create_edge('1b', '2d') 51 | c.create_edge('1b', '2e') 52 | 53 | # layer 3 54 | c.create_node('3') 55 | c.create_edge('2c', '3') 56 | c.create_edge('2d', '3') 57 | c.create_edge('src', '3') # src to layer3 58 | c.create_edge('3', '1a') # layer3 to layer1 for cycle 59 | 60 | # edges to dst 61 | red3 = c.create_edge('2a', 'dst') 62 | c.create_edge('2b', 'dst') 63 | c.create_edge('3', 'dst') 64 | c.create_edge('2e', 'dst') 65 | 66 | c.set_edge_property(red1, 'color', 'red') 67 | c.set_edge_property(red2, 'color', 'red') 68 | c.set_edge_property(red3, 'color', 'red') 69 | 70 | c.end_tx() 71 | 72 | # discover paths between src and dst 73 | paths = c.discover_paths('src', 'dst') 74 | print paths 75 | -------------------------------------------------------------------------------- /tests/python/correctness/read_properties.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # 3 | # =============================================================== 4 | # Description: Basic test for reading node/edge properties. 5 | # 6 | # Created: 2013-12-17 14:50:18 7 | # 8 | # Author: Gregory D. Hill, gdh39@cornell.edu 9 | # 10 | # Copyright (C) 2013, Cornell University, see the LICENSE file 11 | # for licensing agreement 12 | # =============================================================== 13 | # 14 | 15 | import sys 16 | import time 17 | 18 | try: 19 | import weaver.client as client 20 | except ImportError: 21 | import client 22 | 23 | config_file='' 24 | 25 | if len(sys.argv) > 1: 26 | config_file = sys.argv[1] 27 | 28 | coord_id = 0 29 | c = client.Client('127.0.0.1', 2002, config_file) 30 | 31 | # adding node properites 32 | c.begin_tx() 33 | node = c.create_node() 34 | c.set_node_property('color', 'blue', node) 35 | c.set_node_property('size', '27', node) 36 | c.end_tx() 37 | 38 | # reading node properites 39 | read_props = c.get_node_properties(node) 40 | assert len(read_props) == 2 41 | assert read_props['color'] == ['blue'] 42 | assert read_props['size'] == ['27'] 43 | 44 | c.begin_tx() 45 | c.set_node_property('age', '37', node) 46 | c.end_tx() 47 | 48 | read_props = c.get_node_properties(node) 49 | assert len(read_props) == 3 50 | assert read_props['age'] == ['37'] 51 | 52 | # adding edges, edge properites 53 | 54 | c.begin_tx() 55 | neighbors = [] 56 | neighbors.append(c.create_node()) 57 | neighbors.append(c.create_node()) 58 | neighbors.append(c.create_node()) 59 | edges = [] 60 | edges.append(c.create_edge(node, neighbors[0])) 61 | edges.append(c.create_edge(node, neighbors[1])) 62 | edges.append(c.create_edge(node, neighbors[2])) 63 | c.set_edge_property(edges[0], 'created', '1') 64 | c.set_edge_property(edges[1], 'created', '2') 65 | c.set_edge_property(edges[2], 'created', '3') 66 | c.set_edge_property(edges[0], 'cost', '1') 67 | c.set_edge_property(edges[1], 'cost', '2') 68 | c.set_edge_property(edges[2], 'cost', '3') 69 | c.end_tx() 70 | 71 | # reading edge properties 72 | for i in range(3): 73 | read_edge = c.get_edge(edges[i], node) 74 | assert read_edge.handle == edges[i] 75 | assert read_edge.start_node == node 76 | assert read_edge.end_node == neighbors[i] 77 | assert read_edge.properties['created'] == [str(i+1)] 78 | assert read_edge.properties['cost'] == [str(i+1)] 79 | 80 | print 'Pass read_properties.' 81 | -------------------------------------------------------------------------------- /tests/python/correctness/two_neighborhood_testing.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import time 4 | import weaver.client as client 5 | import simple_client 6 | 7 | # creating graph 8 | nodes = [None] * 5 9 | coord_id = 0 10 | c = client.Client(client._CLIENT_ID+1, coord_id) 11 | sc = simple_client.simple_client(c) 12 | 13 | tx_id = c.begin_tx() 14 | center = c.create_node(tx_id) 15 | nodes[0] = c.create_node(tx_id) 16 | nodes[1] = c.create_node(tx_id) 17 | nodes[2] = c.create_node(tx_id) 18 | c.create_edge(tx_id, center, nodes[0]) 19 | c.create_edge(tx_id, center, nodes[1]) 20 | c.create_edge(tx_id, center, nodes[2]) 21 | nodes[3] = c.create_node(tx_id) 22 | c.create_edge(tx_id, nodes[2], nodes[3]) 23 | nodes[4] = c.create_node(tx_id) 24 | c.create_edge(tx_id, nodes[0], nodes[4]) 25 | c.set_node_property(tx_id, center, 'name', 'BAD1') 26 | c.set_node_property(tx_id, nodes[0], 'name', 'BAD2') 27 | c.set_node_property(tx_id, nodes[2], 'name', 'Scarlet') 28 | c.set_node_property(tx_id, nodes[3], 'name', 'Greg') 29 | c.set_node_property(tx_id, nodes[4], 'name', 'Ayush') 30 | c.create_edge(tx_id, nodes[0], nodes[1]) 31 | c.create_edge(tx_id, nodes[1], nodes[2]) 32 | c.end_tx(tx_id) 33 | 34 | print nodes 35 | print sc.two_neighborhood(center, "name", caching=True) 36 | print sc.two_neighborhood(center, "nope", caching=True) 37 | tx_id = c.begin_tx() 38 | new_2_hop = c.create_node(tx_id) 39 | c.create_edge(tx_id, nodes[1], new_2_hop) 40 | c.set_node_property(tx_id, new_2_hop, 'name', 'Sam') 41 | assert(c.end_tx(tx_id)) 42 | print "adding two hop neighbor" 43 | print sc.two_neighborhood(center, "name", caching=True) 44 | 45 | print "adding 1 and two hop neighbors" 46 | tx_id = c.begin_tx() 47 | new_1_hop = c.create_node(tx_id) 48 | to_del = c.create_edge(tx_id, center, new_1_hop) 49 | c.set_node_property(tx_id, new_1_hop, 'name', 'Robert') 50 | 51 | new_2_hops1 = c.create_node(tx_id) 52 | new_2_hops2 = c.create_node(tx_id) 53 | c.create_edge(tx_id, new_1_hop, new_2_hops1) 54 | c.create_edge(tx_id, new_1_hop, new_2_hops2) 55 | c.set_node_property(tx_id, new_2_hops1, 'name', 'Jianeng') 56 | c.set_node_property(tx_id, new_2_hops2, 'name', 'Sarah') 57 | assert(c.end_tx(tx_id)) 58 | print sc.two_neighborhood(center, "name", caching=True) 59 | print "deleting node" 60 | 61 | tx_id = c.begin_tx() 62 | c.delete_edge(tx_id, to_del, center) 63 | c.delete_node(tx_id, new_1_hop) 64 | assert(c.end_tx(tx_id)) 65 | print sc.two_neighborhood(center, "name", caching=True) 66 | print "done!" 67 | -------------------------------------------------------------------------------- /tests/python/create_graph.py: -------------------------------------------------------------------------------- 1 | # 2 | # =============================================================== 3 | # Description: Simple graph creation test for Python API. 4 | # 5 | # Created: 11/11/2013 12:10:09 PM 6 | # 7 | # Author: Ayush Dubey, dubey@cs.cornell.edu 8 | # 9 | # Copyright (C) 2013, Cornell University, see the LICENSE file 10 | # for licensing agreement 11 | # =============================================================== 12 | # 13 | 14 | import sys 15 | 16 | import weaver.client as client 17 | import time 18 | from test_base import test_graph 19 | 20 | # creating line graph 21 | nodes = [] 22 | num_nodes = 2000 23 | num_clients = 1 24 | clients = [] 25 | for i in range(num_clients): 26 | clients.append(client.Client(client._CLIENT_ID+i, i % client._NUM_VTS)) 27 | print 'Created client' 28 | start = time.time() 29 | tg = test_graph(clients, num_nodes, 2*num_nodes) 30 | print 'Node = ' + str(tg.nodes[0]) 31 | print 'Edge to del = ' + str(tg.edge_handles[tg.nodes[0]][0]) 32 | tx_id = clients[0].begin_tx() 33 | clients[0].delete_edge(tx_id, tg.edge_handles[tg.nodes[0]][0], tg.nodes[0]) 34 | clients[0].end_tx(tx_id) 35 | end = time.time() 36 | print 'Created graph in ' + str(end-start) + ' seconds' 37 | -------------------------------------------------------------------------------- /tests/python/simple_client.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # 3 | # =============================================================== 4 | # Copyright (C) 2013, Cornell University, see the LICENSE file 5 | # for licensing agreement 6 | # =============================================================== 7 | # 8 | 9 | import sys 10 | import time 11 | 12 | import weaver.client as client 13 | 14 | class simple_client: 15 | def __init__(self, client_to_wrap): 16 | self.c= client_to_wrap 17 | 18 | def node_props(self, id, keys = []): 19 | rp = client.ReadNodePropsParams(keys = keys) 20 | prog_args = [(id, rp)] 21 | response = self.c.read_node_props(prog_args) 22 | return response.node_props 23 | 24 | def reachability(self, source, dest, edge_props = [], caching = False): 25 | rp = client.ReachParams(dest=dest, edge_props=edge_props, caching=caching) 26 | prog_args = [(source, rp)] 27 | response = self.c.run_reach_program(prog_args) 28 | return (response.reachable, response.path) 29 | 30 | def two_neighborhood(self, id, prop_key, caching = False): 31 | tp = client.TwoNeighborhoodParams(prop_key = prop_key, caching = caching) 32 | prog_args = [(id, tp)] 33 | response = self.c.run_two_neighborhood_program(prog_args) 34 | return response.responses; 35 | 36 | def clustering(self, id): 37 | cp = client.ClusteringParams(caching=False) 38 | prog_args = [(id, cp)] 39 | response = self.c.run_clustering_program(prog_args) 40 | return response.clustering_coeff; 41 | -------------------------------------------------------------------------------- /tests/python/star_cache_testing.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import weaver.client as client 4 | import time 5 | 6 | # creating star graph, tests one source many destinations with disjoint paths (aka max cache size) 7 | nodes = [] 8 | num_nodes = 2000 9 | coord_id = 0 10 | c = client.Client(client._CLIENT_ID, coord_id) 11 | 12 | tx_id = c.begin_tx() 13 | center = c.create_node(tx_id) 14 | for i in range(num_nodes): 15 | nodes.append(c.create_node(tx_id)) 16 | #print 'Created node ' + str(i) 17 | c.end_tx(tx_id) 18 | tx_id = c.begin_tx() 19 | for i in range(num_nodes): 20 | c.create_edge(tx_id, center, nodes[i]) 21 | #print 'Created edge ' + str(i) 22 | c.end_tx(tx_id) 23 | print 'Created graph' 24 | 25 | start = time.time() 26 | print('Running rechability to node: '), 27 | for i in range(num_nodes): 28 | rp = client.ReachParams(dest=nodes[i], caching=False) 29 | prog_args = [(center, rp)] 30 | response = c.run_reach_program(prog_args) 31 | print(str(i) + ','), 32 | sys.stdout.flush() 33 | assert(response.reachable) 34 | for i in range(num_nodes): 35 | rp = client.ReachParams(dest=nodes[i], caching=True) 36 | prog_args = [(center, rp)] 37 | response = c.run_reach_program(prog_args) 38 | print(str(i) + ','), 39 | sys.stdout.flush() 40 | assert(response.reachable) 41 | print 'successful' 42 | print 'Ran reachability in ' + str(time.time()-start) + ' seconds' 43 | -------------------------------------------------------------------------------- /tests/sh/clean.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # clean.sh 4 | # Copyright (C) 2014 Ayush Dubey 5 | # 6 | # See the LICENSE file for licensing agreement 7 | # 8 | 9 | echo 'Killing shard and timestamper.' 10 | pidof weaver | xargs kill -9 11 | pidof weaver-timestamper | xargs kill -9 12 | pidof weaver-shard | xargs kill -9 13 | sleep 1 14 | echo 'Killing weaver support infrastructure.' 15 | "$WEAVER_SRCDIR"/startup_scripts/kill_weaver.sh "$WEAVER_SRCDIR"/conf/weaver.yaml 16 | echo 'Done cleanup.' 17 | -------------------------------------------------------------------------------- /tests/sh/concurrent_clients.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # concurrent_clients.sh 4 | # Copyright (C) 2015 Ayush Dubey 5 | # 6 | # See the LICENSE file for licensing agreement 7 | # 8 | 9 | "$WEAVER_SRCDIR"/tests/sh/setup.sh 10 | python "$WEAVER_SRCDIR"/tests/python/correctness/concurrent_clients.py "$WEAVER_SRCDIR"/conf/weaver.yaml 11 | status=$? 12 | "$WEAVER_SRCDIR"/tests/sh/clean.sh 13 | 14 | exit $status 15 | -------------------------------------------------------------------------------- /tests/sh/empty_graph.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # empty_graph.sh 4 | # Copyright (C) 2014 Ayush Dubey 5 | # 6 | # See the LICENSE file for licensing agreement 7 | # 8 | 9 | "$WEAVER_SRCDIR"/tests/sh/setup.sh 10 | python "$WEAVER_SRCDIR"/tests/python/correctness/empty_graph_sanity_checks.py "$WEAVER_SRCDIR"/conf/weaver.yaml 11 | status=$? 12 | "$WEAVER_SRCDIR"/tests/sh/clean.sh 13 | 14 | exit $status 15 | -------------------------------------------------------------------------------- /tests/sh/env.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # env.sh 4 | # Copyright (C) 2014 Ayush Dubey 5 | # 6 | # See the LICENSE file for licensing agreement 7 | # 8 | 9 | export WEAVER_SRCDIR="$1" 10 | export WEAVER_BUILDDIR="$2" 11 | 12 | export PATH="$2"/:$PATH: 13 | export PYTHONPATH="$2"/bindings/python:"$2"/bindings/python/.libs: 14 | export LD_LIBRARY_PATH="$2"/.libs:$LD_LIBRARY_PATH: 15 | export LD_RUN_PATH="$2"/.libs:$LD_RUN_PATH: 16 | -------------------------------------------------------------------------------- /tests/sh/line_properties.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # line_properties.sh 4 | # Copyright (C) 2014 Ayush Dubey 5 | # 6 | # See the LICENSE file for licensing agreement 7 | # 8 | 9 | "$WEAVER_SRCDIR"/tests/sh/setup.sh 10 | python "$WEAVER_SRCDIR"/tests/python/correctness/line_properties.py "$WEAVER_SRCDIR"/conf/weaver.yaml 11 | status=$? 12 | "$WEAVER_SRCDIR"/tests/sh/clean.sh 13 | 14 | exit $status 15 | -------------------------------------------------------------------------------- /tests/sh/line_reachability.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # line_reachability.sh 4 | # Copyright (C) 2014 Ayush Dubey 5 | # 6 | # See the LICENSE file for licensing agreement 7 | # 8 | 9 | "$WEAVER_SRCDIR"/tests/sh/setup.sh 10 | python "$WEAVER_SRCDIR"/tests/python/correctness/line_reachability.py "$WEAVER_SRCDIR"/conf/weaver.yaml 11 | status=$? 12 | "$WEAVER_SRCDIR"/tests/sh/clean.sh 13 | 14 | exit $status 15 | -------------------------------------------------------------------------------- /tests/sh/multiple_del.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # multiple_del.sh 4 | # Copyright (C) 2015 Ayush Dubey 5 | # 6 | # See the LICENSE file for licensing agreement 7 | # 8 | 9 | "$WEAVER_SRCDIR"/tests/sh/setup.sh 10 | python "$WEAVER_SRCDIR"/tests/python/correctness/multiple_del.py "$WEAVER_SRCDIR"/conf/weaver.yaml 11 | status=$? 12 | "$WEAVER_SRCDIR"/tests/sh/clean.sh 13 | 14 | exit $status 15 | -------------------------------------------------------------------------------- /tests/sh/read_properties.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # read_properties.sh 4 | # Copyright (C) 2014 Ayush Dubey 5 | # 6 | # See the LICENSE file for licensing agreement 7 | # 8 | 9 | "$WEAVER_SRCDIR"/tests/sh/setup.sh 10 | python "$WEAVER_SRCDIR"/tests/python/correctness/read_properties.py "$WEAVER_SRCDIR"/conf/weaver.yaml 11 | status=$? 12 | "$WEAVER_SRCDIR"/tests/sh/clean.sh 13 | 14 | exit $status 15 | -------------------------------------------------------------------------------- /tests/sh/setup.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # setup.sh 4 | # Copyright (C) 2014 Ayush Dubey 5 | # 6 | # See the LICENSE file for licensing agreement 7 | # 8 | 9 | echo 'Setup weaver support infrastructure.' 10 | config_file="$WEAVER_SRCDIR"/conf/weaver.yaml 11 | WEAVER_BUILDDIR="$WEAVER_BUILDDIR" "$WEAVER_SRCDIR"/startup_scripts/kill_weaver.sh $config_file 12 | WEAVER_BUILDDIR="$WEAVER_BUILDDIR" "$WEAVER_SRCDIR"/startup_scripts/start_weaver.sh $config_file 13 | echo 'Start weaver-shard.' 14 | weaver shard --config-file=$config_file & 15 | sleep 1 16 | num_vts=$(weaver-parse-config --config-file=$config_file -c num_vts) 17 | for i in $(seq 1 $num_vts); do 18 | echo 'Start weaver-timestamper.' 19 | weaver timestamper --config-file=$config_file & 20 | done 21 | sleep 1 22 | -------------------------------------------------------------------------------- /tests/sh/simple_test.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # simple_test.sh 4 | # Copyright (C) 2014 Ayush Dubey 5 | # 6 | # See the LICENSE file for licensing agreement 7 | # 8 | 9 | "$WEAVER_SRCDIR"/tests/sh/setup.sh 10 | python "$WEAVER_SRCDIR"/tests/python/correctness/simple_test.py "$WEAVER_SRCDIR"/conf/weaver.yaml 11 | status=$? 12 | "$WEAVER_SRCDIR"/tests/sh/clean.sh 13 | 14 | exit $status 15 | -------------------------------------------------------------------------------- /tests/sh/simple_test_aux_index.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # simple_test_aux_index.sh 4 | # Copyright (C) 2014 Ayush Dubey 5 | # 6 | # See the LICENSE file for licensing agreement 7 | # 8 | 9 | "$WEAVER_SRCDIR"/tests/sh/setup.sh 10 | python "$WEAVER_SRCDIR"/tests/python/correctness/simple_test_aux_index.py "$WEAVER_SRCDIR"/conf/weaver.yaml 11 | status=$? 12 | "$WEAVER_SRCDIR"/tests/sh/clean.sh 13 | 14 | exit $status 15 | -------------------------------------------------------------------------------- /tests/sh/transactions.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # transactions.sh 4 | # Copyright (C) 2014 Ayush Dubey 5 | # 6 | # See the LICENSE file for licensing agreement 7 | # 8 | 9 | "$WEAVER_SRCDIR"/tests/sh/setup.sh 10 | python "$WEAVER_SRCDIR"/tests/python/correctness/transactions_ring.py "$WEAVER_SRCDIR"/conf/weaver.yaml 11 | status=$? 12 | "$WEAVER_SRCDIR"/tests/sh/clean.sh 13 | 14 | exit $status 15 | -------------------------------------------------------------------------------- /tests/snap_cleanup.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # 3 | # =============================================================== 4 | # Description: Tranform SNAP graph file to WEAVER graph file 5 | # 6 | # Created: 04/12/2014 01:30:15 PM 7 | # 8 | # Author: Ayush Dubey, dubey@cs.cornell.edu 9 | # 10 | # Copyright (C) 2013, Cornell University, see the LICENSE file 11 | # for licensing agreement 12 | # =============================================================== 13 | # 14 | 15 | import sys 16 | from sets import Set 17 | 18 | def snap_to_weaver(snap_file): 19 | path = snap_file.split('/') 20 | filename = path[len(path)-1] 21 | path[len(path)-1] = 'weaver' 22 | path.append(filename) 23 | weaver_file = '/'.join(path) 24 | 25 | sfile = open(snap_file, 'r') 26 | nmap = {} 27 | edges = {} 28 | handle = 0 29 | line_count = 0 30 | for line in sfile: 31 | line_count += 1 32 | if line_count % 100000 == 0: 33 | print 'read ' + str(line_count) + ' lines' 34 | if len(line) == 0 or line[0] == '#': 35 | continue 36 | split = line.split() 37 | if len(split) != 2: 38 | print 'Invalid snap file' 39 | return 40 | n1 = int(split[0]) 41 | n2 = int(split[1]) 42 | if not n1 in nmap: 43 | nmap[n1] = handle 44 | edges[handle] = Set([]) 45 | handle += 1 46 | if not n2 in nmap: 47 | nmap[n2] = handle 48 | edges[handle] = Set([]) 49 | handle += 1 50 | edges[nmap[n1]].add(nmap[n2]) 51 | sfile.close() 52 | 53 | wfile = open(weaver_file, 'w') 54 | wfile.write('#' + str(len(nmap)) + '\n') 55 | for n1 in edges: 56 | for n2 in edges[n1]: 57 | wfile.write(str(n1) + ' ' + str(n2) + '\n') 58 | wfile.close() 59 | 60 | if __name__ == '__main__': 61 | if len(sys.argv) != 2: 62 | print 'Needs to be called with only a single argument -- the absolute location of the snap graph file' 63 | else: 64 | snap_to_weaver(sys.argv[1]) 65 | -------------------------------------------------------------------------------- /tests/snap_to_weaver.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # 3 | # =============================================================== 4 | # Description: Tranform SNAP graph files so that node handles 5 | # are range(0, num_nodes). 6 | # 7 | # Created: 12/03/2013 02:17:15 PM 8 | # 9 | # Author: Ayush Dubey, dubey@cs.cornell.edu 10 | # 11 | # Copyright (C) 2013, Cornell University, see the LICENSE file 12 | # for licensing agreement 13 | # =============================================================== 14 | # 15 | 16 | import sys 17 | import random 18 | from sets import Set 19 | 20 | def snap_to_weaver(snap_file): 21 | path = snap_file.split('/') 22 | filename = path[len(path)-1] 23 | path[len(path)-1] = 'weaver' 24 | path.append(filename) 25 | path.append('_50props') 26 | weaver_file = '/'.join(path) 27 | 28 | sfile = open(snap_file, 'r') 29 | nmap = {} 30 | edges = {} 31 | handle = 0 32 | line_count = 0 33 | for line in sfile: 34 | line_count += 1 35 | if line_count % 100000 == 0: 36 | print 'read ' + str(line_count) + ' lines' 37 | if len(line) == 0 or line[0] == '#': 38 | continue 39 | split = line.split() 40 | if len(split) != 2: 41 | print 'Invalid snap file' 42 | return 43 | n1 = int(split[0]) 44 | n2 = int(split[1]) 45 | if not n1 in nmap: 46 | nmap[n1] = handle 47 | edges[handle] = Set([]) 48 | handle += 1 49 | if not n2 in nmap: 50 | nmap[n2] = handle 51 | edges[handle] = Set([]) 52 | handle += 1 53 | edges[nmap[n1]].add(nmap[n2]) 54 | sfile.close() 55 | 56 | wfile = open(weaver_file, 'w') 57 | wfile.write('#' + str(len(nmap)) + '\n') 58 | for n1 in edges: 59 | wfile.write(str(n1) + '\n') 60 | for n1 in edges: 61 | for n2 in edges[n1]: 62 | edge_line = str(n1) + ' ' + str(n2) 63 | if random.random() > 0.5: 64 | edge_line += ' color red\n' 65 | else: 66 | edge_line += '\n' 67 | wfile.write(edge_line) 68 | wfile.close() 69 | 70 | if __name__ == '__main__': 71 | if len(sys.argv) != 2: 72 | print 'Needs to be called with only a single argument -- the absolute location of the snap graph file' 73 | else: 74 | snap_to_weaver(sys.argv[1]) 75 | -------------------------------------------------------------------------------- /weaver.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * =============================================================== 3 | * Description: Weaver wrapper program that calls all other 4 | * Weaver programs. 5 | * 6 | * Created: 2014-11-25 08:55:19 7 | * 8 | * Author: Ayush Dubey, dubey@cs.cornell.edu 9 | * 10 | * Copyright (C) 2014, Cornell University, see the LICENSE file 11 | * for licensing agreement 12 | * =============================================================== 13 | */ 14 | 15 | #ifdef HAVE_CONFIG_H 16 | #include "config.h" 17 | #endif 18 | 19 | #include 20 | #include 21 | 22 | int 23 | main(int argc, const char* argv[]) 24 | { 25 | std::vector cmds; 26 | cmds.push_back(e::subcommand("timestamper", "Start a new Weaver timestamper")); 27 | cmds.push_back(e::subcommand("shard", "Start a new Weaver shard")); 28 | cmds.push_back(e::subcommand("parse-config", "Parse the Weaver configuration file")); 29 | 30 | return dispatch_to_subcommands(argc, argv, 31 | "weaver", "Weaver", 32 | PACKAGE_VERSION, 33 | "weaver-", 34 | "WEAVER_EXEC_PATH", WEAVER_EXEC_DIR, 35 | &cmds.front(), cmds.size()); 36 | } 37 | -------------------------------------------------------------------------------- /weaver.upack: -------------------------------------------------------------------------------- 1 | package weaver 2 | | user_name="Ayush Dubey" 3 | | user_email="dubey@cs.cornell.edu" 4 | | source="weaver" 5 | | section="net" 6 | | dist="Ubuntu" 7 | | version="0.2.0" 8 | | release="1" 9 | | epoch="1" 10 | | license="BSD" 11 | | copyright="2011-2014 Cornell University" 12 | | homepage="http://weaver.systems" 13 | | tarball="http://weaver.systems/weaver-{version}.tar.gz" 14 | | configure="" 15 | | summary="A distributed graph store" 16 | | build-requires="autoconf, autoconf-archive, build-essential, libtool, python-dev, libyaml-dev, libpopt-dev, libgoogle-glog-dev, cython, sparsehash, libpugixml-dev, libpo6-dev, libe-dev, libbusybee-dev, libhyperleveldb-dev, replicant, libreplicant-dev, libreplicant-state-machine-dev, libhyperdex-dev-warp, libhyperdex-client-dev, hyperdex-warp" 17 | | requires="replicant, hyperdex-warp, weaver-shard, weaver-timestamper, libweaverclient, python-weaver-client, weaver-config, libweaverservermanager, libweaverchronosd" 18 | + /etc/weaver.yaml 19 | '''{summary}''' 20 | 21 | subpackage weaver-common 22 | | summary="Weaver command" 23 | + {bindir}/weaver 24 | '''{summary}''' 25 | 26 | subpackage weaver-shard 27 | | summary="Shards for Weaver" 28 | | requires="weaver-common" 29 | + {bindir}/weaver-shard 30 | '''{summary}''' 31 | 32 | subpackage weaver-timestamper 33 | | summary="Timestampers for Weaver" 34 | | requires="weaver-common" 35 | + {bindir}/weaver-timestamper 36 | '''{summary}''' 37 | 38 | subpackage libweaverservermanager 39 | | deb name="libweaverservermanager0" 40 | | summary="Server manager for Weaver" 41 | + {libdir}/libweaverservermanager.so 42 | + {libdir}/libweaverservermanager.so.0 43 | + {libdir}/libweaverservermanager.so.0.0.0 44 | '''{summary}''' 45 | 46 | subpackage libweaverclient 47 | | deb name="libweaverclient0" 48 | | summary="C++ client for Weaver" 49 | + {includedir}/weaver 50 | + {libdir}/libweaverclient.so 51 | + {libdir}/libweaverclient.so.0 52 | + {libdir}/libweaverclient.so.0.0.0 53 | '''{summary}''' 54 | 55 | subpackage python-weaver-client 56 | | summary="Python bindings for Weaver" 57 | | deb requires="libpython2.7, libweaverclient0" 58 | + {pythondir}/weaver/client.so 59 | + {pythondir}/weaver/client.so.0 60 | + {pythondir}/weaver/client.so.0.0.0 61 | + {pythondir}/weaver/__init__.py 62 | '''{summary}''' 63 | 64 | subpackage weaver-config 65 | | summary="Parse weaver configuration" 66 | | requires="weaver-common" 67 | + {bindir}/weaver-parse-config 68 | '''{summary}''' 69 | 70 | subpackage libweaverchronosd 71 | | deb name="libweaverchronosd0" 72 | | summary="Weaver Chronos daemon" 73 | + {libdir}/libweaverchronosd.so 74 | + {libdir}/libweaverchronosd.so.0 75 | + {libdir}/libweaverchronosd.so.0.0.0 76 | '''{summary}''' 77 | -------------------------------------------------------------------------------- /weaver_alpha.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dubey/weaver/56a42fd2d0bbb14867ba792ca5461d16310a7387/weaver_alpha.png --------------------------------------------------------------------------------