├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── LICENSE ├── README.md ├── build.sh ├── cmake ├── euler_core.cmake ├── googletest.cmake ├── grpc.cmake ├── jemalloc.cmake ├── linenoise.cmake ├── utils.cmake └── zookeeper.cmake ├── doc ├── GQL-1.png ├── GQL-2.png ├── GQL-3.png ├── GQL-4.png ├── GQL-5.png ├── data-prepare-1.png ├── data-prepare-2.jpeg ├── message-algo-1.svg ├── message-algo-2.svg ├── message-algo-3.svg ├── messagepassing-1.png ├── messagepassing-2.png ├── overview-1.png └── overview-2.png ├── euler ├── CMakeLists.txt ├── __init__.py ├── client │ ├── CMakeLists.txt │ ├── client_manager.cc │ ├── client_manager.h │ ├── completion_queue_pool.h │ ├── end2end_gp_test.cc │ ├── end2end_local_test.cc │ ├── end2end_test.cc │ ├── graph.h │ ├── graph_config.cc │ ├── graph_config.h │ ├── grpc_channel.cc │ ├── grpc_channel.h │ ├── grpc_channel_test.cc │ ├── grpc_manager.cc │ ├── grpc_manager.h │ ├── impl_register.h │ ├── query.cc │ ├── query.h │ ├── query_proxy.cc │ ├── query_proxy.h │ ├── rpc_client.cc │ ├── rpc_client.h │ ├── rpc_client_end2end_test.cc │ ├── rpc_client_test.cc │ ├── rpc_manager.cc │ ├── rpc_manager.h │ ├── rpc_manager_test.cc │ └── testing │ │ ├── CMakeLists.txt │ │ ├── echo.h │ │ ├── echo.proto │ │ ├── mock_rpc_manager.cc │ │ ├── mock_rpc_manager.h │ │ └── simple_server_monitor.h ├── common │ ├── CMakeLists.txt │ ├── alias_method.cc │ ├── alias_method.h │ ├── bytes_compute.h │ ├── bytes_io.cc │ ├── bytes_io.h │ ├── bytes_io_test.cc │ ├── compact_weighted_collection.h │ ├── compact_weighted_collection_test.cc │ ├── data_types.cc │ ├── data_types.h │ ├── data_types_test.cc │ ├── env.cc │ ├── env.h │ ├── env_posix.cc │ ├── env_test.cc │ ├── error_code.h │ ├── fast_weighted_collection.h │ ├── fast_weighted_collection_test.cc │ ├── file_io.cc │ ├── file_io.h │ ├── hash.cc │ ├── hash.h │ ├── hdfs_file_io.cc │ ├── hdfs_file_io_test.cc │ ├── local_file_io.cc │ ├── local_file_io_test.cc │ ├── logging.cc │ ├── logging.h │ ├── macros.h │ ├── mutex.h │ ├── net_util.cc │ ├── net_util.h │ ├── random.cc │ ├── random.h │ ├── refcount.h │ ├── server_meta.proto │ ├── server_monitor.cc │ ├── server_monitor.h │ ├── server_register.h │ ├── signal.h │ ├── singleton.h │ ├── slice.cc │ ├── slice.h │ ├── status.cc │ ├── status.h │ ├── str_util.cc │ ├── str_util.h │ ├── str_util_test.cc │ ├── time_utils.h │ ├── timmer.cc │ ├── timmer.h │ ├── weighted_collection.h │ ├── zk_server_monitor.cc │ ├── zk_server_monitor.h │ ├── zk_server_register.cc │ ├── zk_server_register.h │ └── zk_util_cache.h ├── core │ ├── CMakeLists.txt │ ├── api │ │ ├── CMakeLists.txt │ │ ├── api.cc │ │ └── api.h │ ├── dag │ │ ├── CMakeLists.txt │ │ ├── dag.cc │ │ ├── dag.h │ │ ├── dag_test.cc │ │ ├── edge.h │ │ ├── node.cc │ │ └── node.h │ ├── dag_def │ │ ├── CMakeLists.txt │ │ ├── dag_def.cc │ │ ├── dag_def.h │ │ ├── dag_def_test.cc │ │ ├── dag_node_def.cc │ │ ├── dag_node_def.h │ │ ├── sub_graph_iso.cc │ │ └── sub_graph_iso.h │ ├── framework │ │ ├── CMakeLists.txt │ │ ├── allocator.cc │ │ ├── allocator.h │ │ ├── attr_value.h │ │ ├── dag.proto │ │ ├── dag_node.proto │ │ ├── executor.cc │ │ ├── executor.h │ │ ├── executor_test.cc │ │ ├── op_kernel.cc │ │ ├── op_kernel.h │ │ ├── tensor.cc │ │ ├── tensor.h │ │ ├── tensor.proto │ │ ├── tensor_shape.h │ │ ├── tensor_shape.proto │ │ ├── tensor_shape_test.cc │ │ ├── tensor_test.cc │ │ ├── tensor_util.cc │ │ ├── tensor_util.h │ │ ├── tensor_util_test.cc │ │ ├── thread_pool.h │ │ ├── types.h │ │ ├── types.proto │ │ ├── udf.cc │ │ └── udf.h │ ├── graph │ │ ├── CMakeLists.txt │ │ ├── edge.cc │ │ ├── edge.h │ │ ├── edge_test.cc │ │ ├── graph.cc │ │ ├── graph.h │ │ ├── graph_builder.cc │ │ ├── graph_builder.h │ │ ├── graph_meta.cc │ │ ├── graph_meta.h │ │ ├── graph_test.cc │ │ ├── local_graph_test.cc │ │ ├── node.cc │ │ ├── node.h │ │ └── node_test.cc │ ├── index │ │ ├── CMakeLists.txt │ │ ├── common_index_result.cc │ │ ├── common_index_result.h │ │ ├── common_index_result_test.cc │ │ ├── hash_index_result.h │ │ ├── hash_index_result_test.cc │ │ ├── hash_range_sample_index.h │ │ ├── hash_range_sample_index_test.cc │ │ ├── hash_sample_index.h │ │ ├── hash_sample_index_test.cc │ │ ├── index_manager.cc │ │ ├── index_manager.h │ │ ├── index_manager_test.cc │ │ ├── index_meta.cc │ │ ├── index_meta.h │ │ ├── index_meta_test.cc │ │ ├── index_result.h │ │ ├── index_types.h │ │ ├── index_util.h │ │ ├── range_index_result.h │ │ ├── range_index_result_test.cc │ │ ├── range_sample_index.h │ │ ├── range_sample_index_test.cc │ │ └── sample_index.h │ └── kernels │ │ ├── CMakeLists.txt │ │ ├── append_merge_op.cc │ │ ├── append_merge_op_test.cc │ │ ├── as_op.cc │ │ ├── broad_cast_split_op.cc │ │ ├── common.cc │ │ ├── common.h │ │ ├── data_gather_op.cc │ │ ├── data_merge_op.cc │ │ ├── data_merge_op_test.cc │ │ ├── data_row_append_merge_op.cc │ │ ├── gather_result_op.cc │ │ ├── get_adj_op.cc │ │ ├── get_edge_op.cc │ │ ├── get_edge_sum_weight_op.cc │ │ ├── get_feature_op.cc │ │ ├── get_graph_by_label_op.cc │ │ ├── get_nb_filter_op.cc │ │ ├── get_nb_filter_op_test.cc │ │ ├── get_neighbor_edge_op.cc │ │ ├── get_neighbor_op.cc │ │ ├── get_node_op.cc │ │ ├── get_node_type_op.cc │ │ ├── gp_unique_merge_op.cc │ │ ├── gp_unique_merge_op_test.cc │ │ ├── id_split_op.cc │ │ ├── id_unique_op.cc │ │ ├── idx_gather_op.cc │ │ ├── idx_merge_op.cc │ │ ├── idx_merge_op_test.cc │ │ ├── idx_row_append_merge_op.cc │ │ ├── layerwise_op_test.cc │ │ ├── local_sample_layer_op.cc │ │ ├── max_udf.cc │ │ ├── mean_udf.cc │ │ ├── min_udf.cc │ │ ├── multi_type_data_merge_op.cc │ │ ├── multi_type_idx_merge_op.cc │ │ ├── multi_type_merge_op_test.cc │ │ ├── ops_test.cc │ │ ├── post_process_op.cc │ │ ├── post_process_op_test.cc │ │ ├── regular_data_merge_op.cc │ │ ├── regular_data_merge_op_test.cc │ │ ├── remote_op.cc │ │ ├── remote_op_test.cc │ │ ├── reshape_op.cc │ │ ├── sample_edge_op.cc │ │ ├── sample_edge_split_op.cc │ │ ├── sample_graph_label_op.cc │ │ ├── sample_layer_op.cc │ │ ├── sample_n_with_types_op.cc │ │ ├── sample_n_with_types_split_op.cc │ │ ├── sample_neighbor_op.cc │ │ ├── sample_node_op.cc │ │ ├── sample_node_split_op.cc │ │ ├── sample_root_op.cc │ │ ├── sparse_gen_adj_op.cc │ │ ├── sparse_get_adj_op.cc │ │ ├── udf_test.cc │ │ └── unique_gather_test.cc ├── parser │ ├── .gitignore │ ├── CMakeLists.txt │ ├── attribute_calculator.cc │ ├── attribute_calculator.h │ ├── compiler.cc │ ├── compiler.h │ ├── compiler_test.cc │ ├── gen_node_def_input_output.cc │ ├── gen_node_def_input_output.h │ ├── gremlin.l │ ├── gremlin.y │ ├── optimize_rule.h │ ├── optimize_type.cc │ ├── optimize_type.h │ ├── optimizer.cc │ ├── optimizer.h │ ├── optimizer_test.cc │ ├── translator.cc │ ├── translator.h │ ├── translator_test.cc │ ├── tree.h │ └── tree_test.cc ├── proto │ ├── CMakeLists.txt │ ├── service.proto │ └── worker.proto ├── python │ ├── __init__.py │ ├── hash_test.py │ └── start_service.py ├── service │ ├── CMakeLists.txt │ ├── async_service_interface.h │ ├── grpc_call.h │ ├── grpc_euler_service.cc │ ├── grpc_euler_service.h │ ├── grpc_server.cc │ ├── grpc_server.h │ ├── grpc_server_test.cc │ ├── grpc_worker.cc │ ├── grpc_worker.h │ ├── grpc_worker_service.cc │ ├── grpc_worker_service.h │ ├── mock_api.cc │ ├── python_api.cc │ ├── server_interface.cc │ ├── server_interface.h │ └── worker.h ├── tools │ ├── CMakeLists.txt │ ├── __init__.py │ ├── edge.py │ ├── gen_graph_partition_test_data.sh │ ├── gen_partitioned_data.sh │ ├── generate_euler_data.py │ ├── graph_meta.py │ ├── json2meta.py │ ├── json2partdat.py │ ├── json2partindex.py │ ├── node.py │ ├── remote_console │ │ ├── CMakeLists.txt │ │ ├── remote_console.cc │ │ └── remote_console.h │ └── util.py └── util │ ├── CMakeLists.txt │ └── python_api.cc ├── euler_estimator ├── README.md ├── __init__.py └── python │ ├── __init__.py │ ├── base_estimator.py │ ├── edge_estimator.py │ ├── gae_estimator.py │ ├── graph_estimator.py │ ├── node_estimator.py │ └── sample_estimator.py ├── examples ├── TransX │ ├── README.md │ ├── run_transD.py │ ├── run_transE.py │ ├── run_transH.py │ ├── run_transR.py │ ├── transD.py │ ├── transE.py │ ├── transH.py │ ├── transR.py │ └── transX.py ├── adaptivegcn │ ├── README.md │ ├── adaptivegcn.py │ └── run_adaptivegcn.py ├── agnn │ ├── README.md │ ├── agnn.py │ └── run_agnn.py ├── appnp │ ├── README.md │ ├── appnp.py │ └── run_appnp.py ├── arma │ ├── README.md │ ├── arma.py │ └── run_arma.py ├── deepwalk │ ├── README.md │ ├── deepwalk.py │ └── run_deepwalk.py ├── dgi │ ├── README.md │ ├── dgi.py │ └── run_dgi.py ├── distmult │ ├── README.md │ ├── distmult.py │ └── run_distmult.py ├── dna │ ├── README.md │ ├── dna.py │ └── run_dna.py ├── fastgcn │ ├── README.md │ ├── fastgcn.py │ └── run_fastgcn.py ├── gae │ ├── README.md │ ├── gae.py │ ├── run_gae.py │ └── run_vgae.py ├── gat │ ├── README.md │ ├── gat.py │ └── run_gat.py ├── gated_graph │ ├── README.md │ ├── gated_graph.py │ └── run_gated_graph.py ├── gcn │ ├── README.md │ ├── gcn.py │ └── run_gcn.py ├── geniepath │ ├── README.md │ ├── geniepath.py │ └── run_geniepath.py ├── gin │ ├── README.md │ ├── gin.py │ └── run_gin.py ├── graphgcn │ ├── README.md │ ├── graphgcn.py │ └── run_graphgcn.py ├── graphsage │ ├── README.md │ ├── graphsage.py │ └── run_graphsage.py ├── lgcn │ ├── README.md │ ├── lgcn.py │ └── run_lgcn.py ├── line │ ├── README.md │ ├── line.py │ └── run_line.py ├── rgcn │ ├── README.md │ ├── rgcn.py │ └── run_rgcn.py ├── sample_solution │ ├── readme.md │ ├── sample.txt │ └── sample_solution_model.py ├── set2set │ ├── README.md │ ├── run_set2set.py │ └── set2set.py ├── sgcn │ ├── README.md │ ├── run_sgcn.py │ └── sgcn.py ├── solution │ ├── readme.md │ ├── run_solution.py │ └── solution_model.py └── tagcn │ ├── README.md │ ├── run_tagcn.py │ └── tagcn.py ├── knn └── knn.py ├── requirements.txt ├── setup.cfg ├── tf_euler ├── CMakeLists.txt ├── __init__.py ├── kernels │ ├── gather_op.cc │ ├── gen_pair_op.cc │ ├── get_binary_feature_op.cc │ ├── get_dense_feature_op.cc │ ├── get_edge_binary_feature_op.cc │ ├── get_edge_dense_feature_op.cc │ ├── get_edge_sparse_feature_op.cc │ ├── get_full_neighbor_op.cc │ ├── get_graph_by_label_op.cc │ ├── get_node_type_op.cc │ ├── get_sorted_full_neighbor_op.cc │ ├── get_sparse_feature_op.cc │ ├── get_top_k_neighbor_op.cc │ ├── inflate_idx_op.cc │ ├── random_walk_op.cc │ ├── sample_edge_op.cc │ ├── sample_fanout_op.cc │ ├── sample_fanout_with_feature_op.cc │ ├── sample_graph_label_op.cc │ ├── sample_n_with_types_op.cc │ ├── sample_neighbor_layerwise_with_adj_op.cc │ ├── sample_neighbor_op.cc │ ├── sample_node_op.cc │ ├── scatter_op.cc │ ├── sparse_gather_op.cc │ └── sparse_get_adj_op.cc ├── ops │ ├── euler_ops.cc │ ├── feature_ops.cc │ ├── mp_ops.cc │ ├── neighbor_ops.cc │ ├── sample_ops.cc │ ├── type_ops.cc │ ├── util_ops.cc │ └── walk_ops.cc ├── python │ ├── __init__.py │ ├── contrib │ │ ├── py_scatter.py │ │ └── spmm.py │ ├── convolution │ │ ├── __init__.py │ │ ├── agnn_conv.py │ │ ├── appnp_conv.py │ │ ├── arma_conv.py │ │ ├── conv.py │ │ ├── conv_test.py │ │ ├── dna_conv.py │ │ ├── gat_conv.py │ │ ├── gated_graph_conv.py │ │ ├── gcn_conv.py │ │ ├── gin_conv.py │ │ ├── graph_conv.py │ │ ├── relation_conv.py │ │ ├── sage_conv.py │ │ ├── sgcn_conv.py │ │ └── tag_conv.py │ ├── dataflow │ │ ├── __init__.py │ │ ├── base_dataflow.py │ │ ├── fast_dataflow.py │ │ ├── gcn_dataflow.py │ │ ├── layerwise_dataflow.py │ │ ├── neighbor_dataflow.py │ │ ├── relation_dataflow.py │ │ ├── sage_dataflow.py │ │ └── whole_dataflow.py │ ├── dataset │ │ ├── README.md │ │ ├── __init__.py │ │ ├── base_dataset.py │ │ ├── citeseer.py │ │ ├── cora.py │ │ ├── fb15k.py │ │ ├── fb15k237.py │ │ ├── gcn_utils.py │ │ ├── ml_1m.py │ │ ├── multigraph_util.py │ │ ├── mutag.py │ │ ├── ppi.py │ │ ├── pubmed.py │ │ ├── pubmed_utils.py │ │ ├── reddit.py │ │ ├── sage_util.py │ │ ├── test_data.py │ │ ├── utils.py │ │ └── wn18.py │ ├── euler_ops │ │ ├── __init__.py │ │ ├── base.py │ │ ├── base_test.py │ │ ├── feature_ops.py │ │ ├── feature_ops_test.py │ │ ├── mp_ops.py │ │ ├── mp_ops_test.py │ │ ├── neighbor_ops.py │ │ ├── neighbor_ops_test.py │ │ ├── sample_ops.py │ │ ├── sample_ops_test.py │ │ ├── type_ops.py │ │ ├── type_ops_test.py │ │ ├── util_ops.py │ │ ├── util_ops_test.py │ │ ├── walk_ops.py │ │ └── walk_ops_test.py │ ├── graph_pool │ │ ├── __init__.py │ │ ├── attention_pool.py │ │ ├── base_pool.py │ │ └── set2set_pool.py │ ├── mp_utils │ │ ├── __init__.py │ │ ├── base.py │ │ ├── base_gae.py │ │ ├── base_gnn.py │ │ ├── base_graph.py │ │ ├── graph_gnn.py │ │ ├── group_gnn.py │ │ └── utils.py │ ├── node_pool │ │ └── __init__.py │ ├── solution │ │ ├── __init__.py │ │ ├── base_sample.py │ │ ├── base_supervise.py │ │ ├── base_unsupervise.py │ │ ├── logits.py │ │ ├── losses.py │ │ ├── samplers.py │ │ └── utils.py │ └── utils │ │ ├── __init__.py │ │ ├── aggregators.py │ │ ├── embedding.py │ │ ├── encoders.py │ │ ├── flags.py │ │ ├── hooks.py │ │ ├── layers.py │ │ ├── metrics.py │ │ ├── optimizers.py │ │ ├── sparse_aggregators.py │ │ ├── to_dense_adj.py │ │ ├── to_dense_adj_test.py │ │ ├── to_dense_batch.py │ │ └── to_dense_batch_test.py ├── scripts │ ├── dist_tf_euler.sh │ ├── run_ppi.sh │ └── start_service.py └── utils │ ├── euler_query_proxy.h │ ├── init_query_proxy.cc │ └── sparse_tensor_builder.h ├── third_party ├── CMakeLists.txt └── hdfs │ └── hdfs.h └── tools ├── .gitignore ├── CMakeLists.txt ├── pip ├── build_wheel.sh └── setup.py ├── start_zk ├── zk_start.sh ├── zk_stop.sh └── zoo.cfg └── test_data ├── graph.json └── meta /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.so 3 | *.pyc 4 | build/ 5 | build-new/ 6 | *.pyc 7 | target 8 | .idea 9 | *.iml 10 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "third_party/grpc"] 2 | path = third_party/grpc 3 | url = https://github.com/grpc/grpc 4 | ignore = dirty 5 | [submodule "third_party/googletest"] 6 | path = third_party/googletest 7 | url = https://github.com/google/googletest.git 8 | [submodule "third_party/zookeeper"] 9 | path = third_party/zookeeper 10 | url = https://github.com/apache/zookeeper 11 | ignore = dirty 12 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export PYTHONPATH=`pwd` 4 | 5 | CMAKE_FLAGS="-DBUILD_TESTING=off" 6 | 7 | if [ "x$1" == "xtest" ] ; then 8 | CMAKE_FLAGS="${CMAKE_FLAGS} -DBUILD_COVERAGE=ON" 9 | fi 10 | 11 | cd $(dirname ${BASH_SOURCE[0]}) 12 | 13 | set -e 14 | 15 | # build zookeeper 16 | (cd third_party/zookeeper; ant compile_jute) 17 | 18 | rm -fr build && mkdir build 19 | (cd build && cmake ${CMAKE_FLAGS} .. && make -j32) 20 | 21 | rm -f tools/remote_console 22 | ln -s $(pwd)/build/euler/tools/remote_console/remote_console tools/ 23 | 24 | python tools/pip/setup.py install 25 | 26 | if [ "x$1" == "xtest" ] ; then 27 | echo "Generating test data ----" 28 | python euler/tools/generate_euler_data.py tools/test_data/graph.json /tmp/euler 2 tools/test_data/meta 29 | 30 | cd build && make test 31 | fi 32 | 33 | -------------------------------------------------------------------------------- /cmake/googletest.cmake: -------------------------------------------------------------------------------- 1 | set(GTEST_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/googletest) 2 | 3 | add_subdirectory(${GTEST_ROOT_DIR} third_party/googletest) 4 | 5 | set(GTEST_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/third_party/googletest/googletest/include" 6 | "${CMAKE_CURRENT_SOURCE_DIR}/third_party/googletest/googlemock/include") 7 | -------------------------------------------------------------------------------- /cmake/grpc.cmake: -------------------------------------------------------------------------------- 1 | set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "disable benchmark testing" FORCE) 2 | set(GRPC_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/grpc) 3 | 4 | add_subdirectory(${PROTOBUF_ROOT_DIR} third_party/grpc) 5 | 6 | # Set grpc package 7 | set(GRPC_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/grpc/include) 8 | 9 | # Set protobuf package 10 | set(PROTOBUF_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/grpc/third_party/protobuf/src/) 11 | -------------------------------------------------------------------------------- /cmake/jemalloc.cmake: -------------------------------------------------------------------------------- 1 | include (ExternalProject) 2 | 3 | set(jemalloc_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/jemalloc/src/jemalloc/include) 4 | set(jemalloc_URL https://github.com/jemalloc/jemalloc/archive/5.1.0.tar.gz) 5 | set(jemalloc_HASH SHA256=ff28aef89df724bd7b6bd6fde8597695514e0e3404d1afad2f1eb8b55ef378d3) 6 | 7 | set(jemalloc_BUILD ${CMAKE_CURRENT_BINARY_DIR}/jemalloc/) 8 | 9 | ExternalProject_Add(jemalloc 10 | PREFIX jemalloc 11 | URL ${jemalloc_URL} 12 | URL_HASH ${jemalloc_HASH} 13 | DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" 14 | BUILD_IN_SOURCE 1 15 | LOG_UPDATE 0 16 | LOG_CONFIGURE 0 17 | LOG_BUILD 0 18 | BUILD_BYPRODUCTS ${jemalloc_STATIC} ${jemalloc_STATIC_PIC} 19 | CONFIGURE_COMMAND ./autogen.sh && ./configure --disable-initial-exec-tls --prefix=${CMAKE_CURRENT_BINARY_DIR}/jemalloc 20 | BUILD_COMMAND ${MAKE} 21 | INSTALL_COMMAND ${CMAKE_COMMAND} -E echo "Skipping install step." 22 | ) 23 | 24 | ExternalProject_Get_Property(jemalloc INSTALL_DIR) 25 | 26 | add_library(jemalloc_STATIC STATIC IMPORTED) 27 | set_property(TARGET jemalloc_STATIC PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/src/jemalloc/lib/libjemalloc.a) 28 | add_dependencies(jemalloc_STATIC jemalloc) 29 | 30 | add_library(jemalloc_STATIC_PIC STATIC IMPORTED) 31 | set_property(TARGET jemalloc_STATIC_PIC PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/src/jemalloc/lib/libjemalloc_pic.a) 32 | add_dependencies(jemalloc_STATIC_PIC jemalloc) 33 | 34 | add_library(jemalloc_SHARED SHARED IMPORTED) 35 | set_property(TARGET jemalloc_SHARED PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/src/jemalloc/lib/libjemalloc.so) 36 | add_dependencies(jemalloc_SHARED jemalloc) 37 | 38 | if (!APPLE) 39 | link_libraries(-Wl,--no-as-needed) 40 | endif(!APPLE) 41 | 42 | link_libraries(dl ${jemalloc_STATIC_PIC}) 43 | -------------------------------------------------------------------------------- /cmake/linenoise.cmake: -------------------------------------------------------------------------------- 1 | include (ExternalProject) 2 | 3 | set(linenoise_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/linenoise/src) 4 | 5 | set(linenoise_URL http://github.com/antirez/linenoise/archive/1.0.tar.gz) 6 | set(linenoise_HASH SHA256=f5054a4fe120d43d85427cf58af93e56b9bb80389d507a9bec9b75531a340014) 7 | 8 | set(linenoise_BUILD ${CMAKE_CURRENT_BINARY_DIR}/linenoise/src/linenoise) 9 | 10 | ExternalProject_Add( 11 | linenoise 12 | PREFIX linenoise 13 | URL ${linenoise_URL} 14 | URL_HASH ${linenoise_HASH} 15 | DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" 16 | BUILD_IN_SOURCE 1 17 | BUILD_BYPRODUCTS linenoise_STATIC 18 | CONFIGURE_COMMAND ${CMAKE_COMMAND} -E echo "Skipping configure" 19 | BUILD_COMMAND cc -c -fPIC linenoise.c && ${CMAKE_AR} rcs liblinenoise.a linenoise.o 20 | INSTALL_COMMAND ${CMAKE_COMMAND} -E echo "Skipping install" 21 | ) 22 | 23 | ExternalProject_Get_Property(linenoise INSTALL_DIR) 24 | 25 | set(linenoise_STATIC ${linenoise_BUILD}/liblinenoise.a) 26 | -------------------------------------------------------------------------------- /cmake/zookeeper.cmake: -------------------------------------------------------------------------------- 1 | set(WANT_CPPUNIT OFF CACHE BOOL "disable test" FORCE) 2 | set(ZOOKEEPER_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/zookeeper/src/c) 3 | add_subdirectory(${ZOOKEEPER_ROOT_DIR} third_party/zookeeper) 4 | set(ZOOKEEPER_INCLUDE_DIRS ${ZOOKEEPER_ROOT_DIR}/include ${ZOOKEEPER_ROOT_DIR}/generated) 5 | -------------------------------------------------------------------------------- /doc/GQL-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/euler/c2a71faae59c1495b6dabcf6aec0acb4d93a7bb1/doc/GQL-1.png -------------------------------------------------------------------------------- /doc/GQL-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/euler/c2a71faae59c1495b6dabcf6aec0acb4d93a7bb1/doc/GQL-2.png -------------------------------------------------------------------------------- /doc/GQL-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/euler/c2a71faae59c1495b6dabcf6aec0acb4d93a7bb1/doc/GQL-3.png -------------------------------------------------------------------------------- /doc/GQL-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/euler/c2a71faae59c1495b6dabcf6aec0acb4d93a7bb1/doc/GQL-4.png -------------------------------------------------------------------------------- /doc/GQL-5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/euler/c2a71faae59c1495b6dabcf6aec0acb4d93a7bb1/doc/GQL-5.png -------------------------------------------------------------------------------- /doc/data-prepare-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/euler/c2a71faae59c1495b6dabcf6aec0acb4d93a7bb1/doc/data-prepare-1.png -------------------------------------------------------------------------------- /doc/data-prepare-2.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/euler/c2a71faae59c1495b6dabcf6aec0acb4d93a7bb1/doc/data-prepare-2.jpeg -------------------------------------------------------------------------------- /doc/message-algo-1.svg: -------------------------------------------------------------------------------- 1 | 2 | \phi()​ 3 | 8 | 13 | -------------------------------------------------------------------------------- /doc/message-algo-3.svg: -------------------------------------------------------------------------------- 1 | 2 | \gamma()​ 3 | 8 | 13 | -------------------------------------------------------------------------------- /doc/messagepassing-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/euler/c2a71faae59c1495b6dabcf6aec0acb4d93a7bb1/doc/messagepassing-1.png -------------------------------------------------------------------------------- /doc/messagepassing-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/euler/c2a71faae59c1495b6dabcf6aec0acb4d93a7bb1/doc/messagepassing-2.png -------------------------------------------------------------------------------- /doc/overview-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/euler/c2a71faae59c1495b6dabcf6aec0acb4d93a7bb1/doc/overview-1.png -------------------------------------------------------------------------------- /doc/overview-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/euler/c2a71faae59c1495b6dabcf6aec0acb4d93a7bb1/doc/overview-2.png -------------------------------------------------------------------------------- /euler/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_definitions(-Wall -Wextra) 2 | 3 | if (BUILD_COVERAGE) 4 | set(CMAKE_CXX_FLAGS "-g -O0 --coverage") 5 | set (CMAKE_EXE_LINKER_FLAGS "--coverage") 6 | endif(BUILD_COVERAGE) 7 | 8 | # Add subdirectories 9 | add_subdirectory(client) 10 | add_subdirectory(parser) 11 | add_subdirectory(core) 12 | add_subdirectory(common) 13 | add_subdirectory(proto) 14 | add_subdirectory(service) 15 | add_subdirectory(util) 16 | add_subdirectory(tools) 17 | -------------------------------------------------------------------------------- /euler/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | from euler.python.start_service import * 21 | -------------------------------------------------------------------------------- /euler/client/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(testing) 2 | 3 | add_library(client SHARED 4 | graph_config.cc 5 | grpc_channel.cc 6 | grpc_manager.cc 7 | rpc_client.cc 8 | rpc_manager.cc 9 | client_manager.cc 10 | query_proxy.cc 11 | query.cc) 12 | target_link_libraries(client common framework core compiler dag index grpc++_unsecure) 13 | 14 | add_executable(rpc_client_end2end_test rpc_client_end2end_test.cc) 15 | target_link_libraries(rpc_client_end2end_test client client_testing_util gtest gtest_main) 16 | add_test(NAME rpc_client_end2end_test COMMAND rpc_client_end2end_test) 17 | 18 | add_executable(rpc_manager_test rpc_manager_test.cc) 19 | target_link_libraries(rpc_manager_test client client_testing_util gmock gtest gtest_main) 20 | add_test(NAME rpc_manager_test COMMAND rpc_manager_test) 21 | 22 | add_executable(rpc_client_test rpc_client_test.cc) 23 | target_link_libraries(rpc_client_test client client_testing_util gmock gtest gtest_main) 24 | add_test(NAME rpc_client_test COMMAND rpc_client_test) 25 | 26 | add_executable(grpc_channel_test grpc_channel_test.cc) 27 | target_link_libraries(grpc_channel_test client client_testing_util gtest gtest_main) 28 | add_test(NAME grpc_channel_test COMMAND grpc_channel_test) 29 | 30 | add_executable(end2end_test end2end_test.cc) 31 | target_link_libraries(end2end_test client ops compiler dag gtest gtest_main grpc++_unsecure dag_def service api) 32 | add_test(NAME end2end_test COMMAND end2end_test) 33 | 34 | add_executable(end2end_gp_test end2end_gp_test.cc) 35 | target_link_libraries(end2end_gp_test client ops compiler dag gtest gtest_main grpc++_unsecure dag_def service api) 36 | 37 | add_executable(end2end_local_test end2end_local_test.cc) 38 | target_link_libraries(end2end_local_test gtest_main euler_core) 39 | add_test(NAME end2end_local_test COMMAND end2end_local_test) 40 | -------------------------------------------------------------------------------- /euler/client/graph_config.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_CLIENT_GRAPH_CONFIG_H_ 17 | #define EULER_CLIENT_GRAPH_CONFIG_H_ 18 | 19 | #include 20 | #include 21 | 22 | namespace euler { 23 | 24 | class GraphConfig { 25 | public: 26 | GraphConfig(); 27 | 28 | bool Load(const std::string& filename); 29 | 30 | bool Get(const std::string& key, std::string* value) const; 31 | bool Get(const std::string& key, int* value) const; 32 | void Add(const std::string& key, const std::string& value); 33 | void Add(const std::string& key, int value); 34 | void Remove(const std::string& key); 35 | 36 | std::string DebugString() const; 37 | 38 | private: 39 | std::map config_; 40 | }; 41 | 42 | } // namespace euler 43 | 44 | #endif // EULER_CLIENT_GRAPH_CONFIG_H_ 45 | -------------------------------------------------------------------------------- /euler/client/grpc_channel.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_CLIENT_GRPC_CHANNEL_H_ 17 | #define EULER_CLIENT_GRPC_CHANNEL_H_ 18 | 19 | #include 20 | #include 21 | 22 | #include "grpcpp/generic/generic_stub.h" 23 | #include "grpcpp/grpcpp.h" 24 | #include "euler/client/rpc_manager.h" 25 | 26 | namespace euler { 27 | 28 | struct GrpcContext: public RpcContext { 29 | GrpcContext(const std::string &method, 30 | google::protobuf::Message *respone, 31 | std::function done) 32 | : RpcContext(method, respone, done) { } 33 | 34 | bool Initialize(const google::protobuf::Message &request); 35 | 36 | grpc::ByteBuffer request_buf; 37 | grpc::ByteBuffer response_buf; 38 | grpc::Status status; 39 | std::unique_ptr context; 40 | std::unique_ptr response_reader; 41 | }; 42 | 43 | class GrpcChannel: public RpcChannel { 44 | public: 45 | GrpcChannel(const std::string& host_port, 46 | std::shared_ptr raw_channel); 47 | 48 | void IssueRpcCall(RpcContext *ctx) override; 49 | 50 | private: 51 | grpc::GenericStub stub_; 52 | grpc::CompletionQueue *cq_; 53 | }; 54 | 55 | } // namespace euler 56 | 57 | #endif // EULER_CLIENT_GRPC_CHANNEL_H_ 58 | -------------------------------------------------------------------------------- /euler/client/grpc_manager.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "euler/client/grpc_manager.h" 17 | 18 | #include 19 | #include 20 | 21 | #include "euler/client/grpc_channel.h" 22 | 23 | namespace euler { 24 | 25 | std::unique_ptr GrpcManager::CreateChannel( 26 | const std::string &host_port, int tag) { 27 | grpc::ChannelArguments args; 28 | args.SetMaxReceiveMessageSize(-1); 29 | args.SetInt("tag", tag); 30 | std::shared_ptr raw_channel = 31 | grpc::CreateCustomChannel(host_port, 32 | grpc::InsecureChannelCredentials(), 33 | args); 34 | return std::unique_ptr(new GrpcChannel(host_port, raw_channel)); 35 | } 36 | 37 | RpcContext *GrpcManager::CreateContext( 38 | const std::string &method, google::protobuf::Message *respone, 39 | std::function done) { 40 | return new GrpcContext(method, respone, done); 41 | } 42 | 43 | } // namespace euler 44 | -------------------------------------------------------------------------------- /euler/client/grpc_manager.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_CLIENT_GRPC_MANAGER_H_ 17 | #define EULER_CLIENT_GRPC_MANAGER_H_ 18 | 19 | #include 20 | #include 21 | 22 | #include "euler/client/rpc_manager.h" 23 | 24 | namespace euler { 25 | 26 | class GrpcManager : public RpcManager { 27 | public: 28 | GrpcManager() : RpcManager() { } 29 | 30 | std::unique_ptr CreateChannel( 31 | const std::string &host_port, int tag) override; 32 | 33 | RpcContext *CreateContext( 34 | const std::string &method, 35 | google::protobuf::Message *respone, 36 | std::function done) override; 37 | }; 38 | 39 | } // namespace euler 40 | 41 | #endif // EULER_CLIENT_GRPC_MANAGER_H_ 42 | -------------------------------------------------------------------------------- /euler/client/testing/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | protobuf_generate_grpc_cpp(echo.proto) 2 | 3 | add_library(client_testing_util echo.pb.cc echo.grpc.pb.cc mock_rpc_manager.cc) 4 | target_link_libraries(client_testing_util libprotobuf grpc++_unsecure) 5 | target_compile_options(client_testing_util PRIVATE -Wno-unused-parameter -Wno-inconsistent-missing-override) 6 | -------------------------------------------------------------------------------- /euler/client/testing/echo.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package euler.testing; 4 | 5 | message EchoRequest { 6 | string message = 1; 7 | } 8 | 9 | message EchoResponse { 10 | string message = 1; 11 | } 12 | 13 | service EchoService { 14 | rpc Echo(EchoRequest) returns (EchoResponse); 15 | } 16 | -------------------------------------------------------------------------------- /euler/client/testing/mock_rpc_manager.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "euler/client/testing/mock_rpc_manager.h" 17 | #include 18 | #include 19 | 20 | namespace euler { 21 | namespace testing { 22 | 23 | std::vector MockRpcChannel::channels; 24 | 25 | } // namespace testing 26 | } // namespace euler 27 | -------------------------------------------------------------------------------- /euler/common/alias_method.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_COMMON_ALIAS_METHOD_H_ 17 | #define EULER_COMMON_ALIAS_METHOD_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "euler/common/random.h" 24 | 25 | namespace euler { 26 | namespace common { 27 | 28 | class AliasMethod { 29 | public: 30 | void Init(const std::vector& weights); 31 | 32 | int64_t Next() const; 33 | 34 | size_t GetSize() const; 35 | 36 | std::string ShowData() const; 37 | 38 | private: 39 | std::vector prob_; 40 | std::vector alias_; 41 | int64_t NextLong(int64_t n) const; 42 | }; 43 | 44 | } // namespace common 45 | } // namespace euler 46 | 47 | #endif // EULER_COMMON_ALIAS_METHOD_H_ 48 | -------------------------------------------------------------------------------- /euler/common/bytes_compute.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_COMMON_BYTES_COMPUTE_H_ 17 | #define EULER_COMMON_BYTES_COMPUTE_H_ 18 | 19 | #include 20 | #include 21 | 22 | namespace euler { 23 | 24 | template 25 | inline uint32_t BytesSize(const T&) { 26 | return sizeof(T); 27 | } 28 | 29 | template<> 30 | inline uint32_t BytesSize(const std::string& s) { 31 | return sizeof(uint32_t) + s.size(); 32 | } 33 | 34 | template 35 | inline uint32_t BytesSize(const std::vector& v) { 36 | return sizeof(T)* v.size() + sizeof(uint32_t); 37 | } 38 | 39 | template 40 | inline uint32_t BytesSize(const std::vector>& v) { 41 | uint32_t num = v.size(); 42 | uint32_t total = sizeof(num); 43 | for (uint32_t i = 0; i < num; ++i) { 44 | total += BytesSize(v[i]); 45 | } 46 | return total; 47 | } 48 | 49 | template<> 50 | inline uint32_t BytesSize(const std::vector& v) { 51 | uint32_t len = sizeof(uint32_t); 52 | for (size_t i = 0; i < v.size(); ++i) { 53 | len += BytesSize(v[i]); 54 | } 55 | return len; 56 | } 57 | 58 | } // namespace euler 59 | 60 | #endif // EULER_COMMON_BYTES_COMPUTE_H_ 61 | -------------------------------------------------------------------------------- /euler/common/bytes_io.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "euler/common/bytes_io.h" 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | namespace euler { 23 | 24 | bool BytesReader::Read(std::string* result) { 25 | uint32_t length = 0; 26 | if (!Read(&length)) { return false; } 27 | 28 | if (total_size_ < begin_idx_ + length) { 29 | return false; 30 | } 31 | 32 | result->resize(length); 33 | std::copy(bytes_ + begin_idx_, bytes_ + begin_idx_ + length, 34 | result->begin()); 35 | begin_idx_ += length; 36 | return true; 37 | } 38 | 39 | 40 | bool BytesWriter::Write(const void* data, size_t size) { 41 | buffer_.append(reinterpret_cast(data), size); 42 | return true; 43 | } 44 | 45 | } // namespace euler 46 | -------------------------------------------------------------------------------- /euler/common/data_types.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "euler/common/data_types.h" 17 | 18 | namespace euler { 19 | namespace common { 20 | 21 | const float DEFAULT_FLOAT = 0; 22 | 23 | const int32_t DEFAULT_INT32 = std::numeric_limits::lowest(); 24 | 25 | const int64_t DEFAULT_INT64 = std::numeric_limits::lowest(); 26 | 27 | const uint64_t DEFAULT_UINT64 = std::numeric_limits::lowest(); 28 | 29 | const char DEFAULT_CHAR = std::numeric_limits::lowest(); 30 | } // namespace common 31 | } // namespace euler 32 | -------------------------------------------------------------------------------- /euler/common/data_types_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include 17 | #include 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "gtest/gtest.h" 25 | 26 | #include "euler/common/data_types.h" 27 | 28 | namespace euler { 29 | namespace common { 30 | 31 | TEST(DataTypes, EdgeIDHashFuncTest) { 32 | EdgeIDHashFunc f; 33 | EdgeID eid = std::make_tuple(1, 2, 1); 34 | std::cout << std::hex << f(eid) << std::endl; 35 | } 36 | } // namespace common 37 | } // namespace euler 38 | 39 | -------------------------------------------------------------------------------- /euler/common/env.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "euler/common/env.h" 17 | 18 | #include 19 | 20 | #include "euler/common/file_io.h" 21 | #include "euler/common/str_util.h" 22 | 23 | namespace euler { 24 | 25 | Status Env::NewFileIO(Slice uri, bool readonly, 26 | std::unique_ptr* file_io) { 27 | Slice scheme, host, path; 28 | ParseURI(uri, &scheme, &host, &path); 29 | 30 | RETURN_IF_ERROR(CreateFileIO(scheme.ToString(), file_io)); 31 | FileIO::ConfigMap config; 32 | config["scheme"] = std::string(scheme); 33 | config["namenode"] = std::string(host); 34 | config["path"] = std::string(path); 35 | config["read"] = readonly ? "true" : "false"; 36 | 37 | if (!(*file_io)->Initialize(config)) { 38 | return Status::Internal("Initialize file system failed, uri: ", uri); 39 | } 40 | 41 | return Status::OK(); 42 | } 43 | 44 | } // namespace euler 45 | -------------------------------------------------------------------------------- /euler/common/env_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "euler/common/env.h" 17 | 18 | #include 19 | 20 | #include "gtest/gtest.h" 21 | 22 | #include "euler/common/logging.h" 23 | 24 | namespace euler { 25 | 26 | TEST(EnvTest, ThreadPool) { 27 | std::unique_ptr threadpool( 28 | Env::Default()->StartThreadPool("test", 4)); 29 | threadpool->Schedule( 30 | [] () { 31 | for (int i = 0; i < 10; ++i) { 32 | EULER_LOG(INFO) << "Info: " << i; 33 | }}); 34 | threadpool->Schedule( 35 | [] () { 36 | for (int i = 0; i < 10; ++i) { 37 | EULER_LOG(INFO) << "Info: " << i * 2; 38 | }}); 39 | threadpool->Schedule( 40 | [] () { 41 | for (int i = 0; i < 10; ++i) { 42 | EULER_LOG(INFO) << "Info: " << i * 3; 43 | }}); 44 | threadpool->Shutdown(); 45 | } 46 | 47 | } // namespace euler 48 | -------------------------------------------------------------------------------- /euler/common/fast_weighted_collection_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "euler/common/fast_weighted_collection.h" 17 | 18 | #include 19 | 20 | #include "gtest/gtest.h" 21 | 22 | namespace euler { 23 | namespace common { 24 | 25 | TEST(FastWeightedCollectionTest, Init) { 26 | FastWeightedCollection fwc; 27 | std::vector ids = {0, 1, 2, 3, 4}; 28 | std::vector weights = {2.0, 4.0, 8.0, 8.0, 16.0}; 29 | fwc.Init(ids, weights); 30 | // test sampling 31 | int32_t cnts[5] = {0}; 32 | for (size_t i = 0; i < 100000; ++i) { 33 | std::pair r = fwc.Sample(); 34 | ++cnts[r.first]; 35 | } 36 | std::cout << "distribution should be 1:2:4:4:8" << std::endl; 37 | std::cout << "distribution:" << std::endl; 38 | for (size_t i = 0; i < 5; ++i) { 39 | std::cout << cnts[i] << " "; 40 | } 41 | std::cout << std::endl; 42 | // test iteration 43 | size_t size = fwc.GetSize(); 44 | for (size_t i = 0; i < size; ++i) { 45 | std::pair id_weight_pair = fwc.Get(i); 46 | ASSERT_EQ(id_weight_pair.first, ids[i]); 47 | ASSERT_EQ(id_weight_pair.second, weights[i]); 48 | } 49 | } // TEST 50 | 51 | } // namespace common 52 | } // namespace euler 53 | -------------------------------------------------------------------------------- /euler/common/file_io.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "euler/common/file_io.h" 17 | 18 | #include 19 | #include 20 | 21 | #include "euler/common/logging.h" 22 | 23 | namespace euler { 24 | 25 | typedef std::unordered_map FileIORegistry; 27 | 28 | static FileIORegistry& GlobalFileIORegistry() { 29 | static FileIORegistry registry; 30 | return registry; 31 | } 32 | 33 | void FileIORegistrar::Register(const std::string& scheme, Factory&& factory) { 34 | if (!GlobalFileIORegistry().insert({scheme, factory}).second) { 35 | EULER_LOG(ERROR) << "Register File IO for scheme: " << scheme << " failed!"; 36 | } 37 | } 38 | 39 | Status CreateFileIO(const std::string& scheme, std::unique_ptr* out) { 40 | auto& registry = GlobalFileIORegistry(); 41 | auto it = registry.find(scheme); 42 | if (it == registry.end()) { 43 | return Status::NotFound("No FileIO for scheme: ", scheme); 44 | } 45 | 46 | std::unique_ptr io((*it->second)()); 47 | *out = std::move(io); 48 | return Status::OK(); 49 | } 50 | 51 | } // namespace euler 52 | -------------------------------------------------------------------------------- /euler/common/hash.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_COMMON_HASH_H_ 17 | #define EULER_COMMON_HASH_H_ 18 | 19 | #include 20 | 21 | namespace euler { 22 | 23 | // MurmurHash3 hash algorithms 24 | 25 | // 32 bit hash 26 | void hash32(void const *data, int size, uint32_t *hash, uint32_t seed = 0); 27 | 28 | // 32 bit hash 29 | inline uint32_t hash32(void const *data, int size, uint32_t seed = 0) { 30 | uint32_t hash; 31 | hash32(data, size, &hash, seed); 32 | return hash; 33 | } 34 | 35 | 36 | // 128 bit hash 37 | void hash128(void const *data, int size, uint64_t *hash1, uint64_t *hash2, 38 | uint32_t seed = 0); 39 | 40 | // 64 bit hash 41 | inline void hash64(void const *data, int size, uint64_t *hash, 42 | uint32_t seed = 0) { 43 | uint64_t dummy; 44 | return hash128(data, size, hash, &dummy, seed); 45 | } 46 | 47 | // 64 bit hash 48 | inline uint64_t hash64(void const *data, int size, uint32_t seed = 0) { 49 | uint64_t hash; 50 | hash64(data, size, &hash, seed); 51 | return hash; 52 | } 53 | 54 | } // namespace euler 55 | 56 | #endif // EULER_COMMON_HASH_H_ 57 | -------------------------------------------------------------------------------- /euler/common/macros.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_COMMON_MACROS_H_ 17 | #define EULER_COMMON_MACROS_H_ 18 | 19 | #define POSIX 1 20 | 21 | #define DISALLOW_COPY_AND_ASSIGN(TypeName) \ 22 | TypeName(const TypeName&) = delete; \ 23 | void operator=(const TypeName&) = delete 24 | 25 | #ifdef __has_builtin 26 | #define HAS_BUILTIN(x) __has_builtin(x) 27 | #else 28 | #define HAS_BUILTIN(x) 0 29 | #endif 30 | 31 | #if (!defined(__NVCC__)) && \ 32 | (HAS_BUILTIN(__builtin_expect) || (defined(__GNUC__) && __GNUC__ >= 3)) 33 | #define PREDICT_FALSE(x) (__builtin_expect(x, 0)) 34 | #define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1)) 35 | #else 36 | #define PREDICT_FALSE(x) (x) 37 | #define PREDICT_TRUE(x) (x) 38 | #endif 39 | 40 | #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L || \ 41 | (defined(_MSC_VER) && _MSC_VER >= 1900) 42 | #define LANG_CXX11 1 43 | #endif 44 | 45 | #endif // EULER_COMMON_MACROS_H_ 46 | -------------------------------------------------------------------------------- /euler/common/net_util.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_COMMON_NET_UTIL_H_ 17 | #define EULER_COMMON_NET_UTIL_H_ 18 | 19 | #include 20 | 21 | #include 22 | 23 | namespace euler { 24 | 25 | std::string GetIP(); 26 | 27 | uint16_t GetFreePort(); 28 | 29 | } // namespace euler 30 | 31 | #endif // EULER_COMMON_NET_UTIL_H_ 32 | -------------------------------------------------------------------------------- /euler/common/random.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "euler/common/random.h" 17 | 18 | namespace euler { 19 | namespace common { 20 | 21 | namespace { 22 | static thread_local std::default_random_engine e(time(0)); 23 | static thread_local std::uniform_real_distribution u(0., 1.); 24 | } // namespace 25 | 26 | double ThreadLocalRandom() { 27 | return u(e); 28 | } 29 | 30 | } // namespace common 31 | } // namespace euler 32 | -------------------------------------------------------------------------------- /euler/common/random.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_COMMON_RANDOM_H_ 17 | #define EULER_COMMON_RANDOM_H_ 18 | 19 | #include 20 | 21 | namespace euler { 22 | namespace common { 23 | 24 | double ThreadLocalRandom(); 25 | 26 | } // namespace common 27 | } // namespace euler 28 | 29 | #endif // EULER_COMMON_RANDOM_H_ 30 | -------------------------------------------------------------------------------- /euler/common/server_meta.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package euler; 4 | 5 | option cc_enable_arenas = true; 6 | option java_outer_classname = "ServerMeta"; 7 | option java_multiple_files = true; 8 | option java_package = "com.alibaba.euler"; 9 | option go_package = "github.com/alibaba/euler/euler/go/proto"; 10 | 11 | message MetaItem { 12 | int32 type = 1; // 0 for root, 1 for shard 13 | bytes key = 2; 14 | bytes value = 3; 15 | } 16 | 17 | message ServerMeta { 18 | repeated MetaItem items = 1; 19 | } -------------------------------------------------------------------------------- /euler/common/server_register.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_COMMON_SERVER_REGISTER_H_ 17 | #define EULER_COMMON_SERVER_REGISTER_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | namespace euler { 24 | 25 | using Meta = std::unordered_map; 26 | using Server = std::string; 27 | 28 | class ServerRegister { 29 | public: 30 | virtual bool Initialize() = 0; 31 | virtual ~ServerRegister() = default; 32 | 33 | virtual bool RegisterShard(size_t shard_index, const Server &server, 34 | const Meta &meta, const Meta &shard_meta) = 0; 35 | virtual bool DeregisterShard(size_t shard_index, const Server &server) = 0; 36 | }; 37 | 38 | std::shared_ptr GetServerRegister(const std::string &zk_addr, 39 | const std::string &zk_path); 40 | 41 | } // namespace euler 42 | 43 | #endif // EULER_COMMON_SERVER_REGISTER_H_ 44 | -------------------------------------------------------------------------------- /euler/common/singleton.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | 17 | #ifndef EULER_COMMON_SINGLETON_H_ 18 | #define EULER_COMMON_SINGLETON_H_ 19 | 20 | #include 21 | #include 22 | 23 | #include "euler/common/mutex.h" 24 | 25 | namespace euler { 26 | 27 | template 28 | class SingletonBase { 29 | public: 30 | friend class std::unique_ptr; 31 | 32 | static T *Get() { 33 | return Instance(); 34 | } 35 | 36 | template 37 | static T *Instance(Args &&... args) { 38 | if (instance_ == nullptr) { 39 | MutexLock lock(&mu_); 40 | if (instance_ == nullptr) { 41 | instance_.reset(new T(std::forward(args)...)); 42 | } 43 | } 44 | return instance_.get(); 45 | } 46 | 47 | protected: 48 | SingletonBase() {} 49 | virtual ~SingletonBase() {} 50 | 51 | private: 52 | static Mutex mu_; 53 | static std::unique_ptr instance_; 54 | }; 55 | 56 | template Mutex SingletonBase::mu_; 57 | template std::unique_ptr SingletonBase::instance_; 58 | 59 | template 60 | class Singleton : public SingletonBase{ }; 61 | 62 | } // namespace euler 63 | 64 | #endif // EULER_COMMON_SINGLETON_H_ 65 | -------------------------------------------------------------------------------- /euler/common/slice.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "euler/common/slice.h" 17 | 18 | #include 19 | #include 20 | 21 | namespace euler { 22 | 23 | std::ostream& operator<<(std::ostream& o, Slice slice) { 24 | o.write(slice.data(), slice.size()); 25 | return o; 26 | } 27 | 28 | size_t Slice::find(Slice pattern, size_t pos) { 29 | if (pos >= size_) { 30 | return npos; 31 | } 32 | 33 | // TODO(xingwo): refine this 34 | std::string s(*this); 35 | return s.find(std::string(pattern), pos); 36 | } 37 | 38 | size_t Slice::find(char c, size_t pos) const { 39 | if (pos >= size_) { 40 | return npos; 41 | } 42 | const char* result = 43 | reinterpret_cast(memchr(data_ + pos, c, size_ - pos)); 44 | return result != nullptr ? result - data_ : npos; 45 | } 46 | 47 | size_t Slice::rfind(char c, size_t pos) const { 48 | if (size_ == 0) return npos; 49 | for (const char* p = data_ + std::min(pos, size_ - 1); p >= data_; p--) { 50 | if (*p == c) { 51 | return p - data_; 52 | } 53 | } 54 | return npos; 55 | } 56 | 57 | Slice Slice::substr(size_t pos, size_t n) const { 58 | if (pos > size_) pos = size_; 59 | if (n > size_ - pos) n = size_ - pos; 60 | return Slice(data_ + pos, n); 61 | } 62 | 63 | } // namespace euler 64 | -------------------------------------------------------------------------------- /euler/common/status.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "euler/common/status.h" 17 | 18 | #include 19 | #include 20 | 21 | namespace euler { 22 | 23 | Status::Status(): code_(ErrorCode::OK) { 24 | } 25 | 26 | Status::Status(ErrorCode code, const Slice& message) 27 | : code_(code), message_(message.data(), message.size()) { 28 | } 29 | 30 | std::string Status::DebugString() const { 31 | if (ok()) { 32 | return "OK"; 33 | } 34 | 35 | std::string result(kCodeToMsg[code_]); 36 | result += ": "; 37 | result += message_; 38 | return result; 39 | } 40 | 41 | std::ostream& operator<<(std::ostream& os, const Status& s) { 42 | os << s.DebugString(); 43 | return os; 44 | } 45 | 46 | } // namespace euler 47 | -------------------------------------------------------------------------------- /euler/common/str_util_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "euler/common/str_util.h" 17 | 18 | #include 19 | 20 | #include "gtest/gtest.h" 21 | 22 | namespace euler { 23 | 24 | TEST(StringUtil, TestAll) { 25 | std::string filename = "euler_123.dat"; 26 | auto vec = Split(filename, "_."); 27 | ASSERT_EQ(3u, vec.size()); 28 | ASSERT_EQ("euler", vec[0]); 29 | ASSERT_EQ("123", vec[1]); 30 | ASSERT_EQ("dat", vec[2]); 31 | } 32 | 33 | } // namespace euler 34 | -------------------------------------------------------------------------------- /euler/common/time_utils.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | 17 | #ifndef EULER_COMMON_TIME_UTILS_H_ 18 | #define EULER_COMMON_TIME_UTILS_H_ 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | namespace euler { 25 | 26 | class TimeUtils { 27 | public: 28 | static uint64_t NowMicros() { 29 | struct timeval tv; 30 | gettimeofday(&tv, NULL); 31 | return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; 32 | } 33 | }; 34 | 35 | } // namespace euler 36 | 37 | #endif // EULER_COMMON_TIME_UTILS_H_ 38 | -------------------------------------------------------------------------------- /euler/common/timmer.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "euler/common/timmer.h" 17 | 18 | namespace euler { 19 | namespace common { 20 | 21 | thread_local struct timeval start; 22 | thread_local struct timeval end; 23 | 24 | void TimmerBegin() { 25 | gettimeofday(&start, NULL); 26 | } 27 | 28 | uint64_t GetTimmerInterval() { 29 | gettimeofday(&end, NULL); 30 | uint64_t diff = 1000000 * (end.tv_sec - start.tv_sec) 31 | + end.tv_usec - start.tv_usec; 32 | return diff; 33 | } 34 | 35 | } // namespace common 36 | } // namespace euler 37 | -------------------------------------------------------------------------------- /euler/common/timmer.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_COMMON_TIMMER_H_ 17 | #define EULER_COMMON_TIMMER_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | namespace euler { 24 | namespace common { 25 | 26 | void TimmerBegin(); 27 | uint64_t GetTimmerInterval(); 28 | 29 | } // namespace common 30 | } // namespace euler 31 | 32 | #endif // EULER_COMMON_TIMMER_H_ 33 | -------------------------------------------------------------------------------- /euler/common/weighted_collection.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_COMMON_WEIGHTED_COLLECTION_H_ 17 | #define EULER_COMMON_WEIGHTED_COLLECTION_H_ 18 | 19 | #include 20 | 21 | #include 22 | #include 23 | 24 | namespace euler { 25 | namespace common { 26 | 27 | template 28 | class WeightedCollection { 29 | public: 30 | virtual ~WeightedCollection() {} 31 | virtual bool Init(const std::vector& ids, 32 | const std::vector& weights) = 0; 33 | virtual bool Init( 34 | const std::vector >& id_weight_pairs) = 0; 35 | virtual std::pair Sample() const = 0; 36 | virtual size_t GetSize() const = 0; 37 | virtual std::pair Get(size_t idx) const = 0; 38 | virtual float GetSumWeight() const = 0; 39 | }; 40 | 41 | } // namespace common 42 | } // namespace euler 43 | 44 | #endif // EULER_COMMON_WEIGHTED_COLLECTION_H_ 45 | -------------------------------------------------------------------------------- /euler/core/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_definitions(-Wall -Wextra) 2 | 3 | if (BUILD_COVERAGE) 4 | set(CMAKE_CXX_FLAGS "-g -O0 --coverage") 5 | set (CMAKE_EXE_LINKER_FLAGS "--coverage") 6 | endif(BUILD_COVERAGE) 7 | 8 | add_subdirectory(framework) 9 | add_subdirectory(kernels) 10 | add_subdirectory(dag) 11 | add_subdirectory(dag_def) 12 | add_subdirectory(index) 13 | add_subdirectory(graph) 14 | add_subdirectory(api) 15 | -------------------------------------------------------------------------------- /euler/core/api/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(api SHARED api.cc) 2 | target_link_libraries(api index framework core) 3 | 4 | -------------------------------------------------------------------------------- /euler/core/dag/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(dag dag.cc node.cc) 2 | target_link_libraries(dag common framework) 3 | 4 | add_executable(dag_test dag_test.cc) 5 | target_link_libraries(dag_test ${CMAKE_THREAD_LIBS_INIT} gtest gtest_main dag) 6 | add_test(NAME dag_test COMMAND dag_test) 7 | -------------------------------------------------------------------------------- /euler/core/dag/dag.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | 17 | #ifndef EULER_CORE_DAG_DAG_H_ 18 | #define EULER_CORE_DAG_DAG_H_ 19 | 20 | #include 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #include "euler/common/macros.h" 28 | #include "euler/common/status.h" 29 | 30 | namespace euler { 31 | 32 | class DAGNode; 33 | class DAGEdge; 34 | class DAGNodeProto; 35 | class DAGProto; 36 | 37 | class DAG { 38 | public: 39 | explicit DAG(const std::string& name); 40 | ~DAG(); 41 | 42 | std::string name() { return name_; } 43 | 44 | int num_nodes() const { return nodes_.size(); } 45 | int num_edges() const { return edges_.size(); } 46 | 47 | std::vector nodes() { return nodes_; } 48 | std::vector edges() { return edges_; } 49 | 50 | DAGNode* AddNode(const DAGNodeProto& node_def, Status* s); 51 | DAGEdge* AddEdge(DAGNode* src, int x, DAGNode* dst, int y); 52 | 53 | static std::unique_ptr NewFromProto(const DAGProto& dag_def); 54 | 55 | private: 56 | std::string name_; 57 | std::vector nodes_; 58 | std::vector edges_; 59 | 60 | DISALLOW_COPY_AND_ASSIGN(DAG); 61 | }; 62 | 63 | } // namespace euler 64 | 65 | #endif // EULER_CORE_DAG_DAG_H_ 66 | -------------------------------------------------------------------------------- /euler/core/dag/dag_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include "gtest/gtest.h" 23 | 24 | #include "euler/core/dag/dag.h" 25 | 26 | namespace euler { 27 | 28 | TEST(DAG, Construction) { 29 | } 30 | 31 | TEST(DAG, Query) { 32 | } 33 | 34 | } // namespace euler 35 | -------------------------------------------------------------------------------- /euler/core/dag/edge.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | 17 | #ifndef EULER_CORE_DAG_EDGE_H_ 18 | #define EULER_CORE_DAG_EDGE_H_ 19 | 20 | #include 21 | 22 | namespace euler { 23 | 24 | class DAGNode; 25 | 26 | class DAGEdge { 27 | public: 28 | int id() { return id_; } 29 | 30 | DAGNode* src() const { return src_; } 31 | DAGNode* dst() const { return dst_; } 32 | 33 | int src_output() { return src_output_; } 34 | int dst_input() { return dst_input_; } 35 | 36 | private: 37 | friend class DAG; 38 | DAGEdge() { } 39 | 40 | int id_; 41 | DAGNode* src_; 42 | DAGNode* dst_; 43 | int src_output_; 44 | int dst_input_; 45 | 46 | DISALLOW_COPY_AND_ASSIGN(DAGEdge); 47 | }; 48 | 49 | } // namespace euler 50 | 51 | #endif // EULER_CORE_DAG_EDGE_H_ 52 | -------------------------------------------------------------------------------- /euler/core/dag_def/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(dag_def dag_node_def.cc dag_def.cc sub_graph_iso.cc) 2 | target_link_libraries(dag_def dag) 3 | 4 | add_executable(dag_def_test dag_def_test.cc) 5 | target_link_libraries(dag_def_test dag_def gtest gtest_main) 6 | add_test(NAME dag_def_test COMMAND dag_def_test) 7 | -------------------------------------------------------------------------------- /euler/core/framework/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | protobuf_generate_cpp(dag_node.proto dag.proto tensor.proto tensor_shape.proto types.proto) 2 | 3 | add_library(framework_proto 4 | dag_node.pb.cc 5 | dag.pb.cc 6 | tensor.pb.cc 7 | tensor_shape.pb.cc 8 | types.pb.cc) 9 | target_link_libraries(framework_proto libprotobuf) 10 | target_compile_options(framework_proto PRIVATE -Wno-unused-parameter) 11 | 12 | add_library(framework 13 | tensor.cc 14 | tensor_util.cc 15 | allocator.cc 16 | executor.cc 17 | op_kernel.cc 18 | udf.cc) 19 | target_link_libraries(framework framework_proto common) 20 | 21 | add_executable(tensor_shape_test tensor_shape_test.cc) 22 | target_link_libraries(tensor_shape_test ${CMAKE_THREAD_LIBS_INIT} gtest gtest_main framework) 23 | add_test(NAME tensor_shape_test COMMAND tensor_shape_test) 24 | 25 | add_executable(tensor_test tensor_test.cc) 26 | target_link_libraries(tensor_test ${CMAKE_THREAD_LIBS_INIT} gtest gtest_main framework) 27 | add_test(NAME tensor_test COMMAND tensor_test) 28 | 29 | add_executable(tensor_util_test tensor_util_test.cc) 30 | target_link_libraries(tensor_util_test ${CMAKE_THREAD_LIBS_INIT} gtest gtest_main framework) 31 | add_test(NAME tensor_util_test COMMAND tensor_util_test) 32 | -------------------------------------------------------------------------------- /euler/core/framework/allocator.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | 17 | #include "euler/core/framework/allocator.h" 18 | 19 | namespace euler { 20 | 21 | Allocator* AllocatorManager::Get(const std::string& name, 22 | std::function creator) { 23 | MutexLock l(&mu_); 24 | auto iter = allocators_.find(name); 25 | if (iter != allocators_.end()) { 26 | return iter->second.get(); 27 | } 28 | Allocator* new_allocator = creator(); 29 | allocators_[name] = RefCountedPtr(new_allocator); 30 | new_allocator->Unref(); 31 | return new_allocator; 32 | } 33 | 34 | } // namespace euler 35 | -------------------------------------------------------------------------------- /euler/core/framework/allocator.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | 17 | #ifndef EULER_CORE_FRAMEWORK_ALLOCATOR_H_ 18 | #define EULER_CORE_FRAMEWORK_ALLOCATOR_H_ 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | #include "euler/common/mutex.h" 25 | #include "euler/common/refcount.h" 26 | #include "euler/common/singleton.h" 27 | 28 | namespace euler { 29 | 30 | class Allocator : public RefCounted { 31 | public: 32 | virtual void* Allocate(size_t size) = 0; 33 | virtual void Deallocate(void* buf) = 0; 34 | }; 35 | 36 | class AllocatorManager : public Singleton { 37 | public: 38 | Allocator* Get(const std::string& name, 39 | std::function creator); 40 | private: 41 | Mutex mu_; 42 | std::unordered_map> allocators_; 43 | }; 44 | 45 | } // namespace euler 46 | 47 | #endif // EULER_CORE_FRAMEWORK_ALLOCATOR_H_ 48 | -------------------------------------------------------------------------------- /euler/core/framework/attr_value.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_CORE_FRAMEWORK_ATTR_VALUE_H_ 17 | #define EULER_CORE_FRAMEWORK_ATTR_VALUE_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "euler/core/framework/types.h" 24 | #include "euler/common/status.h" 25 | #include "euler/core/framework/tensor_shape.h" 26 | 27 | namespace euler { 28 | 29 | struct AttrValue { 30 | enum Type { 31 | kNone, 32 | kString, 33 | kInt, 34 | kFloat, 35 | kBool, 36 | kDataType, 37 | kTensorShape, 38 | kDataTypeList 39 | }; 40 | 41 | Type attr_type; 42 | std::string s; 43 | int64_t i; 44 | float f; 45 | bool b; 46 | DataType type; 47 | TensorShape shape; 48 | std::vector type_list; 49 | }; 50 | 51 | } // namespace euler 52 | 53 | #endif // EULER_CORE_FRAMEWORK_ATTR_VALUE_H_ 54 | -------------------------------------------------------------------------------- /euler/core/framework/dag.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package euler; 4 | 5 | option cc_enable_arenas = true; 6 | option java_outer_classname = "GraphProto"; 7 | option java_multiple_files = true; 8 | option java_package = "com.alibaba.euler"; 9 | option go_package = "github.com/alibaba/euler/euler/go/core/framework"; 10 | 11 | import "euler/core/framework/dag_node.proto"; 12 | 13 | message DAGProto { 14 | bytes name = 1; 15 | repeated DAGNodeProto nodes = 2; 16 | } 17 | -------------------------------------------------------------------------------- /euler/core/framework/dag_node.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package euler; 4 | 5 | option cc_enable_arenas = true; 6 | option java_outer_classname = "NodeProto"; 7 | option java_multiple_files = true; 8 | option java_package = "com.alibaba.euler"; 9 | option go_package = "github.com/alibaba/euler/euler/go/core/framework"; 10 | 11 | message DAGNodeProto { 12 | string name = 1; // "a,1" a is op name, 1 is node id 13 | string op = 2; 14 | repeated string inputs = 3; // "a,1:0" a is op name, 1 is node id, 0 is output slot idx 15 | repeated string dnf = 4; // a eq b,c lt d 16 | repeated string post_process = 5; // "order_by p0 asc", "limit p1", "as p2" 17 | string op_alias = 6; 18 | int32 shard_idx = 7; // remote node field 19 | repeated DAGNodeProto inner_nodes = 8; // remote node field 20 | repeated string output_list = 9; // like "a,1:0" 21 | repeated string remote_output_list = 10; // like "REMOTE,0:0" 22 | int32 output_num = 11; 23 | string udf_name = 12; 24 | repeated string udf_str_params = 13; 25 | repeated string udf_num_params = 14; 26 | } 27 | -------------------------------------------------------------------------------- /euler/core/framework/executor.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_CORE_FRAMEWORK_EXECUTOR_H_ 17 | #define EULER_CORE_FRAMEWORK_EXECUTOR_H_ 18 | 19 | #include 20 | #include 21 | 22 | #include "euler/core/dag/dag.h" 23 | #include "euler/core/framework/tensor.h" 24 | #include "euler/core/framework/op_kernel.h" 25 | #include "euler/common/mutex.h" 26 | 27 | namespace euler { 28 | 29 | class ThreadPool; 30 | 31 | class Executor { 32 | public: 33 | typedef std::function DoneCallback; 34 | 35 | Executor(DAG* dag, ThreadPool* thread_pool, OpKernelContext* ctx); 36 | void Run(); 37 | void Run(DoneCallback callback); 38 | 39 | private: 40 | void RunInternal(); 41 | void Run(DAGNode* node); 42 | void RunDone(DAGNode* node); 43 | 44 | private: 45 | DAG* dag_; 46 | ThreadPool* thread_pool_; 47 | OpKernelContext* ctx_; 48 | std::vector> ref_; 49 | DoneCallback callback_; 50 | std::atomic remain_node_; 51 | }; 52 | 53 | } // namespace euler 54 | 55 | #endif // EULER_CORE_FRAMEWORK_EXECUTOR_H_ 56 | -------------------------------------------------------------------------------- /euler/core/framework/executor_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "euler/core/framework/executor.h" 17 | 18 | #include 19 | #include 20 | 21 | #include "glog/logging.h" 22 | #include "gtest/gtest.h" 23 | 24 | #include "euler/core/dag/node.h" 25 | 26 | namespace euler { 27 | namespace { 28 | 29 | class SimpleOp : public OpKernelAsync { 30 | public: 31 | explicit SimpleOp(const std::string &message) : message_(message) { } 32 | 33 | void Compute(OpKernelContext* ctx, Callback done) { 34 | LOG(INFO) << message_; 35 | done(0); 36 | } 37 | 38 | private: 39 | std::string message_; 40 | }; 41 | 42 | TEST(ExecutorTest, RunSimple) { 43 | DAG dag("simple_graph"); 44 | dag.add_node("a", 0, std::unique_ptr(new SimpleOp("a"))); 45 | dag.add_node("b", 1, std::unique_ptr(new SimpleOp("b"))); 46 | dag.add_node("c", 2, std::unique_ptr(new SimpleOp("c"))); 47 | dag.add_edge(0, 0, 1, 0); 48 | dag.add_edge(1, 0, 2, 0); 49 | 50 | Executor executor(&dag, nullptr); 51 | executor.Run(); 52 | } 53 | 54 | 55 | } // namespace 56 | } // namespace euler 57 | -------------------------------------------------------------------------------- /euler/core/framework/tensor.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package euler; 4 | 5 | option cc_enable_arenas = true; 6 | option java_outer_classname = "TensorProto"; 7 | option java_multiple_files = true; 8 | option java_package = "com.alibaba.euler"; 9 | option go_package = "github.com/alibaba/euler/euler/go/core/framework"; 10 | 11 | import "euler/core/framework/types.proto"; 12 | import "euler/core/framework/tensor_shape.proto"; 13 | 14 | message TensorProto { 15 | DataTypeProto dtype = 1; 16 | TensorShapeProto tensor_shape = 2; 17 | string name = 3; // tensor name, for op output fetch and feed 18 | bytes tensor_content = 4; 19 | } -------------------------------------------------------------------------------- /euler/core/framework/tensor_shape.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package euler; 4 | 5 | option cc_enable_arenas = true; 6 | option java_outer_classname = "TensorShapeProto"; 7 | option java_multiple_files = true; 8 | option java_package = "com.alibaba.euler"; 9 | option go_package = "github.com/alibaba/euler/euler/go/core/framework"; 10 | 11 | message TensorShapeProto { 12 | repeated int64 dims = 2; 13 | }; -------------------------------------------------------------------------------- /euler/core/framework/tensor_shape_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include 17 | #include 18 | 19 | #include "gtest/gtest.h" 20 | 21 | #include "euler/core/framework/tensor_shape.h" 22 | 23 | using std::string; 24 | 25 | namespace euler { 26 | 27 | TEST(TensorShape, GetSet) { 28 | } 29 | 30 | } // namespace euler 31 | -------------------------------------------------------------------------------- /euler/core/framework/tensor_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include "gtest/gtest.h" 23 | 24 | #include "euler/core/framework/tensor.h" 25 | 26 | using std::string; 27 | 28 | namespace euler { 29 | 30 | TEST(Tensor, Constructor) { 31 | } 32 | 33 | TEST(Tensor, Copy) { 34 | } 35 | 36 | TEST(Tensor, FromToProto) { 37 | } 38 | 39 | } // namespace euler 40 | -------------------------------------------------------------------------------- /euler/core/framework/tensor_util.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_CORE_FRAMEWORK_TENSOR_UTIL_H_ 17 | #define EULER_CORE_FRAMEWORK_TENSOR_UTIL_H_ 18 | 19 | #include "euler/common/status.h" 20 | #include "euler/core/framework/tensor.h" 21 | #include "euler/core/framework/tensor.pb.h" 22 | #include "euler/core/framework/tensor_shape.h" 23 | #include "euler/core/framework/tensor_shape.pb.h" 24 | #include "euler/core/framework/types.h" 25 | #include "euler/core/framework/types.pb.h" 26 | 27 | namespace euler { 28 | 29 | Status Encode(const Tensor& tensor, TensorProto* proto); 30 | Status Decode(const TensorProto& proto, Tensor* tensor); 31 | TensorShape ProtoToTensorShape(const TensorShapeProto& proto); 32 | DataType ProtoToDataType(DataTypeProto proto); 33 | 34 | } // namespace euler 35 | 36 | #endif // EULER_CORE_FRAMEWORK_TENSOR_UTIL_H_ 37 | -------------------------------------------------------------------------------- /euler/core/framework/thread_pool.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_CORE_FRAMEWORK_THREAD_POOL_H_ 17 | #define EULER_CORE_FRAMEWORK_THREAD_POOL_H_ 18 | 19 | namespace euler { 20 | 21 | class ThreadPool { 22 | public: 23 | virtual void Schedule(const std::function& func) = 0; 24 | }; 25 | 26 | } 27 | 28 | #endif // EULER_CORE_FRAMEWORK_THREAD_POOL_H_ 29 | -------------------------------------------------------------------------------- /euler/core/framework/types.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package euler; 4 | 5 | option cc_enable_arenas = true; 6 | option java_outer_classname = "TypeProto"; 7 | option java_multiple_files = true; 8 | option java_package = "com.alibaba.euler"; 9 | option go_package = "github.com/alibaba/euler/euler/go/core/framework"; 10 | 11 | enum DataTypeProto { 12 | DT_INT8 = 0; 13 | DT_INT16 = 1; 14 | DT_INT32 = 2; 15 | DT_INT64 = 3; 16 | 17 | DT_UINT8 = 4; 18 | DT_UINT16 = 5; 19 | DT_UINT32 = 6; 20 | DT_UINT64 = 7; 21 | 22 | DT_FLOAT = 8; 23 | DT_DOUBLE = 9; 24 | 25 | DT_BOOL = 10; 26 | DT_STRING = 11; 27 | } -------------------------------------------------------------------------------- /euler/core/graph/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(core SHARED node.cc edge.cc graph.cc graph_builder.cc graph_meta.cc) 2 | target_link_libraries(core common) 3 | 4 | add_executable(node_test node_test.cc) 5 | target_link_libraries(node_test ${cmake_thread_libs_init} gtest gtest_main core) 6 | add_test(NAME node_test COMMAND node_test) 7 | 8 | add_executable(edge_test edge_test.cc) 9 | target_link_libraries(edge_test ${cmake_thread_libs_init} gtest gtest_main core) 10 | add_test(NAME edge_test COMMAND edge_test) 11 | 12 | add_executable(graph_test graph_test.cc) 13 | target_link_libraries(graph_test ${cmake_thread_libs_init} gtest gtest_main core) 14 | add_test(NAME graph_test COMMAND graph_test) 15 | -------------------------------------------------------------------------------- /euler/core/index/index_types.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_CORE_INDEX_INDEX_TYPES_H_ 17 | #define EULER_CORE_INDEX_INDEX_TYPES_H_ 18 | 19 | #include 20 | #include 21 | 22 | #include "euler/core/framework/types.h" 23 | 24 | namespace euler { 25 | 26 | enum IndexType : int32_t { 27 | HASHINDEX = 0, 28 | RANGEINDEX, 29 | HASHRANGEINDEX 30 | }; 31 | 32 | enum IndexResultType : int32_t { 33 | HASHINDEXRESULT = 0, 34 | RANGEINDEXRESULT, 35 | COMMONINDEXRESULT 36 | }; 37 | 38 | enum IndexSearchType : int32_t { 39 | LESS = 0, 40 | LESS_EQ, 41 | EQ, 42 | GREATER, 43 | GREATER_EQ, 44 | NOT_EQ, 45 | IN, 46 | NOT_IN 47 | }; 48 | 49 | typedef DataType IndexDataType; 50 | 51 | } // namespace euler 52 | 53 | #endif // EULER_CORE_INDEX_INDEX_TYPES_H_ 54 | -------------------------------------------------------------------------------- /euler/core/index/index_util.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_CORE_INDEX_INDEX_UTIL_H_ 17 | #define EULER_CORE_INDEX_INDEX_UTIL_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "euler/core/index/index_types.h" 25 | 26 | namespace euler { 27 | 28 | template 29 | inline void VecToPairVec(const std::vector& ids, 30 | const std::vector& weights, 31 | std::vector>* v) { 32 | for (size_t i = 0; i < ids.size(); ++i) { 33 | v->push_back(std::make_pair(ids[i], weights[i])); 34 | } 35 | } 36 | 37 | template 38 | inline OutputIter intersection(PairIter first1, PairIter last1, 39 | Iter first2, Iter last2, 40 | OutputIter result) { 41 | while (first1 != last1 && first2 != last2) { 42 | if ((*first1).first < uint64_t(*first2)) { 43 | ++first1; 44 | } else if (uint64_t(*first2) < (*first1).first) { 45 | ++first2; 46 | } else { 47 | *result = *first1; 48 | ++result; ++first1; ++first2; 49 | } 50 | } 51 | return result; 52 | } 53 | 54 | } // namespace euler 55 | 56 | #endif // EULER_CORE_INDEX_INDEX_UTIL_H_ 57 | -------------------------------------------------------------------------------- /euler/core/kernels/as_op.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include 17 | #include 18 | 19 | #include "euler/common/logging.h" 20 | #include "euler/core/framework/op_kernel.h" 21 | #include "euler/core/framework/dag_node.pb.h" 22 | #include "euler/core/framework/tensor.h" 23 | #include "euler/common/str_util.h" 24 | 25 | namespace euler { 26 | class As: public OpKernel { 27 | public: 28 | explicit As(const std::string& name) : OpKernel(name) {} 29 | void Compute(const DAGNodeProto& node_def, 30 | OpKernelContext* ctx) override; 31 | }; 32 | 33 | void As::Compute(const DAGNodeProto& node_def, 34 | OpKernelContext* ctx) { 35 | std::string alias_name = node_def.op_alias(); 36 | for (int32_t i = 0; i < node_def.inputs_size(); ++i) { 37 | Tensor* t = nullptr; 38 | ctx->tensor(node_def.inputs(i), &t); 39 | ctx->AddAlias(OutputName(alias_name, i), t); 40 | } 41 | } 42 | 43 | REGISTER_OP_KERNEL("AS", As); 44 | 45 | } // namespace euler 46 | -------------------------------------------------------------------------------- /euler/core/kernels/broad_cast_split_op.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include 17 | #include 18 | 19 | #include "euler/core/framework/op_kernel.h" 20 | #include "euler/core/framework/dag_node.pb.h" 21 | #include "euler/core/framework/tensor.h" 22 | #include "euler/common/str_util.h" 23 | #include "euler/client/query_proxy.h" 24 | 25 | namespace euler { 26 | class BroadCastSplit: public OpKernel { 27 | public: 28 | explicit BroadCastSplit(const std::string& name) : OpKernel(name) {} 29 | void Compute(const DAGNodeProto& node_def, 30 | OpKernelContext* ctx) override; 31 | }; 32 | 33 | void BroadCastSplit::Compute(const DAGNodeProto& node_def, 34 | OpKernelContext* ctx) { 35 | int32_t split_num = QueryProxy::GetInstance()->GetShardNum(); 36 | /* get input tensor */ 37 | Tensor* input_t = nullptr; 38 | ctx->tensor(node_def.inputs(0), &input_t); 39 | 40 | /* output */ 41 | for (int32_t i = 0; i < split_num * 2; i += 2) { 42 | std::string output_name = OutputName(node_def, i); 43 | ctx->AddAlias(output_name, input_t); 44 | } 45 | } 46 | 47 | REGISTER_OP_KERNEL("BROAD_CAST_SPLIT", BroadCastSplit); 48 | 49 | } // namespace euler 50 | -------------------------------------------------------------------------------- /euler/core/kernels/gather_result_op.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include 17 | #include 18 | 19 | #include "euler/core/framework/op_kernel.h" 20 | #include "euler/core/framework/dag_node.pb.h" 21 | #include "euler/core/framework/tensor.h" 22 | #include "euler/core/kernels/common.h" 23 | #include "euler/common/str_util.h" 24 | 25 | namespace euler { 26 | class GatherResult: public OpKernel { 27 | public: 28 | explicit GatherResult(const std::string& name) : OpKernel(name) {} 29 | void Compute(const DAGNodeProto& node_def, 30 | OpKernelContext* ctx) override; 31 | }; 32 | 33 | void GatherResult::Compute(const DAGNodeProto& node_def, 34 | OpKernelContext* ctx) { 35 | Tensor* adj_idx_t = nullptr; 36 | Tensor* adj_nb_t = nullptr; 37 | Tensor* l_nb_t = nullptr; 38 | ctx->tensor(node_def.inputs(0), &adj_idx_t); 39 | ctx->tensor(node_def.inputs(1), &adj_nb_t); 40 | ctx->tensor(node_def.inputs(2), &l_nb_t); 41 | 42 | ctx->AddAlias(OutputName(node_def, 0), adj_idx_t); 43 | ctx->AddAlias(OutputName(node_def, 1), adj_nb_t); 44 | ctx->AddAlias(OutputName(node_def, 2), l_nb_t); 45 | } 46 | 47 | REGISTER_OP_KERNEL("API_GATHER_RESULT", GatherResult); 48 | 49 | } // namespace euler 50 | -------------------------------------------------------------------------------- /euler/parser/.gitignore: -------------------------------------------------------------------------------- 1 | lex.yy.c 2 | gremlin.tab.h 3 | gremlin.tab.c 4 | gremlin.output 5 | -------------------------------------------------------------------------------- /euler/parser/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | execute_process( 2 | COMMAND bison -rall ${PROJECT_SOURCE_DIR}/euler/parser/gremlin.y -d -b ${PROJECT_SOURCE_DIR}/euler/parser/gremlin) 3 | execute_process( 4 | COMMAND flex -o ${PROJECT_SOURCE_DIR}/euler/parser/lex.yy.c ${PROJECT_SOURCE_DIR}/euler/parser/gremlin.l) 5 | 6 | SET_SOURCE_FILES_PROPERTIES(lex.yy.c gremlin.tab.c PROPERTIES LANGUAGE CXX) 7 | 8 | add_library(gen_lex lex.yy.c gremlin.tab.c) 9 | target_compile_options(gen_lex PRIVATE -Wno-all) 10 | 11 | add_library(compiler attribute_calculator.cc gen_node_def_input_output.cc translator.cc optimizer.cc compiler.cc optimize_type.cc) 12 | target_link_libraries(compiler gen_lex common dag_def) 13 | 14 | add_executable(tree_test tree_test.cc) 15 | target_link_libraries(tree_test gtest gtest_main) 16 | add_test(NAME tree_test COMMAND tree_test) 17 | 18 | add_executable(translator_test translator_test.cc) 19 | target_link_libraries(translator_test compiler gtest gtest_main) 20 | add_test(NAME translator_test COMMAND translator_test) 21 | 22 | add_executable(compiler_test compiler_test.cc) 23 | target_link_libraries(compiler_test compiler framework gtest gtest_main) 24 | add_test(NAME compiler_test COMMAND compiler_test) 25 | -------------------------------------------------------------------------------- /euler/parser/optimize_type.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "euler/parser/optimize_type.h" 17 | 18 | namespace euler { 19 | 20 | bool compatible_matrix[3][3] = 21 | {{true, false, false}, 22 | {true, true, false}, 23 | {false, false, true }}; 24 | 25 | } // namesapce euler 26 | -------------------------------------------------------------------------------- /euler/parser/optimize_type.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_PARSER_OPTIMIZE_TYPE_H_ 17 | #define EULER_PARSER_OPTIMIZE_TYPE_H_ 18 | 19 | namespace euler { 20 | 21 | enum OptimizerType { 22 | local = 0, 23 | distribute = 1, 24 | graph_partition = 2 25 | }; 26 | 27 | /* run mode -> optimize type */ 28 | extern bool compatible_matrix[3][3]; 29 | 30 | } // namespace euler 31 | 32 | #endif // EULER_PARSER_OPTIMIZE_TYPE_H_ 33 | -------------------------------------------------------------------------------- /euler/parser/tree_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include 17 | 18 | #include "gtest/gtest.h" 19 | 20 | #include "euler/parser/tree.h" 21 | namespace euler { 22 | 23 | TEST(TreeTest, PostTraversalTest) { 24 | TreeNode* n0 = new TreeNode("0"); 25 | Tree t(n0); 26 | TreeNode* n1 = new TreeNode("1"); 27 | TreeNode* n2 = new TreeNode("2"); 28 | TreeNode* n3 = new TreeNode("3"); 29 | TreeNode* n4 = new TreeNode("4"); 30 | TreeNode* n5 = new TreeNode("5"); 31 | TreeNode* n6 = new TreeNode("6"); 32 | TreeNode* n7 = new TreeNode("7"); 33 | /* 34 | 0 35 | / \ 36 | 1 2 37 | |\ / | \ 38 | 3 4 5 6 7 39 | */ 40 | t.AddNode(n0, n1); 41 | t.AddNode(n0, n2); 42 | t.AddNode(n1, n3); 43 | t.AddNode(n1, n4); 44 | t.AddNode(n2, n5); 45 | t.AddNode(n2, n6); 46 | t.AddNode(n2, n7); 47 | // t.DeleteNode(n1); 48 | std::vector results; 49 | t.PostTraversal(n0, &results); 50 | 51 | std::vector expct = 52 | {"3", "4", "1", "5", "6", "7", "2", "0"}; 53 | ASSERT_EQ(results.size(), expct.size()); 54 | for (size_t i = 0; i < expct.size(); ++i) { 55 | ASSERT_EQ(results[i]->GetValue(), expct[i]); 56 | } 57 | std::cout << "=================" << std::endl; 58 | std::cout << t.Deserialize() << std::endl; 59 | } 60 | 61 | } // namespace euler 62 | -------------------------------------------------------------------------------- /euler/proto/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | protobuf_generate_cpp(worker.proto) 2 | add_library(proto worker.pb.cc) 3 | target_link_libraries(proto libprotobuf framework) 4 | target_compile_options(proto PRIVATE -Wno-unused-parameter) 5 | -------------------------------------------------------------------------------- /euler/proto/service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package euler; 4 | 5 | option cc_enable_arenas = true; 6 | option java_outer_classname = "EulerServiceProto"; 7 | option java_multiple_files = true; 8 | option java_package = "com.alibaba.euler"; 9 | option go_package = "github.com/alibaba/euler/euler/go/proto"; 10 | 11 | import "euler/proto/worker.proto"; 12 | 13 | service EulerService { 14 | rpc Ping(PingRequest) returns (PingReply) {} 15 | 16 | // Euler 2.0 Gremlin executor service 17 | rpc Execute (ExecuteRequest) returns (ExecuteReply) {} 18 | 19 | // Euler 1.0 service 20 | rpc SampleNode (SampleNodeRequest) returns (SampleNodeReply) {} 21 | rpc SampleEdge (SampleEdgeRequest) returns (SampleEdgeReply) {} 22 | rpc GetNodeType (GetNodeTypeRequest) returns (GetNodeTypeReply) {} 23 | rpc GetNodeFloat32Feature (GetNodeFloat32FeatureRequest) returns (GetFloat32FeatureReply) {} 24 | rpc GetNodeUInt64Feature (GetNodeUInt64FeatureRequest) returns (GetUInt64FeatureReply) {} 25 | rpc GetNodeBinaryFeature (GetNodeBinaryFeatureRequest) returns (GetBinaryFeatureReply) {} 26 | rpc GetEdgeFloat32Feature (GetEdgeFloat32FeatureRequest) returns (GetFloat32FeatureReply) {} 27 | rpc GetEdgeUInt64Feature (GetEdgeUInt64FeatureRequest) returns (GetUInt64FeatureReply) {} 28 | rpc GetEdgeBinaryFeature (GetEdgeBinaryFeatureRequest) returns (GetBinaryFeatureReply) {} 29 | rpc GetFullNeighbor (GetFullNeighborRequest) returns (GetNeighborReply) {} 30 | rpc GetSortedNeighbor (GetSortedNeighborRequest) returns (GetNeighborReply) {} 31 | rpc GetTopKNeighbor (GetTopKNeighborRequest) returns (GetNeighborReply) {} 32 | rpc SampleNeighbor (SampleNeighborRequest) returns (GetNeighborReply) {} 33 | } 34 | -------------------------------------------------------------------------------- /euler/python/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | -------------------------------------------------------------------------------- /euler/python/hash_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import ctypes 21 | import os 22 | import threading 23 | 24 | _LIB_DIR = os.path.dirname(os.path.realpath(__file__)) 25 | _LIB_NAME = 'libeuler_util.so' 26 | _LIB_PATH = os.path.join(_LIB_DIR, _LIB_NAME) 27 | _LIB = ctypes.CDLL(_LIB_PATH) 28 | 29 | print (_LIB.py_hash64('1234', len('1234'))) 30 | -------------------------------------------------------------------------------- /euler/service/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(service SHARED 2 | server_interface.cc 3 | grpc_server.cc 4 | grpc_euler_service.cc 5 | grpc_worker_service.cc 6 | grpc_worker.cc 7 | python_api.cc) 8 | 9 | target_link_libraries(service api grpc++_unsecure dag common proto jemalloc_STATIC_PIC ${CMAKE_THREAD_LIBS_INIT}) 10 | 11 | add_library(mock_api SHARED mock_api.cc) 12 | 13 | add_executable(grpc_server_test grpc_server_test.cc) 14 | target_link_libraries(grpc_server_test service ops mock_api gtest gtest_main grpc++_unsecure) 15 | add_test(NAME grpc_server_test COMMAND grpc_server_test) 16 | -------------------------------------------------------------------------------- /euler/service/async_service_interface.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_SERVICE_ASYNC_SERVICE_INTERFACE_H_ 17 | #define EULER_SERVICE_ASYNC_SERVICE_INTERFACE_H_ 18 | 19 | namespace euler { 20 | 21 | class AsyncServiceInterface { 22 | public: 23 | virtual ~AsyncServiceInterface() {} 24 | 25 | virtual void Loop() = 0; 26 | 27 | virtual void Shutdown() = 0; 28 | }; 29 | 30 | } // namespace euler 31 | 32 | #endif // EULER_SERVICE_ASYNC_SERVICE_INTERFACE_H_ 33 | -------------------------------------------------------------------------------- /euler/service/grpc_worker.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef EULER_SERVICE_GRPC_WORKER_H_ 17 | #define EULER_SERVICE_GRPC_WORKER_H_ 18 | 19 | #include 20 | 21 | #include "euler/service/worker.h" 22 | 23 | namespace euler { 24 | 25 | class GrpcWorker: public Worker { 26 | public: 27 | explicit GrpcWorker(WorkerEnv* env); 28 | 29 | #define DECLARE_METHOD(Method) \ 30 | void Method##Async(const Method##Request* request, \ 31 | Method##Reply* reply, Callback) override; \ 32 | 33 | DECLARE_METHOD(Ping); 34 | DECLARE_METHOD(Execute); 35 | 36 | #undef DECLARE_METHOD // DECLARE_METHOD 37 | }; 38 | 39 | std::unique_ptr NewGrpcWorker(WorkerEnv* worker_env); 40 | 41 | } // namespace euler 42 | 43 | #endif // EULER_SERVICE_GRPC_WORKER_H_ 44 | -------------------------------------------------------------------------------- /euler/service/grpc_worker_service.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | 17 | #ifndef EULER_SERVICE_GRPC_WORKER_SERVICE_H_ 18 | #define EULER_SERVICE_GRPC_WORKER_SERVICE_H_ 19 | 20 | #include 21 | 22 | #include "grpcpp/server_builder.h" 23 | 24 | namespace euler { 25 | 26 | class AsyncServiceInterface; 27 | class GrpcWorker; 28 | 29 | std::unique_ptr NewGrpcWorkerService( 30 | GrpcWorker* worker, ::grpc::ServerBuilder* builder); 31 | 32 | } // namespace euler 33 | 34 | #endif // EULER_SERVICE_GRPC_WORKER_SERVICE_H_ 35 | -------------------------------------------------------------------------------- /euler/service/mock_api.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "euler/core/api/api.h" 17 | #include "euler/common/logging.h" 18 | 19 | namespace euler { 20 | 21 | ////////////////// Mock Euler Api ////////////////// 22 | 23 | NodeIdVec SampleNode(int node_type, int count) { 24 | EULER_LOG(INFO) << "Here: mock euler api"; 25 | NodeIdVec node_ids(count, node_type); 26 | return node_ids; 27 | } 28 | 29 | // Get node feature 30 | FloatFeatureVec GetNodeFloat32Feature(const NodeIdVec& node_ids, 31 | const std::vector& fids) { 32 | FloatFeatureVec features(node_ids.size()); 33 | for (auto& feature : features) { 34 | feature.resize(fids.size()); 35 | int i = 0; 36 | for (auto& ff : feature) { 37 | ff.resize(10); 38 | for (auto& fff : ff) { 39 | fff = fids[i] + 0.5; 40 | } 41 | i++; 42 | } 43 | } 44 | return features; 45 | } 46 | 47 | FloatFeatureVec GetNodeFloat32Feature( 48 | const NodeIdVec& node_ids, const std::vector& ft_names) { 49 | std::vector fids; 50 | for (auto& ft_name : ft_names) { 51 | fids.push_back(atoi(ft_name->c_str())); 52 | } 53 | return GetNodeFloat32Feature(node_ids, fids); 54 | } 55 | } // namespace euler 56 | -------------------------------------------------------------------------------- /euler/tools/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(remote_console) 2 | -------------------------------------------------------------------------------- /euler/tools/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | from euler.tools import util 21 | from euler.tools import graph_meta 22 | from euler.tools import node 23 | from euler.tools import edge 24 | from euler.tools import json2meta 25 | from euler.tools import json2partdat 26 | from euler.tools import json2partindex 27 | from euler.tools import generate_euler_data 28 | -------------------------------------------------------------------------------- /euler/tools/gen_graph_partition_test_data.sh: -------------------------------------------------------------------------------- 1 | /bin/bash gen_paritioned_data.sh ../../tools/test_data/graph.json ../../tools/test_data/meta /tmp/gp_euler 1 2 | cp /tmp/gp_euler/Edge/0_0.dat /tmp/gp_euler/Edge/0_1.dat 3 | cp /tmp/gp_euler/Node/0_0.dat /tmp/gp_euler/Node/0_1.dat 4 | cp /tmp/gp_euler/Index/att/0_0.dat /tmp/gp_euler/Index/att/0_1.dat 5 | cp /tmp/gp_euler/Index/edge_att/0_0.dat /tmp/gp_euler/Index/edge_att/0_1.dat 6 | cp /tmp/gp_euler/Index/edge_type/0_0.dat /tmp/gp_euler/Index/edge_type/0_1.dat 7 | cp /tmp/gp_euler/Index/edge_value/0_0.dat /tmp/gp_euler/Index/edge_value/0_1.dat 8 | cp /tmp/gp_euler/Index/graph_label/0_0.dat /tmp/gp_euler/Index/graph_label/0_1.dat 9 | cp /tmp/gp_euler/Index/node_type/0_0.dat /tmp/gp_euler/Index/node_type/0_1.dat 10 | cp /tmp/gp_euler/Index/price/0_0.dat /tmp/gp_euler/Index/price/0_1.dat 11 | 12 | -------------------------------------------------------------------------------- /euler/tools/gen_partitioned_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ $# != 4 -a $# != 5 ]; then 3 | echo "sh gen_partitioned_data.sh graph.json index_meta output_dir shard_num [part_no]" 4 | exit 1; 5 | fi 6 | graph_json=$1 7 | index_json=$2 8 | dir=$3 9 | shard_num=$4 10 | part_no=0 11 | 12 | if [ $# == 5 ]; then 13 | part_no=$5 14 | fi 15 | 16 | if [ $part_no == 0 ]; then 17 | rm -fr ${dir} 18 | python json2meta.py $graph_json $dir/euler.meta $shard_num 19 | fi 20 | python json2partdat.py $graph_json $dir/euler.meta $dir $shard_num $part_no 21 | python json2partindex.py $index_json $graph_json $dir $shard_num $part_no 22 | 23 | exit 0 24 | -------------------------------------------------------------------------------- /euler/tools/remote_console/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(remote_console remote_console.cc) 2 | target_link_libraries(remote_console euler_core) 3 | -------------------------------------------------------------------------------- /euler/util/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(euler_util SHARED python_api.cc) 2 | target_link_libraries(euler_util common) 3 | 4 | add_custom_command( 5 | TARGET euler_util POST_BUILD 6 | COMMAND ${CMAKE_COMMAND} -E copy $ ${PROJECT_SOURCE_DIR}/euler/tools 7 | COMMAND ${CMAKE_COMMAND} -E copy $ ${PROJECT_SOURCE_DIR}/euler/tools 8 | COMMENT "Building test data") 9 | -------------------------------------------------------------------------------- /euler/util/python_api.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "euler/common/hash.h" 17 | 18 | namespace euler { 19 | 20 | extern "C" { 21 | extern uint64_t py_hash64(const char* data, uint64_t size) { 22 | return euler::hash64(data, size); 23 | } 24 | } 25 | 26 | } // namespace euler 27 | -------------------------------------------------------------------------------- /euler_estimator/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | from euler_estimator.python.graph_estimator import GraphEstimator 21 | from euler_estimator.python.node_estimator import NodeEstimator 22 | from euler_estimator.python.edge_estimator import EdgeEstimator 23 | from euler_estimator.python.gae_estimator import GaeEstimator 24 | from euler_estimator.python.sample_estimator import SampleEstimator 25 | -------------------------------------------------------------------------------- /euler_estimator/python/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | -------------------------------------------------------------------------------- /examples/adaptivegcn/README.md: -------------------------------------------------------------------------------- 1 | Adaptive Graph Convolutional Neural Networks (AdaptiveGCN) 2 | ============ 3 | 4 | Paper link: [Adaptive Graph Convolutional Neural Networks](https://www.aaai.org/ocs/index.php/AAAI/AAAI18/paper/download/16642/16554) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_adaptivegcn.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | 14 | | Parameter Name | Default | Note | 15 | | ----------------- | -------------- | ------------------------------- | 16 | | dataset | cora | cora/pubmed/citeseer/ppi/reddit | 17 | | hidden_dim | 32 | hidden dimension | 18 | | layers | 2 | GCN layer number | 19 | | batch_size | 32 | running batch size | 20 | | num_epochs | 10 | epochs to run | 21 | | log_steps | 20 | log per steps | 22 | | model_dir | ckpt | checkpoint dir | 23 | | learning_rate | 0.01 | run learning rate | 24 | | optimizer | adam | run optimizer algorithm | 25 | | run_mode | train | train/evaluate | 26 | | fanouts | 400,400 | total neighbor per layer | 27 | 28 | Result 29 | ------ 30 | | Dataset | F1 | 31 | | ---------- | ------------------ | 32 | | cora | 0.821 | 33 | | pubmed | 0.859 | 34 | | citeseer | 0.751 | 35 | 36 | -------------------------------------------------------------------------------- /examples/agnn/README.md: -------------------------------------------------------------------------------- 1 | Attention-based Graph Neural Network(AGNN) 2 | ============ 3 | 4 | Paper link: [Attention-based Graph Neural Network for Semi-supervised Learning](https://arxiv.org/abs/1803.03735) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_agnn.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | 14 | | Parameter Name | Default | Note | 15 | | ----------------- | -------------- | ------------------------------- | 16 | | dataset | cora | cora/pubmed/citeseer/ppi/reddit | 17 | | hidden_dim | 32 | hidden dimension | 18 | | layers | 2 | GCN layer number | 19 | | batch_size | 32 | running batch size | 20 | | num_epochs | 10 | epochs to run | 21 | | log_steps | 20 | log per steps | 22 | | model_dir | ckpt | checkpoint dir | 23 | | learning_rate | 0.01 | run learning rate | 24 | | optimizer | adam | run optimizer algorithm | 25 | | run_mode | train | train/evaluate | 26 | 27 | Result 28 | ------ 29 | | Dataset | F1 | 30 | | ---------- | ------------------ | 31 | | cora | 0.813 | 32 | | pubmed | 0.894 | 33 | | citeseer | 0.719 | 34 | 35 | -------------------------------------------------------------------------------- /examples/appnp/README.md: -------------------------------------------------------------------------------- 1 | Predict then Propagate: Graph Neural Networks meet Personalized PageRank (APPNP) 2 | ============ 3 | 4 | Paper link: [Predict then Propagate: Graph Neural Networks meet Personalized PageRank](https://arxiv.org/abs/1810.05997) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_appnp.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | 14 | | Parameter Name | Default | Note | 15 | | ----------------- | -------------- | ------------------------------- | 16 | | dataset | cora | cora/pubmed/citeseer/ppi/reddit | 17 | | hidden_dim | 32 | hidden dimension | 18 | | layers | 10 | GCN layer number | 19 | | batch_size | 32 | running batch size | 20 | | num_epochs | 20 | epochs to run | 21 | | log_steps | 20 | log per steps | 22 | | model_dir | ckpt | checkpoint dir | 23 | | learning_rate | 0.01 | run learning rate | 24 | | optimizer | adam | run optimizer algorithm | 25 | | run_mode | train | train/evaluate | 26 | 27 | Result 28 | ------ 29 | | Dataset | F1 | 30 | | ---------- | ------------------ | 31 | | cora | 0.813 | 32 | | pubmed | 0.870 | 33 | | citeseer | 0.723 | 34 | 35 | -------------------------------------------------------------------------------- /examples/arma/README.md: -------------------------------------------------------------------------------- 1 | ARMA 2 | ============ 3 | 4 | Paper link: [Graph Neural Networks with convolutional ARMA filters](https://arxiv.org/abs/1901.01343) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_arma.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | 14 | | Parameter Name | Default | Note | 15 | | ----------------- | -------------- | ------------------------------- | 16 | | dataset | cora | cora/pubmed/citeseer/ppi/reddit | 17 | | hidden_dim | 32 | hidden dimension | 18 | | layers | 1 | GCN layer number | 19 | | batch_size | 32 | running batch size | 20 | | num_epochs | 10 | epochs to run | 21 | | log_steps | 20 | log per steps | 22 | | model_dir | ckpt | checkpoint dir | 23 | | learning_rate | 0.01 | run learning rate | 24 | | optimizer | adam | run optimizer algorithm | 25 | | run_mode | train | train/evaluate | 26 | | K | 2 | ARMA K parameter | 27 | 28 | Result 29 | ------ 30 | | Dataset | F1 | 31 | | ---------- | ------------------ | 32 | | cora | 0.822 | 33 | | pubmed | 0.880 | 34 | | citeseer | 0.755 | 35 | 36 | -------------------------------------------------------------------------------- /examples/deepwalk/README.md: -------------------------------------------------------------------------------- 1 | Graph Convolutional Networks (GCN) 2 | ============ 3 | 4 | Paper link: [Semi-Supervised Classification with Graph Convolutional Networks](https://arxiv.org/abs/1609.02907](https://arxiv.org/abs/1609.02907) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_deepwalk.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | 14 | | Parameter Name | Default | Note | 15 | | ----------------- | -------------- | ------------------------------- | 16 | | dataset | cora | cora/pubmed/citeseer/ppi/reddit | 17 | | hidden_dim | 32 | hidden dimension | 18 | | embedding_dim | 32 | embedding dimension | 19 | | walk_len | 3 | Length of random walk | 20 | | p | 1 | DeepWalk return parameter | 21 | | q | 1 | DeepWalk in-out parameter | 22 | | left_win_size | 1 | Left window size | 23 | | right_win_size | 1 | Right window size | 24 | | num_negs | 5 | Negative sample number | 25 | | use_id | True | Whether use id embedding | 26 | | batch_size | 32 | running batch size | 27 | | num_epochs | 10 | epochs to run | 28 | | log_steps | 20 | log per steps | 29 | | model_dir | ckpt | checkpoint dir | 30 | | learning_rate | 0.01 | run learning rate | 31 | | optimizer | adam | run optimizer algorithm | 32 | | run_mode | train | train/evaluate | 33 | 34 | Result 35 | ------ 36 | | Dataset | mrr | 37 | | ---------- | ------------------ | 38 | | cora | 0.905 | 39 | | pubmed | 0.983 | 40 | | citeseer | 0.976 | 41 | 42 | -------------------------------------------------------------------------------- /examples/dgi/README.md: -------------------------------------------------------------------------------- 1 | Deep Graph Infomax (DGI) 2 | ============ 3 | 4 | Paper link: [Deep Graph Infomax](https://arxiv.org/abs/1809.10341) 5 | Author's code repo (in Pytorch): [https://github.com/PetarV-/DGI](https://github.com/PetarV-/DGI) 6 | 7 | Run 8 | ------- 9 | ```python 10 | python run_dgi.py [--optional_params=params] 11 | ``` 12 | 13 | Params: 14 | 15 | | Parameter Name | Default | Note | 16 | | ----------------- | -------------- | ------------------------------- | 17 | | dataset | ppi | cora/pubmed/citeseer/ppi/reddit | 18 | | hidden_dim | 32 | hidden dimension | 19 | | layers | 2 | SAGE convolution layer number | 20 | | fanouts | [10, 10] | GraphSage fanouts | 21 | | batch_size | 32 | mini batch size | 22 | | num_epochs | 20 | epochs to run | 23 | | num_negs | 5 | negative sample number | 24 | | log_steps | 20 | log per steps | 25 | | model_dir | ckpt | checkpoint dir | 26 | | learning_rate | 0.01 | run learning rate | 27 | | optimizer | adam | run optimizer algorithm | 28 | | metric | mrr | mrr/mr/hit10 | 29 | | run_mode | train | train/evaluate | 30 | 31 | -------------------------------------------------------------------------------- /examples/distmult/README.md: -------------------------------------------------------------------------------- 1 | Embedding Entities and Relations for Learning and Inference in Knowledge Bases(DistMult) 2 | ============ 3 | 4 | Paper link: [Embedding Entities and Relations for Learning and Inference in Knowledge Bases](https://arxiv.org/abs/1412.6575) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_distmult.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | 14 | | Parameter Name | Default | Note | 15 | | ----------------- | -------------- | ------------------------------- | 16 | | dataset | fb15k | | 17 | | embedding_dim | 100 | entity/relation hidden dimension | 18 | | num_negs | 1 | Number of negative samplings | 19 | | corrupt | both | both/front/tail | 20 | | margin | 1.0 | Margin of loss | 21 | | metric_name | mrr | mrr/mr/hit10 | 22 | | batch_size | 128 | Mini-batch size | 23 | | l2_regular | True | if use l2 regular | 24 | | regular_param | 0.0001 | l2 regular param | 25 | | num_epochs | 500 | epochs to run | 26 | | log_steps | 20 | log per steps | 27 | | model_dir | ckpt | checkpoint dir | 28 | | learning_rate | 0.01 | run learning rate | 29 | | optimizer | adam | run optimizer algorithm | 30 | | run_mode | train | train/evaluate/infer | 31 | | infer_type | edge | edge_node_src/node_dst | 32 | 33 | -------------------------------------------------------------------------------- /examples/dna/README.md: -------------------------------------------------------------------------------- 1 | Dynamic Neighborhood Aggregation(DNA) 2 | ============ 3 | 4 | Paper link: [Just Jump: Dynamic Neighborhood Aggregation in Graph Neural Networks 5 | ](https://arxiv.org/abs/1609.02907](https://arxiv.org/abs/1904.04849) 6 | 7 | Run 8 | ------- 9 | ```python 10 | python run_dna.py [--optional_params=params] 11 | ``` 12 | 13 | Params: 14 | 15 | | Parameter Name | Default | Note | 16 | | ----------------- | -------------- | ------------------------------- | 17 | | dataset | cora | cora/pubmed/citeseer/ppi/reddit | 18 | | hidden_dim | 32 | hidden dimension | 19 | | layers | 2 | GCN layer number | 20 | | batch_size | 32 | running batch size | 21 | | num_epochs | 15 | epochs to run | 22 | | log_steps | 20 | log per steps | 23 | | model_dir | ckpt | checkpoint dir | 24 | | learning_rate | 0.01 | run learning rate | 25 | | optimizer | adam | run optimizer algorithm | 26 | | run_mode | train | train/evaluate | 27 | | group_num | 8 | DNA group number | 28 | | head_num | 1 | attension head number | 29 | 30 | Result 31 | ------ 32 | | Dataset | F1 | 33 | | ---------- | ------------------ | 34 | | cora | 0.811 | 35 | | pubmed | 0.867 | 36 | | citeseer | 0.710 | 37 | 38 | -------------------------------------------------------------------------------- /examples/fastgcn/README.md: -------------------------------------------------------------------------------- 1 | Fast Learning with Graph Convolutional Networks (FastGCN) 2 | ============ 3 | 4 | Paper link: [FastGCN: Fast Learning with Graph Convolutional Networks via Importance Sampling](https://arxiv.org/abs/1801.10247) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_fastgcn.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | 14 | | Parameter Name | Default | Note | 15 | | ----------------- | -------------- | ------------------------------- | 16 | | dataset | cora | cora/pubmed/citeseer/ppi/reddit | 17 | | hidden_dim | 32 | hidden dimension | 18 | | layers | 2 | GCN layer number | 19 | | batch_size | 32 | running batch size | 20 | | num_epochs | 10 | epochs to run | 21 | | log_steps | 20 | log per steps | 22 | | model_dir | ckpt | checkpoint dir | 23 | | learning_rate | 0.01 | run learning rate | 24 | | optimizer | adam | run optimizer algorithm | 25 | | run_mode | train | train/evaluate | 26 | | fanouts | 400,400 | total neighbor per layer | 27 | 28 | Result 29 | ------ 30 | | Dataset | F1 | 31 | | ---------- | ------------------ | 32 | | cora | 0.803 | 33 | | pubmed | 0.860 | 34 | | citeseer | 0.740 | 35 | 36 | -------------------------------------------------------------------------------- /examples/gae/README.md: -------------------------------------------------------------------------------- 1 | Graph Auto Encoder / Variational Graph Auto Encoder (GAE VGAE) 2 | ============ 3 | 4 | Paper link: [Variational Graph Auto-Encoders](https://arxiv.org/abs/1611.07308) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_gae.py [--optional_params=params] 10 | python run_vgae.py [--optional_params=params] 11 | ``` 12 | 13 | Params: 14 | 15 | | Parameter Name | Default | Note | 16 | | ----------------- | -------------- | ------------------------------- | 17 | | dataset | cora | cora/pubmed/citeseer/ppi/reddit | 18 | | hidden_dim | 32 | hidden dimension | 19 | | layers | 2 | GCN layer number | 20 | | batch_size | 1024 | running batch size | 21 | | num_epochs | 2000 | epochs to run | 22 | | log_steps | 10 | log per steps | 23 | | model_dir | ckpt | checkpoint dir | 24 | | learning_rate | 0.01 | run learning rate | 25 | | optimizer | adam | run optimizer algorithm | 26 | | run_mode | train | train/evaluate | 27 | 28 | GAE Result 29 | ------ 30 | | Dataset | acc | 31 | | ---------- | ------------------ | 32 | | cora | 0.71 | 33 | 34 | VGAE Result 35 | ------ 36 | | Dataset | acc | 37 | | ---------- | ------------------ | 38 | | cora | 0.79 | 39 | -------------------------------------------------------------------------------- /examples/gat/README.md: -------------------------------------------------------------------------------- 1 | Graph Attention Networks (GAT) 2 | ============ 3 | 4 | Paper link: [Graph Attention Networks](https://arxiv.org/abs/1710.10903) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_gat.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | 14 | | Parameter Name | Default | Note | 15 | | ----------------- | -------------- | ------------------------------- | 16 | | dataset | cora | cora/pubmed/citeseer/ppi/reddit | 17 | | hidden_dim | 32 | hidden dimension | 18 | | layers | 2 | GCN layer number | 19 | | batch_size | 32 | running batch size | 20 | | num_epochs | 10 | epochs to run | 21 | | log_steps | 20 | log per steps | 22 | | model_dir | ckpt | checkpoint dir | 23 | | learning_rate | 0.01 | run learning rate | 24 | | optimizer | adam | run optimizer algorithm | 25 | | run_mode | train | train/evaluate | 26 | | concat | True | attention concat | 27 | | improved | True | attention improved | 28 | | num_heads | 1 | attention head number | 29 | 30 | Result 31 | ------ 32 | | Dataset | F1 | 33 | | ---------- | ------------------ | 34 | | cora | 0.823 | 35 | | pubmed | 0.876 | 36 | | citeseer | 0.755 | 37 | 38 | -------------------------------------------------------------------------------- /examples/gated_graph/README.md: -------------------------------------------------------------------------------- 1 | GatedGraph 2 | ============ 3 | 4 | Paper link: [GATED GRAPH SEQUENCE NEURAL NETWORKS](https://arxiv.org/pdf/1511.05493.pdf) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_gatedgraph.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | 14 | | Parameter Name | Default | Note | 15 | | ----------------- | -------------- | ------------------------------- | 16 | | dataset | mutag | dataset | 17 | | hidden_dim | 32 | hidden dimension | 18 | | layers | 2 | GCN layer number | 19 | | process_steps | 2 | LSTM processing steps | 20 | | lstm_layers | 2 | LSTM layer number | 21 | | num_epochs | 1000 | epochs to run | 22 | | log_steps | 20 | log per steps | 23 | | model_dir | ckpt | checkpoint dir | 24 | | learning_rate | 0.01 | run learning rate | 25 | | optimizer | adam | run optimizer algorithm | 26 | | run_mode | train | train/evaluate | 27 | 28 | Result 29 | ------ 30 | | Dataset | Accuracy | 31 | | ---------- | ------------------ | 32 | | mutag | 0.920 | 33 | 34 | -------------------------------------------------------------------------------- /examples/gcn/README.md: -------------------------------------------------------------------------------- 1 | Graph Convolutional Networks (GCN) 2 | ============ 3 | 4 | Paper link: [Semi-Supervised Classification with Graph Convolutional Networks](https://arxiv.org/abs/1609.02907) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_gcn.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | 14 | | Parameter Name | Default | Note | 15 | | ----------------- | -------------- | ------------------------------- | 16 | | dataset | cora | cora/pubmed/citeseer/ppi/reddit | 17 | | hidden_dim | 32 | hidden dimension | 18 | | layers | 2 | GCN layer number | 19 | | batch_size | 32 | running batch size | 20 | | num_epochs | 10 | epochs to run | 21 | | log_steps | 20 | log per steps | 22 | | model_dir | ckpt | checkpoint dir | 23 | | learning_rate | 0.01 | run learning rate | 24 | | optimizer | adam | run optimizer algorithm | 25 | | run_mode | train | train/evaluate | 26 | 27 | Result 28 | ------ 29 | | Dataset | F1 | 30 | | ---------- | ------------------ | 31 | | cora | 0.822 | 32 | | pubmed | 0.871 | 33 | | citeseer | 0.752 | 34 | 35 | -------------------------------------------------------------------------------- /examples/geniepath/README.md: -------------------------------------------------------------------------------- 1 | Genipath 2 | ============ 3 | 4 | Paper link: [GeniePath: Graph Neural Networks with Adaptive Receptive Paths](https://arxiv.org/pdf/1802.00910.pdf) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_genipath.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | 14 | | Parameter Name | Default | Note | 15 | | ----------------- | -------------- | ------------------------------- | 16 | | dataset | cora | cora/pubmed/citeseer/ppi/reddit | 17 | | hidden_dim | 32 | hidden dimension | 18 | | embedding_dim | 32 | embedding dimension | 19 | | use_id | False | Whether use id embedding | 20 | | layers | 2 | GCN layer number | 21 | | batch_size | 32 | running batch size | 22 | | num_epochs | 10 | epochs to run | 23 | | log_steps | 20 | log per steps | 24 | | model_dir | ckpt | checkpoint dir | 25 | | learning_rate | 0.01 | run learning rate | 26 | | run_mode | train | train/evaluate | 27 | 28 | Result 29 | ------ 30 | | Dataset | F1 | 31 | | ---------- | ------------------ | 32 | | cora | 0.742 | 33 | | pubmed | 0.872 | 34 | | citeseer | 0.735 | 35 | 36 | -------------------------------------------------------------------------------- /examples/gin/README.md: -------------------------------------------------------------------------------- 1 | Graph Isomorphism Network (GIN) 2 | ============ 3 | 4 | Paper link: [HOW POWERFUL ARE GRAPH NEURAL NETWORKS?](https://arxiv.org/pdf/1810.00826.pdf) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_gin.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | | Parameter Name | Default | Note | 14 | | ----------------- | -------------- | ------------------------------- | 15 | | dataset | mutag | cora/pubmed/citeseer/ppi/reddit | 16 | | hidden_dim | 32 | hidden dimension | 17 | | layers | 5 | GCN layer number | 18 | | num_epochs | 500 | epochs to run | 19 | | log_steps | 20 | log per steps | 20 | | model_dir | ckpt | checkpoint dir | 21 | | learning_rate | 0.01 | run learning rate | 22 | | optimizer | adam | run optimizer algorithm | 23 | | run_mode | train | train/evaluate | 24 | | train_eps | True | Whether ope train as variable | 25 | | eps | 0.0 | eps initialize | 26 | 27 | Result 28 | ------ 29 | | Dataset | Accuracy | 30 | | ---------- | ------------------ | 31 | | mutag | 0.923 | 32 | 33 | -------------------------------------------------------------------------------- /examples/graphgcn/README.md: -------------------------------------------------------------------------------- 1 | Higher-order Graph Neural Networks 2 | ============ 3 | 4 | Paper link: [Weisfeiler and Leman Go Neural: Higher-order Graph Neural Networks](https://arxiv.org/pdf/1810.02244.pdf) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_graphgcn.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | 14 | | Parameter Name | Default | Note | 15 | | ----------------- | -------------- | ------------------------------- | 16 | | dataset | mutag | cora/pubmed/citeseer/ppi/reddit | 17 | | hidden_dim | 32 | hidden dimension | 18 | | layers | 5 | GCN layer number | 19 | | num_epochs | 500 | epochs to run | 20 | | log_steps | 20 | log per steps | 21 | | model_dir | ckpt | checkpoint dir | 22 | | learning_rate | 0.01 | run learning rate | 23 | | optimizer | adam | run optimizer algorithm | 24 | | run_mode | train | train/evaluate | 25 | 26 | Result 27 | ------ 28 | | Dataset | Accuracy | 29 | | ---------- | ------------------ | 30 | | mutag | 0.891 | 31 | 32 | -------------------------------------------------------------------------------- /examples/graphsage/README.md: -------------------------------------------------------------------------------- 1 | Inductive Representation Learning on Large Graphs (GraphSAGE) 2 | ============ 3 | 4 | Paper link: [Inductive Representation Learning on Large Graphs](http://papers.nips.cc/paper/6703-inductive-representation-learning-on-large-graphs.pdf) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_graphsage.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | 14 | | Parameter Name | Default | Note | 15 | | ----------------- | -------------- | ------------------------------- | 16 | | dataset | cora | cora/pubmed/citeseer/ppi/reddit | 17 | | hidden_dim | 32 | hidden dimension | 18 | | layers | 2 | GCN layer number | 19 | | batch_size | 32 | running batch size | 20 | | num_epochs | 10 | epochs to run | 21 | | log_steps | 20 | log per steps | 22 | | model_dir | ckpt | checkpoint dir | 23 | | learning_rate | 0.01 | run learning rate | 24 | | optimizer | adam | run optimizer algorithm | 25 | | run_mode | train | train/evaluate | 26 | | fanouts | 10,10 | sample neighbor number per layer | 27 | 28 | Result 29 | ------ 30 | | Dataset | F1 | 31 | | ---------- | ------------------ | 32 | | cora | 0.774 | 33 | | pubmed | 0.884 | 34 | | citeseer | 0.731 | 35 | 36 | -------------------------------------------------------------------------------- /examples/lgcn/README.md: -------------------------------------------------------------------------------- 1 | Large-Scale Learnable Graph Convolutional Networks(LGCN) 2 | ============ 3 | 4 | Paper link: [Large-Scale Learnable Graph Convolutional Networks](https://arxiv.org/pdf/1808.03965.pdf) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_lgcn.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | 14 | | Parameter Name | Default | Note | 15 | | ----------------- | -------------- | ------------------------------- | 16 | | dataset | cora | cora/pubmed/citeseer/ppi/reddit | 17 | | hidden_dim | 32 | hidden dimension | 18 | | k | 3 | LGCN top k | 19 | | num_neighbor | 10 | number of neighbor | 20 | | batch_size | 32 | running batch size | 21 | | num_epochs | 10 | epochs to run | 22 | | log_steps | 20 | log per steps | 23 | | model_dir | ckpt | checkpoint dir | 24 | | learning_rate | 0.01 | run learning rate | 25 | | optimizer | adam | run optimizer algorithm | 26 | | run_mode | train | train/evaluate | 27 | 28 | Result 29 | ------ 30 | | Dataset | F1 | 31 | | ---------- | ------------------ | 32 | | cora | 0.641 | 33 | | pubmed | 0.848 | 34 | | citeseer | 0.675 | 35 | 36 | -------------------------------------------------------------------------------- /examples/lgcn/lgcn.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import tensorflow as tf 21 | 22 | from tf_euler.python.utils import encoders 23 | from tf_euler.python.mp_utils.base import SuperviseModel 24 | 25 | 26 | class LGCN(SuperviseModel): 27 | def __init__(self, dim, metapath, 28 | label_idx, label_dim, 29 | feature_idx=-1, feature_dim=0, 30 | k=3, nb_num=10, out_dim=64, 31 | *args, **kwargs): 32 | super(LGCN, self).__init__(label_idx, label_dim, *args, **kwargs) 33 | self._encoder = encoders.LGCEncoder(metapath, feature_idx, feature_dim, 34 | k, dim, nb_num, out_dim) 35 | 36 | def embed(self, n_id): 37 | return self._encoder(n_id) 38 | -------------------------------------------------------------------------------- /examples/line/README.md: -------------------------------------------------------------------------------- 1 | LINE 2 | ============ 3 | 4 | Paper link: [Line: Large-scale information network embedding](https://arxiv.org/abs/1503.03578) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_line.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | 14 | | Parameter Name | Default | Note | 15 | | ----------------- | -------------- | ------------------------------- | 16 | | dataset | cora | cora/pubmed/citeseer/ppi/reddit | 17 | | hidden_dim | 32 | hidden dimension | 18 | | embedding_dim | 32 | embedding dimension | 19 | | order | 2 | line order | 20 | | num_negs | 5 | Negative sample number | 21 | | use_id | True | Whether use id embedding | 22 | | batch_size | 32 | running batch size | 23 | | num_epochs | 10 | epochs to run | 24 | | log_steps | 20 | log per steps | 25 | | model_dir | ckpt | checkpoint dir | 26 | | learning_rate | 0.01 | run learning rate | 27 | | optimizer | adam | run optimizer algorithm | 28 | | run_mode | train | train/evaluate | 29 | 30 | Result 31 | ------ 32 | | Dataset | mrr | 33 | | ---------- | ------------------ | 34 | | cora | 0.900 | 35 | | pubmed | 0.987 | 36 | | citeseer | 0.956 | 37 | 38 | -------------------------------------------------------------------------------- /examples/rgcn/README.md: -------------------------------------------------------------------------------- 1 | Modeling Relational Data with Graph Convolutional Networks (RGCN) 2 | ============ 3 | 4 | Paper link: [Modeling Relational Data with Graph Convolutional Networks](https://arxiv.org/abs/1703.06103) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_rgcn.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | 14 | | Parameter Name | Default | Note | 15 | | ----------------- | -------------- | ------------------------------- | 16 | | dataset | wn18 | wn18/fb15/fb15-237/ 17 | | 18 | | hidden_dim | 32 | hidden dimension | 19 | | embedding_dim | 32 | node embedding dimension | 20 | | num_negs | 5 | negative sample number | 21 | | layers | 2 | RGCN layer number | 22 | | batch_size | 32 | running batch size | 23 | | num_epochs | 10 | epochs to run | 24 | | log_steps | 20 | log per steps | 25 | | model_dir | ckpt | checkpoint dir | 26 | | learning_rate | 0.01 | run learning rate | 27 | | optimizer | adam | run optimizer algorithm | 28 | | metric | mrr | mrr/mr/hit1/hit3/hit10 | 29 | | run_mode | train | train/evaluate | 30 | -------------------------------------------------------------------------------- /examples/sample_solution/sample.txt: -------------------------------------------------------------------------------- 1 | 1,123 2 | -------------------------------------------------------------------------------- /examples/set2set/README.md: -------------------------------------------------------------------------------- 1 | Set2Set 2 | ============ 3 | 4 | Paper link: [ORDER MATTERS: SEQUENCE TO SEQUENCE FOR SETS](https://arxiv.org/pdf/1511.06391.pdf) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_set2set.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | 14 | | Parameter Name | Default | Note | 15 | | ----------------- | -------------- | ------------------------------- | 16 | | dataset | mutag | cora/pubmed/citeseer/ppi/reddit | 17 | | hidden_dim | 32 | hidden dimension | 18 | | layers | 5 | GCN layer number | 19 | | process_steps | 4 | LSTM processing steps | 20 | | lstm_layers | 2 | LSTM layer number | 21 | | num_epochs | 500 | epochs to run | 22 | | log_steps | 20 | log per steps | 23 | | model_dir | ckpt | checkpoint dir | 24 | | learning_rate | 0.01 | run learning rate | 25 | | optimizer | adam | run optimizer algorithm | 26 | | run_mode | train | train/evaluate | 27 | 28 | Result 29 | ------ 30 | | Dataset | Accuracy | 31 | | ---------- | ------------------ | 32 | | mutag | 0.901 | 33 | 34 | -------------------------------------------------------------------------------- /examples/sgcn/README.md: -------------------------------------------------------------------------------- 1 | Simplifying Graph Convolutional Networks(SGCN) 2 | ============ 3 | 4 | Paper link: [Simplifying Graph Convolutional Networks](https://arxiv.org/pdf/1902.07153.pdf) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_sgcn.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | 14 | | Parameter Name | Default | Note | 15 | | ----------------- | -------------- | ------------------------------- | 16 | | dataset | cora | cora/pubmed/citeseer/ppi/reddit | 17 | | hidden_dim | 32 | hidden dimension | 18 | | layers | 2 | GCN layer number | 19 | | batch_size | 32 | running batch size | 20 | | num_epochs | 10 | epochs to run | 21 | | log_steps | 20 | log per steps | 22 | | model_dir | ckpt | checkpoint dir | 23 | | learning_rate | 0.01 | run learning rate | 24 | | optimizer | adam | run optimizer algorithm | 25 | | run_mode | train | train/evaluate | 26 | 27 | Result 28 | ------ 29 | | Dataset | F1 | 30 | | ---------- | ------------------ | 31 | | cora | 0.825 | 32 | | pubmed | 0.866 | 33 | | citeseer | 0.716 | 34 | 35 | -------------------------------------------------------------------------------- /examples/tagcn/README.md: -------------------------------------------------------------------------------- 1 | Topology Adaptive Graph Convolutional Networks(TAGCN) 2 | ============ 3 | 4 | Paper link: [Topology Adaptive Graph Convolutional Networks](https://arxiv.org/abs/1710.10370) 5 | 6 | Run 7 | ------- 8 | ```python 9 | python run_tagcn.py [--optional_params=params] 10 | ``` 11 | 12 | Params: 13 | 14 | | Parameter Name | Default | Note | 15 | | ----------------- | -------------- | ------------------------------- | 16 | | dataset | cora | cora/pubmed/citeseer/ppi/reddit | 17 | | hidden_dim | 32 | hidden dimension | 18 | | layers | 2 | GCN layer number | 19 | | batch_size | 32 | running batch size | 20 | | num_epochs | 15 | epochs to run | 21 | | log_steps | 20 | log per steps | 22 | | model_dir | ckpt | checkpoint dir | 23 | | learning_rate | 0.01 | run learning rate | 24 | | optimizer | adam | run optimizer algorithm | 25 | | run_mode | train | train/evaluate | 26 | 27 | Result 28 | ------ 29 | | Dataset | F1 | 30 | | ---------- | ------------------ | 31 | | cora | 0.817 | 32 | | pubmed | 0.867 | 33 | | citeseer | 0.727 | 34 | 35 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tqdm 2 | networkx==1.11 3 | scipy==1.1.0 4 | faiss-cpu 5 | 6 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [easy_install] 2 | index_url = https://mirrors.aliyun.com/pypi/simple/ 3 | -------------------------------------------------------------------------------- /tf_euler/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | from tf_euler.python import utils 21 | from tf_euler.python.euler_ops import * 22 | from tf_euler.python import dataset 23 | from tf_euler.python import dataflow 24 | from tf_euler.python import convolution 25 | from tf_euler.python import graph_pool 26 | from tf_euler.python import node_pool 27 | from tf_euler.python import mp_utils 28 | from tf_euler.python import solution 29 | -------------------------------------------------------------------------------- /tf_euler/ops/type_ops.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/core/framework/op.h" 17 | #include "tensorflow/core/framework/shape_inference.h" 18 | #include "tensorflow/core/framework/common_shape_fns.h" 19 | 20 | namespace tensorflow { 21 | 22 | REGISTER_OP("GetNodeType") 23 | .Input("nodes: int64") 24 | .Output("types: int32") 25 | .SetShapeFn(shape_inference::UnchangedShape) 26 | .Doc(R"doc( 27 | GetNodeType. 28 | 29 | Get types of nodes. 30 | 31 | nodes: Input, nodes to get types for)doc"); 32 | 33 | 34 | REGISTER_OP("GetNodeTypeId") 35 | .Input("type_names: string") 36 | .Output("type_ids: int32") 37 | .SetShapeFn(shape_inference::UnchangedShape) 38 | .Doc(R"doc( 39 | GetNodeTypeId. 40 | 41 | Get type ids by node type names. 42 | 43 | type_names: Input, node type names to get type ids for)doc"); 44 | 45 | REGISTER_OP("GetEdgeTypeId") 46 | .Input("type_names: string") 47 | .Output("type_ids: int32") 48 | .SetShapeFn(shape_inference::UnchangedShape) 49 | .Doc(R"doc( 50 | GetEdgeTypeId. 51 | 52 | Get type ids by edge type names. 53 | 54 | type_names: Input, node type names to get type ids for)doc"); 55 | 56 | } // namespace tensorflow 57 | -------------------------------------------------------------------------------- /tf_euler/python/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | -------------------------------------------------------------------------------- /tf_euler/python/contrib/spmm.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import tensorflow as tf 21 | 22 | 23 | def spmm_add(src, edge_index, size, flow='target_to_source'): 24 | adj_indices = tf.stack([edge_index[0], edge_index[1]], axis=1) 25 | adj_values = tf.ones(tf.shape(edge_index)[1]) 26 | adj_shape = size 27 | adj = tf.SparseTensor(tf.cast(adj_indices, tf.int64), 28 | adj_values, 29 | adj_shape) 30 | adj = tf.sparse_reorder(adj) 31 | return tf.sparse_tensor_dense_matmul(adj, src) 32 | 33 | 34 | def spmm_mean(src, edge_index, size, flow='target_to_source'): 35 | out = spmm_add(src, edge_index, size, flow) 36 | count = spmm_add(tf.ones([tf.shape(src)[0], 1]), edge_index, size, flow) 37 | return out / count 38 | 39 | 40 | def spmm_(op, src, edge_index, size, flow='target_to_source'): 41 | return globals()['spmm_' + op](src, edge_index, size, flow) 42 | -------------------------------------------------------------------------------- /tf_euler/python/convolution/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | from tf_euler.python.convolution.conv import Conv 21 | from tf_euler.python.convolution.gat_conv import GATConv 22 | from tf_euler.python.convolution.gcn_conv import GCNConv 23 | from tf_euler.python.convolution.sage_conv import SAGEConv 24 | from tf_euler.python.convolution.tag_conv import TAGConv 25 | from tf_euler.python.convolution.agnn_conv import AGNNConv 26 | from tf_euler.python.convolution.sgcn_conv import SGCNConv 27 | from tf_euler.python.convolution.gin_conv import GINConv 28 | from tf_euler.python.convolution.graph_conv import GraphConv 29 | from tf_euler.python.convolution.appnp_conv import APPNPConv 30 | from tf_euler.python.convolution.arma_conv import ARMAConv 31 | from tf_euler.python.convolution.dna_conv import DNAConv 32 | from tf_euler.python.convolution.relation_conv import RelationConv 33 | from tf_euler.python.convolution.gated_graph_conv import GatedConv 34 | -------------------------------------------------------------------------------- /tf_euler/python/convolution/conv.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import sys 21 | import inspect 22 | 23 | import tensorflow as tf 24 | from tf_euler.python.euler_ops import mp_ops 25 | 26 | 27 | class Conv(object): 28 | 29 | def __init__(self, aggr='add'): 30 | self.aggr = aggr 31 | assert self.aggr in ['add', 'mean', 'max'] 32 | 33 | def gather_feature(self, features, edge_index): 34 | convert_features = [] 35 | 36 | for feature in features: 37 | convert_feature = [] 38 | assert isinstance(feature, tuple) or isinstance(feature, list) 39 | assert len(feature) == 2 40 | if feature[1] is None: 41 | feature[1] = feature[0] 42 | for idx, tmp in enumerate(feature): 43 | if tmp is not None: 44 | tmp = mp_ops.gather(tmp, edge_index[idx]) 45 | convert_feature.append(tmp) 46 | convert_features.append(convert_feature) 47 | return convert_features 48 | 49 | def apply_edge(self, x_j): 50 | return x_j 51 | 52 | def apply_node(self, aggr_out): 53 | return aggr_out 54 | -------------------------------------------------------------------------------- /tf_euler/python/convolution/conv_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import tensorflow as tf 21 | 22 | from tf_euler.python.convolution import * 23 | from tf_euler.python.euler_ops import mp_ops 24 | 25 | 26 | class SimpleConv(Conv): 27 | 28 | def __call__(self, x, edge_index, size=None): 29 | gather_x, = self.gather_feature([x], edge_index) 30 | out = self.apply_edge(gather_x[1]) 31 | out = mp_ops.scatter_(self.aggr, out, edge_index[0], size=size[0]) 32 | out = self.apply_node(out) 33 | return out 34 | 35 | 36 | class ConvTest(tf.test.TestCase): 37 | 38 | def testConv(self): 39 | edge_index = tf.constant([[0, 1, 1, 2], [1, 0, 2, 1]], dtype=tf.int32) 40 | x = tf.constant([[-1], [0], [1]], dtype=tf.float32) 41 | x = [x, x] 42 | conv = SimpleConv() 43 | x1 = conv(x, edge_index, size=[3, 3]) 44 | 45 | with self.test_session(): 46 | self.assertAllEqual([[0.], [0.], [0.]], x1.eval()) 47 | 48 | 49 | if __name__ == '__main__': 50 | tf.test.main() 51 | -------------------------------------------------------------------------------- /tf_euler/python/convolution/graph_conv.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import tensorflow as tf 21 | 22 | from tf_euler.python.convolution import conv 23 | from tf_euler.python.euler_ops import mp_ops 24 | 25 | 26 | class GraphConv(conv.Conv): 27 | 28 | def __init__(self, dim, **kwargs): 29 | super(GraphConv, self).__init__(aggr='mean', **kwargs) 30 | self.fc = tf.layers.Dense(dim, use_bias=False) 31 | self.liner = tf.layers.Dense(dim, use_bias=True) 32 | 33 | def __call__(self, x, edge_index, size=None, **kwargs): 34 | h = [None if x[0] is None else self.fc(x[0]), 35 | None if x[1] is None else self.fc(x[1])] 36 | gather_x, gather_h = self.gather_feature([x, h], edge_index) 37 | out = self.apply_edge(gather_h[1]) 38 | out = mp_ops.scatter_(self.aggr, out, edge_index[0], size=size[0]) 39 | out = self.apply_node(out, x[0]) 40 | return out 41 | 42 | def apply_edge(self, x_j): 43 | return x_j 44 | 45 | def apply_node(self, aggr_out, x): 46 | return self.liner(x) + aggr_out 47 | -------------------------------------------------------------------------------- /tf_euler/python/convolution/sage_conv.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import tensorflow as tf 21 | 22 | from tf_euler.python.convolution import conv 23 | from tf_euler.python.euler_ops import mp_ops 24 | 25 | 26 | class SAGEConv(conv.Conv): 27 | 28 | def __init__(self, dim, **kwargs): 29 | super(SAGEConv, self).__init__(aggr='mean', **kwargs) 30 | self.self_fc = tf.layers.Dense(dim, use_bias=False) 31 | self.neigh_fc = tf.layers.Dense(dim, use_bias=False) 32 | 33 | def __call__(self, x, edge_index, size=None, **kwargs): 34 | gather_x, = self.gather_feature([x], edge_index) 35 | out = self.apply_edge(gather_x[1]) 36 | out = mp_ops.scatter_(self.aggr, out, edge_index[0], size=size[0]) 37 | out = self.apply_node(out, x[0]) 38 | return out 39 | 40 | def apply_edge(self, x_j): 41 | return x_j 42 | 43 | def apply_node(self, aggr_out, x): 44 | return self.self_fc(x) + self.neigh_fc(aggr_out) 45 | -------------------------------------------------------------------------------- /tf_euler/python/dataflow/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | from tf_euler.python.dataflow.sage_dataflow import SageDataFlow 21 | from tf_euler.python.dataflow.gcn_dataflow import GCNDataFlow 22 | from tf_euler.python.dataflow.fast_dataflow import FastGCNDataFlow 23 | from tf_euler.python.dataflow.layerwise_dataflow import \ 24 | LayerwiseDataFlow, LayerwiseEachDataFlow 25 | from tf_euler.python.dataflow.whole_dataflow import WholeDataFlow 26 | from tf_euler.python.dataflow.relation_dataflow import RelationDataFlow 27 | -------------------------------------------------------------------------------- /tf_euler/python/dataflow/base_dataflow.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import tensorflow as tf 21 | 22 | 23 | class Block(object): 24 | def __init__(self, n_id, res_n_id, e_id, edge_index, size): 25 | self.n_id = n_id 26 | self.res_n_id = res_n_id 27 | self.e_id = e_id 28 | self.edge_index = edge_index 29 | self.size = size 30 | 31 | 32 | class DataFlow(object): 33 | def __init__(self, n_id): 34 | self.n_id = n_id 35 | self.__last_n_id__ = n_id 36 | self.blocks = [] 37 | 38 | def append(self, n_id, res_n_id, e_id, edge_index): 39 | size = [tf.shape(self.__last_n_id__)[0], tf.shape(n_id)[0]] 40 | block = Block(n_id, res_n_id, e_id, edge_index, size) 41 | self.blocks.append(block) 42 | self.__last_n_id__ = n_id 43 | 44 | def __len__(self): 45 | return len(self.blocks) 46 | 47 | def __getitem__(self, idx): 48 | return self.blocks[::-1][idx] 49 | 50 | def __iter__(self): 51 | for block in self.blocks[::-1]: 52 | yield block 53 | -------------------------------------------------------------------------------- /tf_euler/python/dataset/README.md: -------------------------------------------------------------------------------- 1 | # Load build-in dataset 2 | You can load tf_euler build-in dataset in python. 3 | 4 | Datasets: 5 | - cora 6 | - pubmed 7 | - citeseer 8 | - ppi 9 | - reddit 10 | - fb15k 11 | 12 | Examples: 13 | ```python 14 | import tf_euler 15 | ppi = tf_euler.dataset.get_dataset('ppi') 16 | ppi.load_graph() 17 | ''' 18 | 19 | -------------------------------------------------------------------------------- /tf_euler/python/dataset/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | from tf_euler.python.dataset.utils import get_dataset 21 | -------------------------------------------------------------------------------- /tf_euler/python/euler_ops/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | from tf_euler.python.euler_ops.base import * 21 | from tf_euler.python.euler_ops.feature_ops import * 22 | from tf_euler.python.euler_ops.neighbor_ops import * 23 | from tf_euler.python.euler_ops.sample_ops import * 24 | from tf_euler.python.euler_ops.walk_ops import * 25 | from tf_euler.python.euler_ops.type_ops import * 26 | from tf_euler.python.euler_ops.util_ops import * 27 | from tf_euler.python.euler_ops.mp_ops import * 28 | -------------------------------------------------------------------------------- /tf_euler/python/euler_ops/base_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Graph Initialize python api test""" 17 | 18 | from __future__ import absolute_import 19 | from __future__ import division 20 | from __future__ import print_function 21 | 22 | import os 23 | import subprocess 24 | 25 | from tensorflow.python.platform import test 26 | from tf_euler.python.euler_ops import base 27 | 28 | 29 | class BaseTest(test.TestCase): 30 | """Graph Initialize test. 31 | 32 | Test python wrapper for Graph c++ api works as expected. 33 | """ 34 | 35 | @classmethod 36 | def setUpClass(cls): 37 | """Build Graph data for test""" 38 | cls._cur_dir = os.path.dirname(os.path.realpath(__file__)) 39 | 40 | def testInitializeGraph(self): 41 | self.assertTrue( 42 | base.initialize_graph({ 43 | 'mode': 'local', 44 | 'data_path': '/tmp/euler', 45 | 'sampler_type': u'all' 46 | })) 47 | 48 | 49 | if __name__ == "__main__": 50 | test.main() 51 | -------------------------------------------------------------------------------- /tf_euler/python/euler_ops/type_ops_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Euler type ops test""" 17 | 18 | from __future__ import absolute_import 19 | from __future__ import division 20 | from __future__ import print_function 21 | 22 | import os 23 | import subprocess 24 | 25 | from tensorflow.python.platform import test 26 | import tensorflow as tf 27 | 28 | from tf_euler.python.euler_ops import base 29 | from tf_euler.python.euler_ops import type_ops as ops 30 | 31 | 32 | class TypeOpsTest(test.TestCase): 33 | @classmethod 34 | def setUpClass(cls): 35 | base.initialize_graph({ 36 | 'mode': 'local', 37 | 'data_path': '/tmp/euler', 38 | 'sampler_type': 'all', 39 | 'data_type': 'all' 40 | }) 41 | 42 | def testGetNodeType(self): 43 | """Test euler get node type""" 44 | 45 | op = ops.get_node_type([1, 2, 3, 4, 5, 4]) 46 | with tf.Session() as sess: 47 | node_types = sess.run(op) 48 | self.assertAllEqual([0, 1, 0, 1, 0, 1], node_types) 49 | 50 | 51 | if __name__ == "__main__": 52 | test.main() 53 | -------------------------------------------------------------------------------- /tf_euler/python/euler_ops/util_ops.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import ctypes 21 | import os 22 | 23 | import tensorflow as tf 24 | 25 | from tf_euler.python.euler_ops import base 26 | 27 | inflate_idx = base._LIB_OP.inflate_idx 28 | sparse_gather = base._LIB_OP.sparse_gather 29 | -------------------------------------------------------------------------------- /tf_euler/python/euler_ops/walk_ops.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import tensorflow as tf 21 | 22 | from tf_euler.python.euler_ops import base 23 | from tf_euler.python.euler_ops import type_ops 24 | 25 | gen_pair = base._LIB_OP.gen_pair 26 | _random_walk = base._LIB_OP.random_walk 27 | 28 | 29 | def random_walk(nodes, edge_types, p=1.0, q=1.0, default_node=-1): 30 | ''' 31 | Random walk from a list of nodes. 32 | 33 | Args: 34 | nodes: start node ids, 1-d Tensor 35 | edge_types: list of 1-d Tensor of edge types 36 | p: back probality 37 | q: forward probality 38 | default_node: default fill nodes 39 | ''' 40 | 41 | edge_types = [type_ops.get_edge_type_id(edge_type) 42 | for edge_type in edge_types] 43 | return _random_walk(nodes, edge_types, p, q, default_node) 44 | -------------------------------------------------------------------------------- /tf_euler/python/graph_pool/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | from tf_euler.python.graph_pool.base_pool import Pooling 21 | from tf_euler.python.graph_pool.set2set_pool import Set2SetPool 22 | from tf_euler.python.graph_pool.attention_pool import AttentionPool 23 | -------------------------------------------------------------------------------- /tf_euler/python/graph_pool/attention_pool.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | import tensorflow as tf 17 | 18 | from tf_euler.python.euler_ops import mp_ops 19 | from tf_euler.python.graph_pool.base_pool import Pooling 20 | 21 | 22 | class AttentionPool(Pooling): 23 | 24 | def __init__(self, gate_nn=None, nn=None, aggr='add'): 25 | super(AttentionPool, self).__init__(aggr=aggr) 26 | if gate_nn is None: 27 | self.gate_nn = tf.layers.Dense(1, use_bias=False) 28 | else: 29 | self.gate_nn = gate_nn 30 | self.nn = nn 31 | self.aggr = aggr 32 | 33 | def __call__(self, inputs, index, size=None): 34 | size = tf.reduce_max(index) + 1 if size is None else size 35 | 36 | gate = self.gate_nn(inputs) 37 | inputs = self.nn(inputs) if self.nn is not None else inputs 38 | 39 | gate = mp_ops.scatter_softmax(gate, index, size=size) 40 | outputs = mp_ops.scatter_(self.aggr, gate * inputs, index, size=size) 41 | return outputs 42 | -------------------------------------------------------------------------------- /tf_euler/python/graph_pool/base_pool.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | import tensorflow as tf 17 | 18 | from tf_euler.python.euler_ops.mp_ops import scatter_ 19 | 20 | 21 | class Pooling(object): 22 | def __init__(self, aggr='add'): 23 | assert aggr in ['add', 'mean', 'max'] 24 | self.aggr = aggr 25 | 26 | def __call__(self, inputs, index, size=None): 27 | size = tf.reduce_max(index) + 1 if size is None else size 28 | out = scatter_(self.aggr, inputs, index, size) 29 | return out 30 | -------------------------------------------------------------------------------- /tf_euler/python/mp_utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | from tf_euler.python.mp_utils import utils 21 | -------------------------------------------------------------------------------- /tf_euler/python/mp_utils/base_graph.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import tensorflow as tf 21 | import tf_euler 22 | 23 | 24 | class GraphModel(object): 25 | def __init__(self, label_dim): 26 | self.out_fc = tf.layers.Dense(label_dim, use_bias=False) 27 | 28 | def embed(self, n_id): 29 | raise NotImplementedError 30 | 31 | def __call__(self, inputs, label=None, graph_index=None): 32 | if isinstance(inputs, dict): 33 | label = inputs['graph_label'] 34 | graph_index = inputs['node_graph_idx'] 35 | inputs = inputs['node_idx'] 36 | assert (label is not None or graph_index is not None) 37 | graph_index = tf.cast(graph_index, tf.int32) 38 | embedding = self.embed(inputs, graph_index) 39 | logit = self.out_fc(embedding) 40 | label = tf.cast(label, tf.float32) 41 | 42 | _, acc = tf.metrics.accuracy( 43 | label, tf.floor(tf.nn.sigmoid(logit) + 0.5)) 44 | loss = tf.nn.sigmoid_cross_entropy_with_logits( 45 | labels=label, logits=logit) 46 | loss = tf.reduce_mean(loss) 47 | return (embedding, loss, 'accuracy', acc) 48 | -------------------------------------------------------------------------------- /tf_euler/python/node_pool/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | -------------------------------------------------------------------------------- /tf_euler/python/solution/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | from tf_euler.python.solution.base_sample import SuperviseSampleSolution, UnsuperviseSampleSolution 21 | from tf_euler.python.solution.base_supervise import SuperviseSolution 22 | from tf_euler.python.solution.base_unsupervise import UnsuperviseSolution 23 | from tf_euler.python.solution.logits import DenseLogits, PosNegLogits, CosineLogits 24 | from tf_euler.python.solution.losses import sigmoid_loss, xent_loss 25 | from tf_euler.python.solution.samplers import SampleNegWithTypes, SamplePosWithTypes 26 | from tf_euler.python.solution.utils import GetLabelFromFea 27 | -------------------------------------------------------------------------------- /tf_euler/python/solution/base_supervise.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import tensorflow as tf 21 | import tf_euler 22 | 23 | from tf_euler.python.solution.losses import sigmoid_loss 24 | 25 | 26 | class SuperviseSolution(object): 27 | def __init__(self, 28 | get_label_fn, 29 | encoder_fn, 30 | logit_fn, 31 | metric_name='f1', 32 | loss_fn=sigmoid_loss): 33 | self.get_label_fn = get_label_fn 34 | self.metric_name = metric_name 35 | self.metric_class = tf_euler.utils.metrics.get(metric_name) 36 | self.encoder = encoder_fn 37 | self.logit_fn = logit_fn 38 | self.loss_fn = loss_fn 39 | 40 | def embed(self, n_id): 41 | return self.encoder(n_id) 42 | 43 | def __call__(self, inputs): 44 | label = self.get_label_fn(inputs) 45 | embedding = self.embed(inputs) 46 | logit = self.logit_fn(embedding) 47 | 48 | metric = self.metric_class(label, tf.nn.sigmoid(logit)) 49 | loss = self.loss_fn(label, logit) 50 | return (embedding, loss, self.metric_name, metric) 51 | -------------------------------------------------------------------------------- /tf_euler/python/solution/logits.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import tensorflow as tf 21 | import tf_euler 22 | 23 | class DenseLogits(object): 24 | def __init__(self, logits_dim): 25 | self.out_fc = tf.layers.Dense(logits_dim, use_bias=False) 26 | 27 | def __call__(self, inputs, **kwargs): 28 | return self.out_fc(inputs) 29 | 30 | class PosNegLogits(object): 31 | def __call__(self, emb, pos_emb, neg_emb): 32 | logit = tf.matmul(emb, pos_emb, transpose_b=True) 33 | neg_logit = tf.matmul(emb, neg_emb, transpose_b=True) 34 | return logit, neg_logit 35 | 36 | class CosineLogits(object): 37 | def __call__(self, target_emb, context_emb): 38 | normalized_x = tf.nn.l2_normalize(target_emb, axis=-1) 39 | normalized_y = tf.nn.l2_normalize(context_emb, axis=-1) 40 | logits = tf.reduce_sum(normalized_x * normalized_y, -1, True) 41 | logits = logits * 5.0 42 | return logits 43 | 44 | 45 | -------------------------------------------------------------------------------- /tf_euler/python/solution/losses.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import tensorflow as tf 21 | 22 | def sigmoid_loss(labels, logits): 23 | loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits) 24 | loss = tf.reduce_mean(loss) 25 | return loss 26 | 27 | def xent_loss(logits, neg_logits): 28 | true_xent = tf.nn.sigmoid_cross_entropy_with_logits( 29 | labels=tf.ones_like(logits), logits=logits) 30 | negative_xent = tf.nn.sigmoid_cross_entropy_with_logits( 31 | labels=tf.zeros_like(neg_logits), logits=neg_logits) 32 | loss = tf.reduce_mean(tf.concat([tf.reshape(true_xent, [-1, 1]), tf.reshape(negative_xent, [-1, 1])], 0)) 33 | return loss 34 | -------------------------------------------------------------------------------- /tf_euler/python/solution/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import tensorflow as tf 21 | import tf_euler 22 | 23 | class GetLabelFromFea(object): 24 | def __init__(self, label_idx, label_dim): 25 | self.label_idx = label_idx 26 | self.label_dim = label_dim 27 | 28 | def __call__(self, inputs): 29 | label, = tf_euler.get_dense_feature(inputs, 30 | [self.label_idx], 31 | [self.label_dim]) 32 | return label 33 | -------------------------------------------------------------------------------- /tf_euler/python/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | from tf_euler.python.utils import embedding 21 | from tf_euler.python.utils import flags 22 | from tf_euler.python.utils import hooks 23 | from tf_euler.python.utils import aggregators 24 | from tf_euler.python.utils import encoders 25 | from tf_euler.python.utils import layers 26 | from tf_euler.python.utils import metrics 27 | from tf_euler.python.utils import optimizers 28 | from tf_euler.python.utils.to_dense_batch import to_dense_batch 29 | from tf_euler.python.utils.to_dense_adj import to_dense_adj 30 | -------------------------------------------------------------------------------- /tf_euler/python/utils/flags.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import tensorflow as tf 21 | 22 | 23 | def set_defaults(**kwargs): 24 | for key, value in kwargs.items(): 25 | tf.flags.FLAGS.set_default(name=key, value=value) 26 | -------------------------------------------------------------------------------- /tf_euler/python/utils/hooks.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import time 21 | 22 | import tensorflow as tf 23 | 24 | 25 | class SyncExitHook(tf.train.SessionRunHook): 26 | def __init__(self, num_workers): 27 | self._num_workers = num_workers 28 | self._num_finished_workers = tf.Variable( 29 | 0, name="num_finished_workers") 30 | self._finish_self = tf.assign_add( 31 | self._num_finished_workers, 1, use_locking=True) 32 | 33 | def end(self, session): 34 | session.run(self._finish_self) 35 | num_finished_workers = session.run(self._num_finished_workers) 36 | while num_finished_workers < self._num_workers: 37 | tf.logging.info("%d workers have finished ...", 38 | num_finished_workers) 39 | time.sleep(1) 40 | num_finished_workers = session.run(self._num_finished_workers) 41 | -------------------------------------------------------------------------------- /tf_euler/python/utils/optimizers.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import tensorflow as tf 21 | 22 | optimizers = { 23 | 'sgd': lambda lr: tf.train.MomentumOptimizer(lr, 0.0), 24 | 'momentum': lambda lr: tf.train.MomentumOptimizer(lr, 0.9), 25 | 'adagrad': tf.train.AdagradOptimizer, 26 | 'adam': tf.train.AdamOptimizer 27 | } 28 | 29 | 30 | def get(optimizer): 31 | return optimizers.get(optimizer) 32 | -------------------------------------------------------------------------------- /tf_euler/python/utils/to_dense_batch.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | import tensorflow as tf 17 | from tf_euler.python.euler_ops import mp_ops 18 | 19 | 20 | def to_dense_batch(x, batch): 21 | assert batch is not None 22 | batch_size = tf.reduce_max(batch) + 1 23 | num_nodes = mp_ops.scatter_('add', tf.ones([tf.shape(batch)[0], 1]), 24 | batch, batch_size) 25 | num_nodes = tf.cast(tf.reshape(num_nodes, [-1]), dtype=tf.int32) 26 | 27 | cum_nodes = tf.concat([tf.zeros(1, dtype=tf.int32), 28 | tf.cumsum(num_nodes, axis=0)], axis=0) 29 | max_num_nodes = tf.reduce_max(num_nodes) 30 | 31 | idx = tf.range(tf.reduce_sum(num_nodes)) 32 | 33 | n = tf.gather(cum_nodes, batch) 34 | idx = idx - n + batch * max_num_nodes 35 | 36 | idx = tf.reshape(idx, [-1, 1]) 37 | 38 | size = [batch_size * max_num_nodes, tf.shape(x)[-1]] 39 | 40 | out = tf.scatter_nd(idx, x, shape=size) 41 | 42 | out_size = [batch_size, max_num_nodes, tf.shape(x)[-1]] 43 | out = tf.reshape(out, out_size) 44 | 45 | mask = tf.scatter_nd(idx, 46 | tf.ones(tf.shape(batch)[0]), 47 | shape=[batch_size * max_num_nodes]) 48 | return out, out_size, mask 49 | -------------------------------------------------------------------------------- /tf_euler/python/utils/to_dense_batch_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from tensorflow.python.platform import test 17 | from tf_euler.python.utils import to_dense_batch 18 | 19 | import tensorflow as tf 20 | 21 | 22 | class ToDenseBatchTest(test.TestCase): 23 | def test_to_dense_batch(self): 24 | x = tf.constant([ 25 | [1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12] 26 | ]) 27 | batch = tf.constant([0, 0, 1, 2, 2, 2]) 28 | 29 | out_op, mask_op = to_dense_batch(x, batch) 30 | 31 | init = tf.global_variables_initializer() 32 | with self.test_session() as sess: 33 | sess.run(init) 34 | out, mask = sess.run([out_op, mask_op]) 35 | 36 | expected = [ 37 | [[1, 2], [3, 4], [0, 0]], 38 | [[5, 6], [0, 0], [0, 0]], 39 | [[7, 8], [9, 10], [11, 12]], 40 | ] 41 | 42 | assert out.shape == (3, 3, 2) 43 | assert out.tolist() == expected 44 | assert mask.tolist() == [1, 1, 0, 1, 0, 0, 1, 1, 1] 45 | 46 | 47 | if __name__ == '__main__': 48 | test.main() 49 | -------------------------------------------------------------------------------- /tf_euler/scripts/dist_tf_euler.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eux 3 | NUM_PSES=2 4 | NUM_WORKERS=2 5 | 6 | PS_HOSTS="" 7 | for I in $(seq $(($NUM_PSES - 1)) -1 0) 8 | do 9 | PS_HOST="localhost:$((1999 - $I))" 10 | if [ "$PS_HOSTS" == "" ]; then 11 | PS_HOSTS="$PS_HOST" 12 | else 13 | PS_HOSTS="$PS_HOSTS,$PS_HOST" 14 | fi 15 | done 16 | 17 | WORKER_HOSTS="" 18 | for I in $(seq 0 $(($NUM_WORKERS - 1))) 19 | do 20 | WORKER_HOST="localhost:$((2000 + $I))" 21 | if [ "$WORKER_HOSTS" == "" ]; then 22 | WORKER_HOSTS="$WORKER_HOST" 23 | else 24 | WORKER_HOSTS="$WORKER_HOSTS,$WORKER_HOST" 25 | fi 26 | done 27 | 28 | for I in $(seq 0 $(($NUM_PSES - 1))) 29 | do 30 | python -m tf_euler \ 31 | --ps_hosts=$PS_HOSTS \ 32 | --worker_hosts=$WORKER_HOSTS \ 33 | --job_name=ps --task_index=$I &> /tmp/log.ps.$I & 34 | done 35 | 36 | for I in $(seq 0 $(($NUM_WORKERS - 1))) 37 | do 38 | python -m tf_euler \ 39 | --ps_hosts=$PS_HOSTS \ 40 | --worker_hosts=$WORKER_HOSTS \ 41 | --job_name=worker --task_index=$I $@ &> /tmp/log.worker.$I & 42 | WORKERS[$I]=$! 43 | done 44 | 45 | trap 'kill $(jobs -p) 2> /dev/null; exit' SIGINT SIGTERM 46 | 47 | wait ${WORKERS[*]} 48 | 49 | kill $(jobs -p) 2> /dev/null 50 | -------------------------------------------------------------------------------- /tf_euler/scripts/run_ppi.sh: -------------------------------------------------------------------------------- 1 | set -eux 2 | ./dist_tf_euler.sh \ 3 | --data_dir /tmp/ppi_data/ \ 4 | --euler_zk_addr 127.0.0.1:2181 --euler_zk_path /euler-2.0ppi \ 5 | --max_id 56944 --feature_idx f1 --feature_dim 50 --label_idx label --label_dim 121 \ 6 | --model graphsage_supervised --mode train 7 | -------------------------------------------------------------------------------- /tf_euler/scripts/start_service.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | import euler 17 | import sys 18 | import time 19 | 20 | 21 | if __name__ == '__main__': 22 | shard_idx = sys.argv[1] 23 | shard_num = sys.argv[2] 24 | 25 | euler.start(directory='/tmp/ppi_data', 26 | shard_idx=shard_idx, 27 | shard_num=shard_num, 28 | zk_addr='127.0.0.1:2181', 29 | zk_path='/euler-2.0ppi-test', 30 | global_sampler_type='all', 31 | server_thread_num=4) 32 | 33 | time.sleep(100000) 34 | -------------------------------------------------------------------------------- /tf_euler/utils/euler_query_proxy.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TF_EULER_UTILS_EULER_QUERY_PROXY_H_ 17 | #define TF_EULER_UTILS_EULER_QUERY_PROXY_H_ 18 | 19 | #include "euler/client/query_proxy.h" 20 | #include "euler/client/query.h" 21 | #include "euler/core/framework/tensor_shape.h" 22 | #include "euler/core/framework/tensor.h" 23 | #include "euler/common/logging.h" 24 | #include "euler/core/framework/types.h" 25 | 26 | #endif // TF_EULER_UTILS_EULER_QUERY_PROXY_H_ 27 | -------------------------------------------------------------------------------- /tf_euler/utils/init_query_proxy.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tf_euler/utils/euler_query_proxy.h" 17 | #include "euler/common/str_util.h" 18 | 19 | extern "C" { 20 | bool InitQueryProxy(const char* conf) { 21 | euler::GraphConfig config; 22 | auto vec = euler::Split(conf, ';'); 23 | if (vec.empty()) { 24 | return false; 25 | } 26 | for (auto it = vec.begin(); it != vec.end(); ++it) { 27 | auto key_value = euler::Split(*it, '='); 28 | if (key_value.size() != 2 || key_value[0].empty() || 29 | key_value[1].empty()) { 30 | return false; 31 | } 32 | config.Add(key_value[0], key_value[1]); 33 | } 34 | euler::QueryProxy::Init(config); 35 | return true; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /third_party/CMakeLists.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/euler/c2a71faae59c1495b6dabcf6aec0acb4d93a7bb1/third_party/CMakeLists.txt -------------------------------------------------------------------------------- /tools/.gitignore: -------------------------------------------------------------------------------- 1 | *.jar 2 | *.class 3 | *.meta 4 | *.json 5 | *.log 6 | 7 | -------------------------------------------------------------------------------- /tools/CMakeLists.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/euler/c2a71faae59c1495b6dabcf6aec0acb4d93a7bb1/tools/CMakeLists.txt -------------------------------------------------------------------------------- /tools/pip/build_wheel.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2018 Alibaba Group Holding Limited. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | set -e 18 | 19 | TF_VERSION="1.12.0" 20 | 21 | SCRIPT_DIR=$(dirname "$0") 22 | PROJECT_DIR="$SCRIPT_DIR/../.." 23 | cd "$PROJECT_DIR" 24 | 25 | python tools/pip/setup.py bdist_wheel --python-tag cp27 -p manylinux1_x86_64 --tf_version "$TF_VERSION" 26 | -------------------------------------------------------------------------------- /tools/start_zk/zk_start.sh: -------------------------------------------------------------------------------- 1 | rm -fr /tmp/zookeeper 2 | ../../third_party/zookeeper/bin/zkServer.sh --config . start 3 | exit 0 4 | -------------------------------------------------------------------------------- /tools/start_zk/zk_stop.sh: -------------------------------------------------------------------------------- 1 | ../../third_party/zookeeper/bin/zkServer.sh --config . stop 2 | -------------------------------------------------------------------------------- /tools/start_zk/zoo.cfg: -------------------------------------------------------------------------------- 1 | # The number of milliseconds of each tick 2 | tickTime=2000 3 | # The number of ticks that the initial 4 | # synchronization phase can take 5 | initLimit=10 6 | # The number of ticks that can pass between 7 | # sending a request and getting an acknowledgement 8 | syncLimit=5 9 | # the directory where the snapshot is stored. 10 | # do not use /tmp for storage, /tmp here is just 11 | # example sakes. 12 | dataDir=/tmp/zookeeper 13 | # the port at which the clients will connect 14 | clientPort=2181 15 | # the maximum number of client connections. 16 | # increase this if you need to handle more clients 17 | #maxClientCnxns=60 18 | # 19 | # Be sure to read the maintenance section of the 20 | # administrator guide before turning on autopurge. 21 | # 22 | # http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance 23 | # 24 | # The number of snapshots to retain in dataDir 25 | #autopurge.snapRetainCount=3 26 | # Purge task interval in hours 27 | # Set to "0" to disable auto purge feature 28 | #autopurge.purgeInterval=1 29 | -------------------------------------------------------------------------------- /tools/test_data/meta: -------------------------------------------------------------------------------- 1 | { 2 | "node": 3 | { 4 | "type" : "node_type:int32_t:uint64_t:hash_index", 5 | "features":{ 6 | "f4": 7 | { 8 | "1":"price:float:uint64_t:range_index" 9 | }, 10 | "f3": 11 | { 12 | "0":"att:float:uint64_t:neighbor_index" 13 | }, 14 | "graph_label": "graph_label:string:uint64_t:hash_index" 15 | } 16 | }, 17 | "edge": 18 | { 19 | "type" : "edge_type:int32_t:uint64_t:hash_index", 20 | "features":{ 21 | "f3": 22 | { 23 | "0":"edge_value:float:uint64_t:range_index" 24 | }, 25 | "f4": 26 | { 27 | "1":"edge_att:float:uint64_t:neighbor_index" 28 | } 29 | } 30 | } 31 | } 32 | --------------------------------------------------------------------------------