├── .clang-format ├── .dockerignore ├── .gitattributes ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ └── docs.yaml ├── .gitignore ├── .gitmodules ├── .pre-commit-config.yaml ├── CMakeLists.txt ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md ├── VERSION ├── apps ├── age │ ├── init-age.sql │ ├── sssp-age-ud.sql │ ├── sssp-recursive-d.sql │ ├── sssp-recursive-ud.sql │ └── test-age.py ├── analytical_engine │ ├── .clang-format │ ├── .gitignore │ ├── CMakeLists.txt │ ├── README-zh.md │ ├── README.md │ ├── apps │ │ └── gart │ │ │ ├── property_pagerank.h │ │ │ ├── property_sssp.h │ │ │ └── property_wcc.h │ ├── cmake │ │ ├── CheckGCCABICompatibility.cmake │ │ ├── FindArrow.cmake │ │ ├── FindGFlags.cmake │ │ ├── FindGRPC.cmake │ │ ├── FindGlog.cmake │ │ ├── FindLibUnwind.cmake │ │ └── FindRdkafka.cmake │ ├── core │ │ ├── app │ │ │ ├── app_base.h │ │ │ ├── app_invoker.h │ │ │ ├── parallel_property_app_base.h │ │ │ ├── pregel │ │ │ │ ├── aggregators │ │ │ │ │ ├── aggregator.h │ │ │ │ │ ├── aggregator_factory.h │ │ │ │ │ ├── bool_aggregator.h │ │ │ │ │ ├── numeric_aggregator.h │ │ │ │ │ └── text_aggregator.h │ │ │ │ ├── cython_vertex_program.h │ │ │ │ ├── export.h │ │ │ │ ├── i_vertex_program.h │ │ │ │ ├── pregel_app_base.h │ │ │ │ ├── pregel_compute_context.h │ │ │ │ ├── pregel_context.h │ │ │ │ ├── pregel_property_app_base.h │ │ │ │ ├── pregel_property_vertex.h │ │ │ │ └── pregel_vertex.h │ │ │ ├── property_app_base.h │ │ │ └── property_auto_app_base.h │ │ ├── communication │ │ │ └── shuffle.h │ │ ├── config.h │ │ ├── context │ │ │ ├── column.h │ │ │ ├── context_protocols.h │ │ │ ├── gart_vertex_data_context.h │ │ │ ├── i_context.h │ │ │ ├── java_context_base.h │ │ │ ├── java_pie_projected_context.h │ │ │ ├── java_pie_property_context.h │ │ │ ├── labeled_vertex_property_context.h │ │ │ ├── mpi_object_sync.h │ │ │ ├── selector.h │ │ │ ├── tensor_context.h │ │ │ ├── tensor_dataframe_builder.h │ │ │ ├── vertex_data_context.h │ │ │ └── vertex_property_context.h │ │ ├── error.h │ │ ├── flags.cc │ │ ├── flags.h │ │ ├── fragment │ │ │ ├── arrow_flattened_fragment.h │ │ │ ├── arrow_projected_fragment.h │ │ │ ├── arrow_projected_fragment_base.h │ │ │ ├── arrow_projected_fragment_mapper.h │ │ │ ├── dynamic_fragment.h │ │ │ ├── dynamic_projected_fragment.h │ │ │ └── fragment_reporter.h │ │ ├── grape_engine.cc │ │ ├── grape_instance.cc │ │ ├── grape_instance.h │ │ ├── io │ │ │ └── property_parser.h │ │ ├── java │ │ │ ├── fragment_getter.h │ │ │ ├── graphx_client.h │ │ │ ├── graphx_loader.h │ │ │ ├── graphx_raw_data.h │ │ │ ├── graphx_runner.cc │ │ │ ├── graphx_runner.h │ │ │ ├── java_loader_invoker.h │ │ │ ├── java_messages.h │ │ │ ├── javasdk.cc │ │ │ ├── javasdk.h │ │ │ ├── rdd_transfer_client.h │ │ │ ├── type_alias.h │ │ │ └── utils.h │ │ ├── launcher.cc │ │ ├── launcher.h │ │ ├── loader │ │ │ ├── arrow_fragment_loader.h │ │ │ ├── arrow_to_dynamic_converter.h │ │ │ └── dynamic_to_arrow_converter.h │ │ ├── object │ │ │ ├── app_entry.h │ │ │ ├── dynamic.cc │ │ │ ├── dynamic.h │ │ │ ├── fragment_wrapper.h │ │ │ ├── graph_utils.h │ │ │ ├── gs_object.h │ │ │ ├── i_fragment_wrapper.h │ │ │ ├── object_manager.h │ │ │ └── projector.h │ │ ├── parallel │ │ │ ├── gart_parallel.h │ │ │ ├── parallel_property_message_manager.h │ │ │ ├── property_auto_message_manager.h │ │ │ ├── property_message_manager.h │ │ │ └── thread_local_property_message_buffer.h │ │ ├── server │ │ │ ├── analytical_server.cc │ │ │ ├── analytical_server.h │ │ │ ├── command_detail.cc │ │ │ ├── command_detail.h │ │ │ ├── dispatcher.cc │ │ │ ├── dispatcher.h │ │ │ ├── graphscope_service.cc │ │ │ ├── graphscope_service.h │ │ │ └── rpc_utils.h │ │ ├── utils │ │ │ ├── convert_utils.h │ │ │ ├── fragment_traits.h │ │ │ ├── gart_vertex_array.h │ │ │ ├── lib_utils.h │ │ │ ├── mpi_utils.h │ │ │ ├── msgpack_utils.h │ │ │ ├── partitioner.h │ │ │ ├── trait_utils.h │ │ │ ├── transform_utils.h │ │ │ └── trivial_tensor.h │ │ ├── vertex_map │ │ │ ├── arrow_projected_vertex_map.h │ │ │ ├── extra_vertex_map.h │ │ │ └── global_vertex_map.h │ │ └── worker │ │ │ ├── default_property_worker.h │ │ │ ├── default_worker.h │ │ │ ├── parallel_property_worker.h │ │ │ └── property_auto_worker.h │ ├── exported_symbols_osx.lds │ ├── frame │ │ ├── app_frame.cc │ │ ├── ctx_wrapper_builder.h │ │ ├── cython_app_frame.cc │ │ ├── cython_pie_app_frame.cc │ │ ├── flash_app_frame.cc │ │ ├── project_frame.cc │ │ └── property_graph_frame.cc │ ├── graphscope-analytical-config-version.in.cmake │ ├── graphscope-analytical-config.in.cmake │ ├── misc │ │ └── cpplint.py │ ├── python │ │ └── graphscope │ │ │ ├── VERSION │ │ │ ├── __init__.py │ │ │ ├── config.py │ │ │ ├── proto │ │ │ ├── __init__.py │ │ │ ├── attr_value.proto │ │ │ ├── coordinator_service.proto │ │ │ ├── data_types.proto │ │ │ ├── ddl_service.proto │ │ │ ├── engine_service.proto │ │ │ ├── error_codes.proto │ │ │ ├── graph_def.proto │ │ │ ├── message.proto │ │ │ ├── op_def.proto │ │ │ ├── proto_generator.py │ │ │ ├── types.proto │ │ │ └── write_service.proto │ │ │ └── version.py │ └── test │ │ ├── app_tests.sh │ │ ├── flags.cc │ │ ├── flags.h │ │ ├── gart_test.sh │ │ ├── giraph_runner.cc │ │ ├── giraph_runner.h │ │ ├── graphx_loader_test.cc │ │ ├── projected_fragment_mapper_test.cc │ │ ├── run_app.cc │ │ ├── run_app.h │ │ ├── run_ctx.cc │ │ ├── run_empty_property.cc │ │ ├── run_gart_app.cc │ │ ├── run_gart_reader.cc │ │ ├── run_java_app.cc │ │ ├── run_java_string_app.cc │ │ ├── run_load_from_stream.cc │ │ ├── run_pregel_app.cc │ │ ├── run_property_ctx.cc │ │ ├── run_string_oid.cc │ │ ├── run_vy_app.cc │ │ ├── run_vy_app_local_vm.cc │ │ ├── run_vy_ldbc.cc │ │ ├── test_convert.cc │ │ ├── test_project_string.cc │ │ └── test_string_vertex_map.cc ├── interactive_engine │ └── gremlin.py ├── networkx │ ├── CMakeLists.txt │ ├── client │ │ ├── python │ │ │ ├── setup.py │ │ │ └── src │ │ │ │ └── gart │ │ │ │ ├── __init__.py │ │ │ │ ├── archieve.py │ │ │ │ ├── client.py │ │ │ │ ├── coreviews.py │ │ │ │ ├── dict_factory.py │ │ │ │ ├── digraph.py │ │ │ │ ├── local_graph.py │ │ │ │ ├── proto │ │ │ │ └── __init__.py │ │ │ │ └── reportviews.py │ │ └── test │ │ │ ├── algorithm.py │ │ │ ├── networkx_graph.py │ │ │ ├── test.py │ │ │ ├── test_sssp.py │ │ │ └── test_weighted_sssp.py │ ├── cmake │ │ └── FindGRPC.cmake │ ├── proto │ │ ├── proto_generator.py │ │ └── types.proto │ ├── python_bindings │ │ ├── CMakeLists.txt │ │ ├── fragment_builder.cc │ │ ├── fragment_builder.h │ │ └── python_bindings.cc │ └── server │ │ ├── graph_reporter.h │ │ ├── graph_server.cc │ │ ├── graph_server_flags.cc │ │ ├── graph_server_flags.h │ │ └── utils │ │ ├── dynamic.cc │ │ ├── dynamic.h │ │ ├── msgpack_utils.h │ │ └── property_converter.h ├── pgx │ ├── Makefile │ ├── compile_commands.json │ ├── gart--0.0.1.sql │ ├── gart-pgx-config-template.ini │ ├── gart.c │ ├── gart.control │ ├── run.sh │ ├── utility.c │ └── utility.h └── rdbms │ ├── init_schema.py │ ├── insert_db.py │ └── insert_db_txn.py ├── charts └── gart │ ├── .helmignore │ ├── Chart.yaml │ ├── templates │ ├── _helpers.tpl │ ├── auth │ │ └── role_and_binding.yaml │ ├── configmap.yaml │ ├── converter │ │ ├── deployment.yaml │ │ └── svc.yaml │ ├── debezium │ │ ├── configmap.yaml │ │ ├── deployment.yaml │ │ └── svc.yaml │ ├── gie_frontend │ │ ├── deployment.yaml │ │ └── svc.yaml │ └── writer │ │ ├── statefulset.yaml │ │ └── svc.yaml │ └── values.yaml ├── converter ├── CMakeLists.txt ├── binlog_convert.cc ├── debzium_log_format.json ├── flags.cc ├── flags.h ├── json2yaml.cc ├── kafka_helper.h ├── parser.cc └── parser.h ├── coordinator ├── .dockerignore ├── .travis.yml ├── Dockerfile ├── README.md ├── flex │ └── server │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── controllers │ │ ├── __init__.py │ │ ├── alert_controller.py │ │ ├── data_source_controller.py │ │ ├── deployment_controller.py │ │ ├── graph_controller.py │ │ ├── job_controller.py │ │ ├── security_controller.py │ │ ├── service_controller.py │ │ ├── stored_procedure_controller.py │ │ └── utils_controller.py │ │ ├── encoder.py │ │ ├── models │ │ ├── __init__.py │ │ ├── base_edge_type.py │ │ ├── base_edge_type_vertex_type_pair_relations_inner.py │ │ ├── base_edge_type_vertex_type_pair_relations_inner_x_csr_params.py │ │ ├── base_model.py │ │ ├── base_property_meta.py │ │ ├── base_vertex_type.py │ │ ├── base_vertex_type_x_csr_params.py │ │ ├── change_graph_version_request.py │ │ ├── column_mapping.py │ │ ├── column_mapping_column.py │ │ ├── create_alert_receiver_request.py │ │ ├── create_alert_rule_request.py │ │ ├── create_dataloading_job_response.py │ │ ├── create_edge_type.py │ │ ├── create_graph_by_pgql_request.py │ │ ├── create_graph_by_yaml_request.py │ │ ├── create_graph_request.py │ │ ├── create_graph_response.py │ │ ├── create_graph_schema_request.py │ │ ├── create_property_meta.py │ │ ├── create_stored_proc_request.py │ │ ├── create_stored_proc_response.py │ │ ├── create_vertex_type.py │ │ ├── dataloading_job_config.py │ │ ├── dataloading_job_config_edges_inner.py │ │ ├── dataloading_job_config_loading_config.py │ │ ├── dataloading_job_config_loading_config_format.py │ │ ├── dataloading_job_config_vertices_inner.py │ │ ├── dataloading_mr_job_config.py │ │ ├── date_type.py │ │ ├── edge_mapping.py │ │ ├── edge_mapping_type_triplet.py │ │ ├── error.py │ │ ├── get_alert_message_response.py │ │ ├── get_alert_receiver_response.py │ │ ├── get_alert_rule_response.py │ │ ├── get_edge_type.py │ │ ├── get_graph_response.py │ │ ├── get_graph_schema_response.py │ │ ├── get_graph_version_response.py │ │ ├── get_pod_log_response.py │ │ ├── get_property_meta.py │ │ ├── get_resource_usage_response.py │ │ ├── get_storage_usage_response.py │ │ ├── get_stored_proc_response.py │ │ ├── get_vertex_type.py │ │ ├── gs_data_type.py │ │ ├── job_status.py │ │ ├── long_text.py │ │ ├── node_status.py │ │ ├── parameter.py │ │ ├── pod_status.py │ │ ├── primitive_type.py │ │ ├── resource_usage.py │ │ ├── running_deployment_info.py │ │ ├── running_deployment_status.py │ │ ├── schema_mapping.py │ │ ├── service_status.py │ │ ├── service_status_sdk_endpoints.py │ │ ├── start_service_request.py │ │ ├── stored_procedure_meta.py │ │ ├── string_type.py │ │ ├── string_type_string.py │ │ ├── temporal_type.py │ │ ├── temporal_type_temporal.py │ │ ├── time_stamp_type.py │ │ ├── update_alert_message_status_request.py │ │ ├── update_stored_proc_request.py │ │ ├── upload_file_response.py │ │ └── vertex_mapping.py │ │ ├── openapi │ │ └── openapi.yaml │ │ ├── test │ │ ├── __init__.py │ │ ├── test_alert_controller.py │ │ ├── test_data_source_controller.py │ │ ├── test_deployment_controller.py │ │ ├── test_graph_controller.py │ │ ├── test_job_controller.py │ │ ├── test_service_controller.py │ │ ├── test_stored_procedure_controller.py │ │ └── test_utils_controller.py │ │ ├── typing_utils.py │ │ └── util.py ├── requirements.txt ├── setup.py ├── test-requirements.txt └── tox.ini ├── docs ├── .nojekyll ├── Doxyfile ├── Makefile ├── _static │ ├── css │ │ ├── brands.min.css │ │ ├── custom.css │ │ ├── index.css │ │ ├── panels.css │ │ ├── termynal.css │ │ └── v4-shims.min.css │ ├── js │ │ └── termynal.js │ └── webfonts │ │ ├── fa-brands-400.ttf │ │ └── fa-brands-400.woff2 ├── _templates │ ├── base.html │ ├── index.html │ ├── layout.html │ └── page.html ├── conf.py ├── docs.rst ├── documentation │ ├── deployment │ │ ├── deploy-distributed.rst │ │ ├── deploy-k8s.rst │ │ └── env-setup.rst │ ├── design │ │ ├── architecture.rst │ │ ├── graph-storage.rst │ │ └── rgmapping.rst │ ├── getting-started │ │ ├── intro-content.rst │ │ ├── introduction.rst │ │ ├── quick-start.rst │ │ └── use-cases.rst │ └── tutorials │ │ ├── data-source-config.rst │ │ ├── data-source-config │ │ ├── mysql.rst │ │ ├── postgresql.rst │ │ └── storage.md │ │ ├── log-capturer-config.rst │ │ ├── pgx.rst │ │ ├── rgmapping-config.rst │ │ └── run-gap.rst └── images │ ├── arch.png │ ├── graph-arch.png │ ├── ldbc-snb.jpg │ ├── rgmapping.png │ └── use-case-fr.png ├── interfaces ├── fragment │ ├── gart_fragment.h │ ├── gart_fragment_traits.h │ ├── iterator.h │ ├── property_util.h │ └── types.h └── grin │ ├── CMakeLists.txt │ ├── predefine.h │ ├── rust │ ├── Cargo.toml │ ├── gart_all.h │ ├── grin_gart.rs │ └── parse.py │ ├── src │ ├── common │ │ ├── error.cc │ │ └── message.cc │ ├── index │ │ ├── external_id.cc │ │ └── internal_id.cc │ ├── partition │ │ ├── partition.cc │ │ ├── reference.cc │ │ └── topology.cc │ ├── predefine.cc │ ├── predefine.h │ ├── property │ │ ├── property.cc │ │ ├── propertylist.cc │ │ ├── row.cc │ │ ├── topology.cc │ │ ├── type.cc │ │ └── value.cc │ └── topology │ │ ├── adjacentlist.cc │ │ ├── structure.cc │ │ └── vertexlist.cc │ └── test │ ├── run_grin_reader.cc │ └── test.cc ├── k8s ├── converter-deployment.yaml ├── debezium-config-template.yaml ├── debezium-deployment.yaml ├── debezium-job.yaml ├── dockerfiles │ └── interactive.Dockerfile ├── etcd-deployment.yaml ├── gart-config-template.yaml ├── gart-namespace.yaml ├── kafka-deployment.yaml ├── metallb-config.yaml ├── writer-deployment-template.yaml └── zk-deployment.yaml ├── pgql ├── CMakeLists.txt ├── pom.xml ├── run.sh └── src │ └── main │ └── java │ └── gart │ └── pgql │ ├── CustomRepresenter.java │ ├── CustomYaml.java │ ├── GSchema.java │ ├── Main.java │ ├── PgqlConverter.java │ ├── YamlConverter.java │ └── YamlFormatter.java ├── requirements-dev.txt ├── scripts ├── check_process.sh ├── controller.py ├── cpplint.py ├── distributed_deployment.py ├── extract_table_schema.py ├── gart ├── gart_cli.py ├── generate_schema_for_protal.py ├── install-deps.sh ├── install-mysql.sh ├── install-psql.sh ├── k8s_deployment.py ├── kube_ssh ├── launch_etcd.py ├── launch_gie_executor.sh ├── pause_resume_data_processing.sh ├── stop-gart ├── update_debezium_config.py ├── update_gie_executor_config.py ├── update_gie_schema.py ├── update_ip_tables.sh └── update_kafka_config_file.py └── vegito ├── CMakeLists.txt ├── cmake ├── FindGFlags.cmake ├── FindGlog.cmake ├── FindRdkafka.cmake └── FindTBB.cmake ├── include ├── fragment │ ├── id_parser.h │ └── shared_storage.h ├── seggraph │ ├── block_manager.hpp │ ├── blocks.hpp │ ├── bloom_filter.hpp │ ├── edge_iterator.hpp │ ├── epoch_graph_reader.hpp │ ├── epoch_graph_writer.hpp │ ├── futex.hpp │ ├── seggraph.hpp │ ├── segment_graph.hpp │ ├── types.hpp │ └── utils.hpp └── util │ ├── allocator.hpp │ ├── atomic-template.h │ ├── atomic.h │ ├── bitset.h │ ├── inline_str.h │ ├── macros.h │ ├── serializer.h │ ├── spinlock.h │ ├── status.h │ ├── util.h │ └── varint.h ├── src ├── framework │ ├── bench_runner.cc │ ├── bench_runner.h │ ├── config.cc │ └── config.h ├── graph │ ├── ddl.cc │ ├── ddl.h │ ├── graph_ops.h │ ├── graph_ops │ │ ├── add_edge.cc │ │ ├── add_vertex.cc │ │ ├── del_edge.cc │ │ ├── del_vertex.cc │ │ └── update_vertex.cc │ ├── graph_store.cc │ ├── graph_store.h │ └── type_def.h ├── main.cc ├── memory │ ├── buffer_manager.cc │ └── buffer_manager.h ├── property │ ├── property.h │ ├── property_col_array.cc │ ├── property_col_array.h │ ├── property_col_paged.cc │ └── property_col_paged.h ├── seggraph │ ├── epoch_graph_reader.cpp │ ├── epoch_graph_writer.cpp │ └── segment_graph.cpp ├── system_flags.cc └── system_flags.h └── test ├── config ├── connect-debezium-mysql.properties ├── connect-debezium-postgresql.properties └── connect-standalone.properties ├── data └── test_graph.txt ├── load_graph_test.cc ├── run_test.sh └── schema ├── db-schema-ldbc.yaml ├── distributed_config.json ├── financial_transactions.sql ├── k8s_config.json ├── ldbc-data-source.json ├── ldbc-graph-schema.json ├── ldbc.ddl ├── rgmapping-imdb.yaml ├── rgmapping-ldbc-gie.yaml ├── rgmapping-ldbc.json ├── rgmapping-ldbc.sql ├── rgmapping-ldbc.yaml ├── rgmapping-micro.json ├── rgmapping-smallbank.sql └── rgmapping-smallbank.yaml /.clang-format: -------------------------------------------------------------------------------- 1 | BasedOnStyle: Google 2 | DerivePointerAlignment: false 3 | PointerAlignment: Left 4 | Cpp11BracedListStyle: true 5 | IndentCaseLabels: false 6 | AllowShortBlocksOnASingleLine: true 7 | AllowShortLoopsOnASingleLine: false 8 | AllowShortIfStatementsOnASingleLine: false 9 | Standard: 'Cpp11' 10 | SpaceAfterCStyleCast: true 11 | AlignAfterOpenBracket: Align 12 | SortIncludes: true 13 | IncludeBlocks: Preserve 14 | ForEachMacros: 15 | - BOOST_FOREACH -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | analytical_engine/java/** linguist-vendored 2 | interactive_engine/groot-module/** linguist-vendored 3 | 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[BUG] " 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Environment (please complete the following information):** 27 | - GraphScope version: [e.g., v0.1, v0.2.2, master] 28 | - OS: [e.g. MacOS, Linux] 29 | - Version [e.g. 10.15] 30 | - Kubernetes Version [e.g., 1.19] 31 | 32 | **Additional context** 33 | Add any other context about the problem here. 34 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 4 | 5 | ## What do these changes do? 6 | 7 | 8 | 9 | ## Related issue number 10 | 11 | 12 | 13 | Fixes 14 | 15 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "interfaces/grin/include"] 2 | path = interfaces/grin/include 3 | url = https://github.com/GraphScope/GRIN.git 4 | [submodule "third_party/abseil-cpp"] 5 | path = third_party/abseil-cpp 6 | url = https://github.com/abseil/abseil-cpp.git 7 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/awslabs/git-secrets.git 3 | rev: master 4 | hooks: 5 | - id: git-secrets 6 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.0.1 2 | -------------------------------------------------------------------------------- /apps/age/sssp-age-ud.sql: -------------------------------------------------------------------------------- 1 | -- Time: 929599.329 ms (15:29.599) 2 | SELECT * FROM cypher('ldbc', $$ 3 | MATCH paths = (a:Person {id: "2199023256077"})-[:knows*1..4]-(b:Person {id: "933"}) 4 | WITH paths, relationships(paths) AS rels 5 | UNWIND rels AS rel 6 | WITH nodes(paths) AS nodes, 7 | COLLECT(rel) AS knowRels, 8 | COUNT(rel) AS totalLength 9 | RETURN nodes, knowRels, totalLength 10 | $$) AS (nodes agtype, knowRels agtype, totalLength bigint) 11 | ORDER BY totalLength 12 | LIMIT 10; 13 | -------------------------------------------------------------------------------- /apps/age/sssp-recursive-d.sql: -------------------------------------------------------------------------------- 1 | WITH RECURSIVE ShortestPath AS ( 2 | SELECT 3 | p_id AS vertex, 4 | 0 AS distance, 5 | ARRAY[p_id] AS path, 6 | ARRAY[p_id] AS visited 7 | FROM person 8 | WHERE p_id = 933 -- 指定起点ID 9 | 10 | UNION ALL 11 | 12 | SELECT 13 | k.dst AS vertex, 14 | sp.distance + 1 AS distance, 15 | sp.path || k.dst AS path, 16 | sp.visited || k.dst AS visited 17 | FROM ShortestPath sp 18 | JOIN knows k ON sp.vertex = k.src 19 | WHERE NOT k.dst = ANY(sp.visited) 20 | AND sp.vertex <> 2199023256077 -- 这个条件确保我们只在没有到达目标节点时继续递归 21 | AND sp.distance < 5 -- 这个条件确保我们只在距离小于5时继续递归 22 | ) 23 | SELECT vertex, distance, path 24 | FROM ShortestPath 25 | WHERE vertex = 2199023256077 -- 选择到达目标ID的路径 26 | ORDER BY distance 27 | LIMIT 10; -- 由于边权重为10，第一个结果即为最短路径 28 | -------------------------------------------------------------------------------- /apps/age/sssp-recursive-ud.sql: -------------------------------------------------------------------------------- 1 | WITH RECURSIVE ShortestPath AS ( 2 | SELECT 3 | p_id AS vertex, 4 | 0 AS distance, 5 | ARRAY[p_id] AS path, 6 | ARRAY[p_id] AS visited, 7 | 1 AS depth -- 添加递归深度计数器 8 | FROM person 9 | WHERE p_id = 933 10 | UNION ALL 11 | SELECT 12 | -- 考虑无向边，我们需要选择 `k.src` 和 `k.dst` 中的 "另一端" 13 | CASE 14 | WHEN sp.vertex = k.src THEN k.dst 15 | WHEN sp.vertex = k.dst THEN k.src 16 | END AS vertex, 17 | sp.distance + 1 AS distance, 18 | sp.path || CASE 19 | WHEN sp.vertex = k.src THEN k.dst 20 | WHEN sp.vertex = k.dst THEN k.src 21 | END AS path, 22 | sp.visited || CASE 23 | WHEN sp.vertex = k.src THEN k.dst 24 | WHEN sp.vertex = k.dst THEN k.src 25 | END AS visited, 26 | sp.depth + 1 AS depth -- 递增深度 27 | FROM ShortestPath sp 28 | JOIN knows k ON (sp.vertex = k.src OR sp.vertex = k.dst) -- 修改这里的 JOIN 条件 29 | WHERE NOT (CASE 30 | WHEN sp.vertex = k.src THEN k.dst 31 | WHEN sp.vertex = k.dst THEN k.src 32 | END) = ANY(sp.visited) 33 | AND sp.depth < 5 -- 限制递归深度 34 | ) 35 | SELECT vertex, distance, path 36 | FROM ShortestPath 37 | WHERE vertex = 2199023256077 38 | ORDER BY distance 39 | LIMIT 10; 40 | -------------------------------------------------------------------------------- /apps/analytical_engine/.clang-format: -------------------------------------------------------------------------------- 1 | BasedOnStyle: Google 2 | DerivePointerAlignment: false 3 | PointerAlignment: Left 4 | Cpp11BracedListStyle: true 5 | IndentCaseLabels: false 6 | AllowShortBlocksOnASingleLine: true 7 | AllowShortLoopsOnASingleLine: false 8 | AllowShortIfStatementsOnASingleLine: false 9 | Standard: 'Cpp11' 10 | SpaceAfterCStyleCast: true 11 | AlignAfterOpenBracket: Align 12 | SortIncludes: true 13 | IncludeBlocks: Preserve 14 | -------------------------------------------------------------------------------- /apps/analytical_engine/.gitignore: -------------------------------------------------------------------------------- 1 | .cache/ 2 | 3 | -------------------------------------------------------------------------------- /apps/analytical_engine/README-zh.md: -------------------------------------------------------------------------------- 1 | # GraphScope 图分析引擎 - GRAPE 2 | 3 | [![Translation](https://shields.io/badge/README-English-blue)](https://github.com/alibaba/GraphScope/tree/main/analytical_engine) 4 | 5 | 6 | GraphScope 中的图分析引擎继承自 **GRAPE**，该系统实现了论文 [Parallelizing Sequential Graph Computations](https://dl.acm.org/doi/10.1145/3282488) 中提出的不动点计算模型。 7 | 8 | 与现有系统不同，GRAPE 通过自动并行化整体的单机顺序图算法，[即插即用](https://github.com/alibaba/libgrape-lite/blob/master/examples/analytical_apps/sssp/sssp_auto.h)已有的图算法程序，使其很容易的运行在分布式环境，高效处理大规模图。除了易于编程之外，**GRAPE** 还被设计为[高效](https://github.com/alibaba/libgrape-lite/blob/master/Performance.md)和[高度可拓展](https://github.com/alibaba/libgrape-lite/blob/master/examples/gnn_sampler)的，以应对现实图应用程序多变的规模，多样性和复杂性。 9 | 10 | GRAPE 的核心轻量版本以 [libgrape-lite](https://github.com/alibaba/libgrape-lite/) 开源。GraphScope 中的分析引擎扩展了 libgrape-lite 的功能，支持了可变子图，[vineyard](https://github.com/v6d-io/v6d) 支持以及引擎的服务模式等。 11 | 12 | ## Java PIE SDK 13 | 14 | 除了在Python中提供的PIE编程接口之外，GraphScope也提供一个高效的**Java SDK**，借助于它，用户可以在Java中实现图分析算法，并且提交到GraphScope上运行。更多细节请查看[GRAPE-JDK](java/)。 15 | 16 | ## 论文列表 17 | 18 | - Wenfei Fan, Jingbo Xu, Wenyuan Yu, Jingren Zhou, Xiaojian Luo, Ping Lu, Qiang Yin, Yang Cao, and Ruiqi Xu. [Parallelizing Sequential Graph Computations](https://dl.acm.org/doi/10.1145/3282488). ACM Transactions on Database Systems (TODS) 43(4): 18:1-18:39. 19 | 20 | - Wenfei Fan, Jingbo Xu, Yinghui Wu, Wenyuan Yu, Jiaxin Jiang. [GRAPE: Parallelizing Sequential Graph Computations](http://www.vldb.org/pvldb/vol10/p1889-fan.pdf). The 43rd International Conference on Very Large Data Bases (VLDB), demo, 2017 (the Best Demo Award). 21 | 22 | - Wenfei Fan, Jingbo Xu, Yinghui Wu, Wenyuan Yu, Jiaxin Jiang, Zeyu Zheng, Bohan Zhang, Yang Cao, and Chao Tian. [Parallelizing Sequential Graph Computations](https://dl.acm.org/doi/10.1145/3035918.3035942). ACM SIG Conference on Management of Data (SIGMOD), 2017 (the Best Paper Award). 23 | -------------------------------------------------------------------------------- /apps/analytical_engine/cmake/CheckGCCABICompatibility.cmake: -------------------------------------------------------------------------------- 1 | macro(check_gcc_compatible) 2 | if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") 3 | execute_process(COMMAND "${CMAKE_CXX_COMPILER}" -v 4 | OUTPUT_VARIABLE GCC_VERSION_OUT 5 | ERROR_VARIABLE GCC_VERSION_ERR) 6 | if(GCC_VERSION_OUT MATCHES ".*gcc4-compatible.*" OR GCC_VERSION_ERR MATCHES ".*gcc4-compatible.*") 7 | set(GCC_ABI_BACKWARDS_COMPATIBLE 1) 8 | else() 9 | set(GCC_ABI_BACKWARDS_COMPATIBLE 0) 10 | endif() 11 | else() 12 | set(GCC_ABI_BACKWARDS_COMPATIBLE 0) 13 | endif() 14 | endmacro(check_gcc_compatible) 15 | -------------------------------------------------------------------------------- /apps/analytical_engine/cmake/FindGFlags.cmake: -------------------------------------------------------------------------------- 1 | # This file is used to find gflags library in CMake script, based on code 2 | # from 3 | # 4 | # https://github.com/BVLC/caffe/blob/master/cmake/Modules/FindGFlags.cmake 5 | # 6 | # which is licensed under the 3-Clause BSD License. 7 | # 8 | # - Try to find GFLAGS 9 | # 10 | # The following variables are optionally searched for defaults 11 | # GFLAGS_ROOT_DIR: Base directory where all GFLAGS components are found 12 | # 13 | # The following are set after configuration is done: 14 | # GFLAGS_FOUND 15 | # GFLAGS_INCLUDE_DIRS 16 | # GFLAGS_LIBRARIES 17 | # GFLAGS_LIBRARY_DIRS 18 | 19 | include(FindPackageHandleStandardArgs) 20 | 21 | set(GFLAGS_ROOT_DIR "" CACHE PATH "Folder contains Gflags") 22 | 23 | # We are testing only a couple of files in the include directories 24 | if(WIN32) 25 | find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h 26 | PATHS ${GFLAGS_ROOT_DIR}/src/windows) 27 | else() 28 | find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h 29 | PATHS ${GFLAGS_ROOT_DIR}) 30 | endif() 31 | 32 | if(MSVC) 33 | find_library(GFLAGS_LIBRARY_RELEASE 34 | NAMES libgflags 35 | PATHS ${GFLAGS_ROOT_DIR} 36 | PATH_SUFFIXES Release) 37 | 38 | find_library(GFLAGS_LIBRARY_DEBUG 39 | NAMES libgflags-debug 40 | PATHS ${GFLAGS_ROOT_DIR} 41 | PATH_SUFFIXES Debug) 42 | 43 | set(GFLAGS_LIBRARY optimized ${GFLAGS_LIBRARY_RELEASE} debug ${GFLAGS_LIBRARY_DEBUG}) 44 | else() 45 | find_library(GFLAGS_LIBRARY gflags) 46 | endif() 47 | 48 | find_package_handle_standard_args(GFlags DEFAULT_MSG GFLAGS_INCLUDE_DIR GFLAGS_LIBRARY) 49 | 50 | 51 | if(GFLAGS_FOUND) 52 | set(GFLAGS_INCLUDE_DIRS ${GFLAGS_INCLUDE_DIR}) 53 | set(GFLAGS_LIBRARIES ${GFLAGS_LIBRARY}) 54 | message(STATUS "Found gflags (include: ${GFLAGS_INCLUDE_DIR}, library: ${GFLAGS_LIBRARY})") 55 | mark_as_advanced(GFLAGS_LIBRARY_DEBUG GFLAGS_LIBRARY_RELEASE 56 | GFLAGS_LIBRARY GFLAGS_INCLUDE_DIR GFLAGS_ROOT_DIR) 57 | endif() 58 | -------------------------------------------------------------------------------- /apps/analytical_engine/cmake/FindGlog.cmake: -------------------------------------------------------------------------------- 1 | # This file is used to find glog library in CMake script, based on code 2 | # from 3 | # 4 | # https://github.com/BVLC/caffe/blob/master/cmake/Modules/FindGlog.cmake 5 | # 6 | # which is licensed under the 2-Clause BSD License. 7 | # 8 | # - Try to find Glog 9 | # 10 | # The following variables are optionally searched for defaults 11 | # GLOG_ROOT_DIR: Base directory where all GLOG components are found 12 | # 13 | # The following are set after configuration is done: 14 | # GLOG_FOUND 15 | # GLOG_INCLUDE_DIRS 16 | # GLOG_LIBRARIES 17 | # GLOG_LIBRARYRARY_DIRS 18 | 19 | include(FindPackageHandleStandardArgs) 20 | 21 | set(GLOG_ROOT_DIR "" CACHE PATH "Folder contains Google glog") 22 | 23 | if(WIN32) 24 | find_path(GLOG_INCLUDE_DIR glog/logging.h 25 | PATHS ${GLOG_ROOT_DIR}/src/windows) 26 | else() 27 | find_path(GLOG_INCLUDE_DIR glog/logging.h 28 | PATHS ${GLOG_ROOT_DIR}) 29 | endif() 30 | 31 | if(MSVC) 32 | find_library(GLOG_LIBRARY_RELEASE libglog_static 33 | PATHS ${GLOG_ROOT_DIR} 34 | PATH_SUFFIXES Release) 35 | 36 | find_library(GLOG_LIBRARY_DEBUG libglog_static 37 | PATHS ${GLOG_ROOT_DIR} 38 | PATH_SUFFIXES Debug) 39 | 40 | set(GLOG_LIBRARY optimized ${GLOG_LIBRARY_RELEASE} debug ${GLOG_LIBRARY_DEBUG}) 41 | else() 42 | find_library(GLOG_LIBRARY glog 43 | PATHS ${GLOG_ROOT_DIR} 44 | PATH_SUFFIXES lib lib64) 45 | endif() 46 | 47 | find_package_handle_standard_args(Glog DEFAULT_MSG GLOG_INCLUDE_DIR GLOG_LIBRARY) 48 | 49 | if(GLOG_FOUND) 50 | set(GLOG_INCLUDE_DIRS ${GLOG_INCLUDE_DIR}) 51 | set(GLOG_LIBRARIES ${GLOG_LIBRARY}) 52 | message(STATUS "Found glog (include: ${GLOG_INCLUDE_DIR}, library: ${GLOG_LIBRARY})") 53 | mark_as_advanced(GLOG_ROOT_DIR GLOG_LIBRARY_RELEASE GLOG_LIBRARY_DEBUG 54 | GLOG_LIBRARY GLOG_INCLUDE_DIR) 55 | endif() 56 | -------------------------------------------------------------------------------- /apps/analytical_engine/cmake/FindRdkafka.cmake: -------------------------------------------------------------------------------- 1 | # This file is used to find librdkafka library in CMake script, modifeid from the 2 | # code from 3 | # 4 | # https://github.com/BVLC/caffe/blob/master/cmake/Modules/FindGlog.cmake 5 | # 6 | # which is licensed under the 2-Clause BSD License. 7 | # 8 | # - Try to find librdkafka 9 | # 10 | # The following variables are optionally searched for defaults 11 | # RDKAFKA_ROOT_DIR: Base directory where all RDKAFKA components are found 12 | # 13 | # The following are set after configuration is done: 14 | # RDKAFKA_FOUND 15 | # RDKAFKA_INCLUDE_DIRS 16 | # RDKAFKA_LIBRARIES 17 | # RDKAFKA_LIBRARY_DIRS 18 | 19 | include(FindPackageHandleStandardArgs) 20 | 21 | set(RDKAFKA_ROOT_DIR "" CACHE PATH "Folder contains librdkafka") 22 | 23 | # We are testing only a couple of files in the include directories 24 | find_path(RDKAFKA_INCLUDE_DIR librdkafka PATHS ${RDKAFKA_ROOT_DIR}/include) 25 | 26 | find_library(RDKAFKA_LIBRARY rdkafka PATHS ${RDKAFKA_ROOT_DIR}/lib) 27 | find_library(RDKAFKA++_LIBRARY rdkafka++ PATHS ${RDKAFKA_ROOT_DIR}/lib) 28 | 29 | find_package_handle_standard_args(RDKAFKA DEFAULT_MSG RDKAFKA_INCLUDE_DIR RDKAFKA_LIBRARY) 30 | 31 | 32 | if(RDKAFKA_FOUND) 33 | set(RDKAFKA_INCLUDE_DIRS ${RDKAFKA_INCLUDE_DIR}) 34 | # The RDKAFKA_LIBRARY comes later, since it is depended by the former. 35 | set(RDKAFKA_LIBRARIES ${RDKAFKA++_LIBRARY} ${RDKAFKA_LIBRARY}) 36 | message(STATUS "Found rdkafka (include: ${RDKAFKA_INCLUDE_DIRS}, library: ${RDKAFKA_LIBRARIES})") 37 | mark_as_advanced(RDKAFKA_LIBRARY_DEBUG RDKAFKA_LIBRARY_RELEASE 38 | RDKAFKA_LIBRARY RDKAFKA_INCLUDE_DIR RDKAFKA_ROOT_DIR) 39 | endif() 40 | -------------------------------------------------------------------------------- /apps/analytical_engine/core/app/pregel/aggregators/text_aggregator.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #ifndef ANALYTICAL_ENGINE_CORE_APP_PREGEL_AGGREGATORS_TEXT_AGGREGATOR_H_ 17 | #define ANALYTICAL_ENGINE_CORE_APP_PREGEL_AGGREGATORS_TEXT_AGGREGATOR_H_ 18 | 19 | #include 20 | 21 | #include "core/app/pregel/aggregators/aggregator.h" 22 | 23 | namespace gs { 24 | /** 25 | * @brief A pregel aggregator for the string type. The aggregator concatenates 26 | * string values with empty delimiter. 27 | * @tparam AGGR_TYPE 28 | */ 29 | class TextAppendAggregator : public Aggregator { 30 | public: 31 | void Aggregate(std::string value) override { 32 | Aggregator::SetCurrentValue( 33 | Aggregator::GetCurrentValue() + value); 34 | } 35 | 36 | void Init() override { Aggregator::SetCurrentValue(""); } 37 | 38 | void Reset() override { Aggregator::SetCurrentValue(""); } 39 | }; 40 | 41 | } // namespace gs 42 | 43 | #endif // ANALYTICAL_ENGINE_CORE_APP_PREGEL_AGGREGATORS_TEXT_AGGREGATOR_H_ 44 | -------------------------------------------------------------------------------- /apps/analytical_engine/core/app/pregel/export.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #ifndef ANALYTICAL_ENGINE_CORE_APP_PREGEL_EXPORT_H_ 17 | #define ANALYTICAL_ENGINE_CORE_APP_PREGEL_EXPORT_H_ 18 | 19 | #include "vineyard/graph/fragment/arrow_fragment.h" 20 | 21 | #include "core/app/pregel/aggregators/aggregator.h" 22 | #include "core/app/pregel/i_vertex_program.h" 23 | #include "core/app/pregel/pregel_property_vertex.h" 24 | 25 | namespace pregel { 26 | 27 | template 28 | using Vertex = gs::PregelPropertyVertex< 29 | vineyard::ArrowFragment<_OID_TYPE, 30 | vineyard::property_graph_types::VID_TYPE>, 31 | VD_T, MD_T>; 32 | 33 | template 34 | using Context = gs::PregelPropertyComputeContext< 35 | vineyard::ArrowFragment<_OID_TYPE, 36 | vineyard::property_graph_types::VID_TYPE>, 37 | VD_T, MD_T>; 38 | 39 | template 40 | using Neighbor = gs::PregelPropertyNeighbor< 41 | vineyard::ArrowFragment<_OID_TYPE, 42 | vineyard::property_graph_types::VID_TYPE>, 43 | VD_T, MD_T>; 44 | 45 | template 46 | using AdjList = gs::PregelPropertyAdjList< 47 | vineyard::ArrowFragment<_OID_TYPE, 48 | vineyard::property_graph_types::VID_TYPE>, 49 | VD_T, MD_T>; 50 | 51 | using gs::Aggregator; 52 | using gs::MessageIterator; 53 | using gs::PregelAggregatorType; 54 | 55 | } // namespace pregel 56 | 57 | #endif // ANALYTICAL_ENGINE_CORE_APP_PREGEL_EXPORT_H_ 58 | -------------------------------------------------------------------------------- /apps/analytical_engine/core/config.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef ANALYTICAL_ENGINE_CORE_CONFIG_H_ 17 | #define ANALYTICAL_ENGINE_CORE_CONFIG_H_ 18 | 19 | #include // IWYU pragma: export 20 | 21 | #include "grape/config.h" // IWYU pragma: export 22 | 23 | namespace gs { 24 | using fid_t = grape::fid_t; 25 | } // namespace gs 26 | 27 | #endif // ANALYTICAL_ENGINE_CORE_CONFIG_H_ 28 | -------------------------------------------------------------------------------- /apps/analytical_engine/core/flags.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #include 16 | 17 | #include "core/flags.h" 18 | 19 | /* flags related to the job. */ 20 | 21 | DEFINE_string(host, "localhost", "the host to listen by gRPC server"); 22 | DEFINE_int32(port, 60001, "the port to listen by gRPC server"); 23 | 24 | // for vineyard 25 | DEFINE_string(vineyard_socket, "", "Unix domain socket path for vineyardd"); 26 | DEFINE_string(vineyard_shared_mem, "2048000000", 27 | "Init size of vineyard shared memory"); 28 | DEFINE_string(etcd_endpoint, "http://127.0.0.1:2379", 29 | "Etcd endpoint that will be used to launch vineyardd"); 30 | 31 | DEFINE_string(dag_file, "", "Engine reads serialized dag proto from dag_file."); 32 | -------------------------------------------------------------------------------- /apps/analytical_engine/core/flags.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef ANALYTICAL_ENGINE_CORE_FLAGS_H_ 17 | #define ANALYTICAL_ENGINE_CORE_FLAGS_H_ 18 | 19 | #include 20 | 21 | DECLARE_string(host); 22 | DECLARE_int32(port); 23 | 24 | DECLARE_string(dag_file); 25 | 26 | // vineyard 27 | DECLARE_string(vineyard_socket); 28 | DECLARE_string(vineyard_shared_mem); 29 | DECLARE_string(etcd_endpoint); 30 | 31 | #endif // ANALYTICAL_ENGINE_CORE_FLAGS_H_ 32 | -------------------------------------------------------------------------------- /apps/analytical_engine/core/fragment/arrow_projected_fragment_base.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #ifndef ANALYTICAL_ENGINE_CORE_FRAGMENT_ARROW_PROJECTED_FRAGMENT_BASE_H_ 16 | #define ANALYTICAL_ENGINE_CORE_FRAGMENT_ARROW_PROJECTED_FRAGMENT_BASE_H_ 17 | #include "vineyard/client/ds/i_object.h" 18 | 19 | namespace gs { 20 | /** 21 | * @brief This is the base class of ArrowProjectedFragment 22 | */ 23 | class ArrowProjectedFragmentBase : public vineyard::Object { 24 | public: 25 | virtual ~ArrowProjectedFragmentBase() = default; 26 | }; 27 | } // namespace gs 28 | #endif // ANALYTICAL_ENGINE_CORE_FRAGMENT_ARROW_PROJECTED_FRAGMENT_BASE_H_ 29 | -------------------------------------------------------------------------------- /apps/analytical_engine/core/launcher.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef ANALYTICAL_ENGINE_CORE_LAUNCHER_H_ 17 | #define ANALYTICAL_ENGINE_CORE_LAUNCHER_H_ 18 | 19 | #include 20 | #include 21 | 22 | #include "boost/process/detail/child_decl.hpp" 23 | #include "core/flags.h" 24 | #include "grape/worker/comm_spec.h" 25 | 26 | namespace vineyard { 27 | class Client; 28 | } // namespace vineyard 29 | 30 | namespace gs { 31 | 32 | /** 33 | * @brief VineyardServer is a launcher for vineyardd 34 | */ 35 | class VineyardServer { 36 | public: 37 | explicit VineyardServer(const grape::CommSpec& comm_spec) 38 | : comm_spec_(comm_spec), vineyard_socket_(FLAGS_vineyard_socket) {} 39 | 40 | ~VineyardServer() { this->Stop(); } 41 | 42 | const std::string& vineyard_socket() const { return vineyard_socket_; } 43 | 44 | void Start(); 45 | 46 | void Stop(); 47 | 48 | private: 49 | grape::CommSpec comm_spec_; 50 | std::string vineyard_socket_; 51 | std::unique_ptr proc_; 52 | }; 53 | 54 | void EnsureClient(std::shared_ptr& client, 55 | const std::string& vineyard_socket); 56 | 57 | } // namespace gs 58 | 59 | #endif // ANALYTICAL_ENGINE_CORE_LAUNCHER_H_ 60 | -------------------------------------------------------------------------------- /apps/analytical_engine/core/object/dynamic.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifdef NETWORKX 17 | #include "core/object/dynamic.h" 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | /* static definition */ 25 | gs::dynamic::AllocatorT gs::dynamic::Value::allocator_{}; 26 | 27 | std::size_t gs::dynamic::Value::hash() const { 28 | switch (GetType()) { 29 | case rapidjson::kNullType: 30 | return 0xBAAAAAAD; 31 | case rapidjson::kArrayType: { 32 | std::size_t hash_value = 0; 33 | for (const auto& val : GetArray()) { 34 | if (val.IsString()) { 35 | hash_value += std::hash()(val.GetString()); 36 | } else if (val.IsInt64()) { 37 | hash_value += std::hash()(val.GetInt64()); 38 | } else if (val.IsDouble()) { 39 | hash_value += std::hash()(val.GetDouble()); 40 | } 41 | } 42 | return hash_value; 43 | } 44 | case rapidjson::kNumberType: { 45 | if (IsDouble()) { 46 | return std::hash()(GetDouble()); 47 | } else { 48 | return std::hash()(GetInt64()); 49 | } 50 | } 51 | case rapidjson::kTrueType: 52 | case rapidjson::kFalseType: 53 | return std::hash()(GetBool()); 54 | case rapidjson::kStringType: 55 | return std::hash()(GetString()); 56 | case rapidjson::kObjectType: 57 | throw std::runtime_error("Object value can't not be hashed."); 58 | } 59 | return rapidjson::kNullType; // avoid -Wreturn-type warnings 60 | } 61 | 62 | #endif // NETWORKX 63 | -------------------------------------------------------------------------------- /apps/analytical_engine/core/server/analytical_server.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #include "core/server/analytical_server.h" 16 | 17 | #include 18 | #include 19 | #include 20 | 21 | #include "glog/logging.h" 22 | #include "grpcpp/security/server_credentials.h" 23 | #include "grpcpp/server.h" 24 | #include "grpcpp/server_builder.h" 25 | 26 | #include "core/server/graphscope_service.h" 27 | 28 | namespace gs { 29 | 30 | namespace rpc { 31 | void AnalyticalServer::StartServer() { 32 | std::string server_address = host_ + ":" + std::to_string(port_); 33 | GraphScopeService service(dispatcher_); 34 | 35 | grpc::ServerBuilder builder; 36 | builder.SetMaxReceiveMessageSize(std::numeric_limits::max()); 37 | builder.SetMaxSendMessageSize(std::numeric_limits::max()); 38 | 39 | builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); 40 | builder.RegisterService(&service); 41 | 42 | grpc_server_ = builder.BuildAndStart(); 43 | LOG(INFO) << "Analytical server is listening on " << server_address; 44 | 45 | grpc_server_->Wait(); 46 | } 47 | 48 | void AnalyticalServer::StopServer() { grpc_server_->Shutdown(); } 49 | } // namespace rpc 50 | } // namespace gs 51 | -------------------------------------------------------------------------------- /apps/analytical_engine/core/server/analytical_server.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef ANALYTICAL_ENGINE_CORE_SERVER_ANALYTICAL_SERVER_H_ 17 | #define ANALYTICAL_ENGINE_CORE_SERVER_ANALYTICAL_SERVER_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "grpcpp/server.h" 24 | 25 | namespace gs { 26 | class Dispatcher; 27 | 28 | namespace rpc { 29 | 30 | /** 31 | * @brief AnalyticalServer is responsible for create and start the gRPC service 32 | */ 33 | class AnalyticalServer { 34 | public: 35 | AnalyticalServer(std::shared_ptr dispatcher, std::string host, 36 | int port) 37 | : dispatcher_(std::move(dispatcher)), 38 | host_(std::move(host)), 39 | port_(port) {} 40 | 41 | void StartServer(); 42 | 43 | void StopServer(); 44 | 45 | private: 46 | std::shared_ptr dispatcher_; 47 | std::string host_; 48 | int port_; 49 | std::unique_ptr grpc_server_; 50 | }; 51 | } // namespace rpc 52 | } // namespace gs 53 | 54 | #endif // ANALYTICAL_ENGINE_CORE_SERVER_ANALYTICAL_SERVER_H_ 55 | -------------------------------------------------------------------------------- /apps/analytical_engine/core/utils/fragment_traits.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef ANALYTICAL_ENGINE_CORE_UTILS_FRAGMENT_TRAITS_H_ 17 | #define ANALYTICAL_ENGINE_CORE_UTILS_FRAGMENT_TRAITS_H_ 18 | 19 | #include 20 | 21 | #include "vineyard/common/util/typename.h" 22 | #include "vineyard/graph/fragment/fragment_traits.h" // IWYU pragma: export 23 | 24 | #include "core/object/dynamic.h" 25 | 26 | namespace vineyard { 27 | 28 | #if defined(NETWORKX) 29 | template <> 30 | struct typename_t { 31 | inline static const std::string name() { return "dynamic::Value"; } 32 | }; 33 | #endif 34 | 35 | } // namespace vineyard 36 | 37 | #endif // ANALYTICAL_ENGINE_CORE_UTILS_FRAGMENT_TRAITS_H_ 38 | -------------------------------------------------------------------------------- /apps/analytical_engine/core/utils/lib_utils.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef ANALYTICAL_ENGINE_CORE_UTILS_LIB_UTILS_H_ 17 | #define ANALYTICAL_ENGINE_CORE_UTILS_LIB_UTILS_H_ 18 | 19 | #include 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | #include "boost/leaf/result.hpp" 26 | 27 | #include "core/error.h" 28 | 29 | namespace bl = boost::leaf; 30 | 31 | namespace gs { 32 | inline bl::result open_lib(const char* path) { 33 | void* handle = dlopen(path, RTLD_LAZY); 34 | auto* p_error_msg = dlerror(); 35 | if (p_error_msg) { 36 | RETURN_GS_ERROR(vineyard::ErrorCode::kIOError, 37 | "Failed to open library: " + std::string(path) + 38 | ". Reason: " + std::string(p_error_msg)); 39 | } 40 | return handle; 41 | } 42 | 43 | inline bl::result get_func_ptr(const std::string& lib_path, void* handle, 44 | const char* symbol) { 45 | auto* p_func = dlsym(handle, symbol); 46 | auto* p_error_msg = dlerror(); 47 | if (p_error_msg) { 48 | std::stringstream ss; 49 | 50 | ss << "Failed to get symbol " << symbol << " from " << lib_path 51 | << ". Reason: " << std::string(p_error_msg); 52 | 53 | RETURN_GS_ERROR(vineyard::ErrorCode::kIOError, ss.str()); 54 | } 55 | return p_func; 56 | } 57 | } // namespace gs 58 | #endif // ANALYTICAL_ENGINE_CORE_UTILS_LIB_UTILS_H_ 59 | -------------------------------------------------------------------------------- /apps/analytical_engine/core/utils/mpi_utils.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef ANALYTICAL_ENGINE_CORE_UTILS_MPI_UTILS_H_ 17 | #define ANALYTICAL_ENGINE_CORE_UTILS_MPI_UTILS_H_ 18 | 19 | #include 20 | 21 | #include 22 | #include 23 | 24 | #include "vineyard/graph/utils/mpi_utils.h" // IWYU pragma: export 25 | 26 | namespace gs { 27 | 28 | #if SIZE_MAX == UCHAR_MAX 29 | #define MPI_SIZE_T MPI_UNSIGNED_CHAR 30 | #elif SIZE_MAX == USHRT_MAX 31 | #define MPI_SIZE_T MPI_UNSIGNED_SHORT 32 | #elif SIZE_MAX == UINT_MAX 33 | #define MPI_SIZE_T MPI_UNSIGNED 34 | #elif SIZE_MAX == ULONG_MAX 35 | #define MPI_SIZE_T MPI_UNSIGNED_LONG 36 | #elif SIZE_MAX == ULLONG_MAX 37 | #define MPI_SIZE_T MPI_UNSIGNED_LONG_LONG 38 | #else 39 | #error "Unsupported size" 40 | #endif 41 | 42 | } // namespace gs 43 | 44 | #endif // ANALYTICAL_ENGINE_CORE_UTILS_MPI_UTILS_H_ 45 | -------------------------------------------------------------------------------- /apps/analytical_engine/core/utils/trait_utils.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef ANALYTICAL_ENGINE_CORE_UTILS_TRAIT_UTILS_H_ 17 | #define ANALYTICAL_ENGINE_CORE_UTILS_TRAIT_UTILS_H_ 18 | 19 | #include 20 | 21 | #include "vineyard/common/util/static_if.h" 22 | 23 | namespace gs { 24 | template 26 | class ArrowFlattenedFragment; 27 | 28 | template 29 | struct is_flattened_fragment { 30 | using type = std::false_type; 31 | static constexpr bool value = false; 32 | }; 33 | 34 | template 36 | struct is_flattened_fragment< 37 | ArrowFlattenedFragment> { 38 | using type = std::true_type; 39 | static constexpr bool value = true; 40 | }; 41 | 42 | } // namespace gs 43 | #endif // ANALYTICAL_ENGINE_CORE_UTILS_TRAIT_UTILS_H_ 44 | -------------------------------------------------------------------------------- /apps/analytical_engine/exported_symbols_osx.lds: -------------------------------------------------------------------------------- 1 | __ZN2gs7dynamic5Value10allocator_E -------------------------------------------------------------------------------- /apps/analytical_engine/graphscope-analytical-config-version.in.cmake: -------------------------------------------------------------------------------- 1 | set(PACKAGE_VERSION "@GRAPHSCOPE_ANALYTICAL_VERSION@") 2 | 3 | # Check whether the requested PACKAGE_FIND_VERSION is compatible 4 | if("${PACKAGE_VERSION}" VERSION_LESS "${PACKAGE_FIND_VERSION}") 5 | set(PACKAGE_VERSION_COMPATIBLE FALSE) 6 | else() 7 | set(PACKAGE_VERSION_COMPATIBLE TRUE) 8 | if ("${PACKAGE_VERSION}" VERSION_EQUAL "${PACKAGE_FIND_VERSION}") 9 | set(PACKAGE_VERSION_EXACT TRUE) 10 | endif() 11 | endif() -------------------------------------------------------------------------------- /apps/analytical_engine/graphscope-analytical-config.in.cmake: -------------------------------------------------------------------------------- 1 | # - Config file for the graphscope-analytical package 2 | # 3 | # It defines the following variables 4 | # 5 | # GRAPHSCOPE_ANALYTICAL_INCLUDE_DIR - include directory for graphscope-analytical 6 | # GRAPHSCOPE_ANALYTICAL_INCLUDE_DIRS - include directories for graphscope-analytical 7 | # GRAPHSCOPE_ANALYTICAL_LIBRARIES - libraries to link against 8 | # GRAPHSCOPE_ANALYTICAL_ENGINE_EXECUTABLE - the grape-engine executable 9 | 10 | set(GRAPHSCOPE_ANALYTICAL_HOME "${CMAKE_CURRENT_LIST_DIR}/../../..") 11 | include("${CMAKE_CURRENT_LIST_DIR}/graphscope-analytical-targets.cmake" OPTIONAL) 12 | 13 | set(GRAPHSCOPE_ANALYTICAL_LIBRARIES gs_proto gs_util) 14 | set(GRAPHSCOPE_ANALYTICAL_INCLUDE_DIR "${GRAPHSCOPE_ANALYTICAL_HOME}/include" 15 | "${GRAPHSCOPE_ANALYTICAL_HOME}/include/graphscope") 16 | set(GRAPHSCOPE_ANALYTICAL_INCLUDE_DIRS "${GRAPHSCOPE_ANALYTICAL_INCLUDE_DIR}") 17 | set(GRAPHSCOPE_ANALYTICAL_ENGINE_EXECUTABLE "${GRAPHSCOPE_ANALYTICAL_HOME}/bin/grape_engine") 18 | 19 | set(GRAPHSCOPE_GCC_ABI_BACKWARDS_COMPATIBLE @GCC_ABI_BACKWARDS_COMPATIBLE@) -------------------------------------------------------------------------------- /apps/analytical_engine/python/graphscope/VERSION: -------------------------------------------------------------------------------- 1 | ../../VERSION -------------------------------------------------------------------------------- /apps/analytical_engine/python/graphscope/proto/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /apps/analytical_engine/python/graphscope/proto/engine_service.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | syntax = "proto3"; 17 | 18 | package gs.rpc; 19 | 20 | import "graphscope/proto/message.proto"; 21 | 22 | service EngineService { 23 | // Drives the graph computation. 24 | rpc RunStep(stream RunStepRequest) returns (stream RunStepResponse); 25 | 26 | // Heart beat between coordinator and analytical engine 27 | rpc HeartBeat(HeartBeatRequest) returns (HeartBeatResponse); 28 | } 29 | -------------------------------------------------------------------------------- /apps/analytical_engine/python/graphscope/version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | import os 19 | 20 | from packaging import version 21 | 22 | version_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "VERSION") 23 | 24 | with open(version_file_path, "r", encoding="utf-8") as fp: 25 | sv = version.parse(fp.read().strip()) 26 | __is_prerelease__ = sv.is_prerelease 27 | __version__ = str(sv) 28 | 29 | __version_tuple__ = (v for v in __version__.split(".")) 30 | 31 | del version_file_path 32 | -------------------------------------------------------------------------------- /apps/analytical_engine/test/flags.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include "flags.h" 17 | 18 | #include 19 | 20 | DEFINE_string(etcd_endpoint, "http://127.0.0.1:2379", 21 | "etcd endpoint to get graph schema."); 22 | DEFINE_int32(read_epoch, 0, "read epoch for graph reader."); 23 | DEFINE_string(meta_prefix, "gart_meta_", "meta prefix for etcd."); 24 | DEFINE_string(app_name, "sssp", "gart app name."); 25 | DEFINE_string(sssp_source_label, "", "source label id for sssp."); 26 | DEFINE_int32(sssp_source_oid, 0, "source oid for sssp."); 27 | DEFINE_string(sssp_weight_name, "", "weight name for sssp."); 28 | -------------------------------------------------------------------------------- /apps/analytical_engine/test/flags.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef ANALYTICAL_ENGINE_TEST_FLAGS_H_ 17 | #define ANALYTICAL_ENGINE_TEST_FLAGS_H_ 18 | 19 | #include 20 | 21 | DECLARE_string(etcd_endpoint); 22 | DECLARE_int32(read_epoch); 23 | DECLARE_string(meta_prefix); 24 | DECLARE_string(app_name); 25 | DECLARE_string(sssp_source_label); 26 | DECLARE_int32(sssp_source_oid); 27 | DECLARE_string(sssp_weight_name); 28 | 29 | #endif // ANALYTICAL_ENGINE_TEST_FLAGS_H_ -------------------------------------------------------------------------------- /apps/analytical_engine/test/gart_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # A script to perform tests for gart. 4 | 5 | gart_engine_dir="../../../vegito/test" 6 | gart_test_dir="../../../vegito/test" 7 | gart_test_data_dir="../../../vegito/test/data" 8 | gart_test_config_dir="../../../vegito/test/config" 9 | analytical_engine_dir="../build" 10 | 11 | socket_file="/opt/tmp/tmp.sock" 12 | 13 | function start_vineyard() { 14 | sudo pkill vineyardd || true 15 | sudo pkill etcd || true 16 | echo "[INFO] vineyardd will using the socket_file on ${socket_file}" 17 | #start vineyardd 18 | touch ${socket_file} 19 | /usr/local/bin/vineyardd --etcd_endpoint=127.0.0.1:2379 --socket ${socket_file} --size 500G & 20 | sleep 5 21 | info "vineyardd started." 22 | } 23 | 24 | function start_gart_writer() { 25 | #start gart writer 26 | pushd ${gart_engine_dir} 27 | mpirun -n 2 run_test.sh 28 | sleep 5 29 | info "gart writer started." 30 | popd 31 | } 32 | 33 | 34 | function start_gart_reader() { 35 | #start gart reader 36 | pushd ${analytical_engine_dir} 37 | mpirun -n 2 ./run_gart_reader 38 | popd 39 | } 40 | 41 | start_vineyard 42 | start_gart_writer 43 | echo "gart_reader..." 44 | start_gart_reader 45 | -------------------------------------------------------------------------------- /apps/networkx/client/python/src/gart/__init__.py: -------------------------------------------------------------------------------- 1 | from . import proto 2 | from .digraph import DiGraph 3 | from .local_graph import LocalDiGraph 4 | from .client import Client 5 | from .proto import * 6 | -------------------------------------------------------------------------------- /apps/networkx/client/python/src/gart/coreviews.py: -------------------------------------------------------------------------------- 1 | from networkx.classes.coreviews import AdjacencyView as _AdjacencyView 2 | from networkx.classes.coreviews import AtlasView as _AtlasView 3 | 4 | 5 | class AtlasView(_AtlasView): 6 | def __eq__(self, other): 7 | return self._atlas.__eq__(other) 8 | 9 | def __contains__(self, key): 10 | return key in self._atlas 11 | 12 | 13 | class AdjacencyView(AtlasView): 14 | __slots__ = () # Still uses AtlasView slots names _atlas 15 | 16 | def __getitem__(self, name): 17 | return AtlasView(self._atlas[name]) 18 | -------------------------------------------------------------------------------- /apps/networkx/client/python/src/gart/dict_factory.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Mapping 2 | 3 | 4 | class AdjListDict(Mapping): 5 | def __init__(self, graph, pred=False): 6 | self._graph = graph 7 | self._pred = pred 8 | 9 | def __contains__(self, key): 10 | return self._graph.has_node(key) 11 | 12 | def __len__(self): 13 | return self._graph.number_of_nodes() 14 | 15 | def __getitem__(self, key): 16 | if self._pred is False: 17 | return self._graph.get_succ_neighbor_attr_pair(key) 18 | return self._graph.get_pred_neighbor_attr_pair(key) 19 | 20 | def __iter__(self): 21 | return self._graph.__iter__() 22 | 23 | 24 | class NeighborDict(Mapping): 25 | def __init__(self, graph): 26 | self._graph = graph 27 | 28 | def __iter__(self): 29 | for node in self._graph: 30 | yield (node, self._graph.get_succ_neighbor_attr_pair(node)) 31 | 32 | def __getitem__(self, key): 33 | return self._graph.get_succ_neighbor_attr_pair(key) 34 | 35 | def __contains__(self, key): 36 | return self._graph.has_node(key) 37 | 38 | def __len__(self): 39 | return self._graph.number_of_nodes() 40 | -------------------------------------------------------------------------------- /apps/networkx/client/python/src/gart/proto/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | sys.path.insert(0, os.path.dirname(__file__)) 5 | -------------------------------------------------------------------------------- /apps/networkx/client/test/test.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import sys 3 | import time 4 | 5 | from gart import Client 6 | 7 | 8 | def get_parser(): 9 | parser = argparse.ArgumentParser( 10 | description="NX Client Test", 11 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 12 | ) 13 | 14 | parser.add_argument( 15 | "--host", default="127.0.0.1", help="NetworkX server host address" 16 | ) 17 | parser.add_argument("--port", default=50051, help="NetworkX server port") 18 | 19 | return parser 20 | 21 | 22 | if __name__ == "__main__": 23 | args = get_parser().parse_args() 24 | 25 | hostname = f"{args.host}:{args.port}" 26 | 27 | client = Client(hostname) 28 | version = client.get_latest_version() 29 | g = client.get_graph(version) 30 | 31 | print("number of nodes:", len(g)) 32 | 33 | # for node in g: 34 | # print(node, " in g is ", node in g) 35 | # pass 36 | # print(node) 37 | 38 | node = ("organisation", 0) 39 | 40 | sys.exit(0) 41 | # print(g[node]) 42 | 43 | start = time.time() 44 | length = client.run_sssp(g, node, "wa_work_from") 45 | print(len(length)) 46 | print("SSSP took", 1000 * (time.time() - start), " ms") 47 | 48 | # for node in g.nodes(data="org_id", default="default"): 49 | # pass 50 | # print(node) 51 | # print(g.nodes(data=True)) 52 | 53 | node = ("person", 933) 54 | print(g[node]) 55 | 56 | print(g._adj[node]) 57 | 58 | print(len(g[node]) == len(g._adj[node])) 59 | 60 | node = ("organisation", 1) 61 | print(g.adj[node]) 62 | dst = ("person", 24189255812047) 63 | print(g._adj[node][dst]) 64 | 65 | # for node in g.nodes(data=True): 66 | # pass 67 | # print(node) 68 | # print(g.nodes(data=True)) 69 | 70 | # for edge in g.edges(data="wa_work_from", default="default"): 71 | # print(edge) 72 | 73 | # for src in g.adjacency(): 74 | # print(src) 75 | for src in g.degree(weight="span"): 76 | print(src) 77 | -------------------------------------------------------------------------------- /apps/networkx/proto/types.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Alibaba Group Holding Limited. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | syntax = "proto3"; 17 | 18 | package gart.rpc; 19 | 20 | service QueryGraphService{ 21 | rpc getData (Request) returns (stream Response); 22 | } 23 | 24 | // For simulating networkx reporting functionalities 25 | enum ReportType { 26 | NODE_NUM = 0; 27 | EDGE_NUM = 1; 28 | HAS_NODE = 2; 29 | HAS_EDGE = 3; 30 | NODE_DATA = 4; 31 | EDGE_DATA = 5; 32 | SUCCS_BY_NODE = 6; 33 | PREDS_BY_NODE = 7; 34 | SELFLOOPS_NUM = 8; 35 | NODE_ID_CACHE_BY_GID = 9; 36 | NODE_ATTR_CACHE_BY_GID = 10; 37 | SUCC_BY_GID = 11; 38 | PRED_BY_GID = 12; 39 | SUCC_ATTR_BY_GID = 13; 40 | PRED_ATTR_BY_GID = 14; 41 | SUCC_ATTR_BY_NODE = 15; 42 | PRED_ATTR_BY_NODE = 16; 43 | NODES = 17; 44 | NODES_GID = 18; 45 | LATEST_GRAPH_VERSION = 19; 46 | RUN_GAE_SSSP = 20; 47 | CONNECT_INFO = 21; 48 | } 49 | 50 | // 51 | message Request { 52 | ReportType op = 1; 53 | int64 version = 2; 54 | string args = 3; 55 | } 56 | 57 | message Response { 58 | bytes result = 1; 59 | } 60 | 61 | enum DataType { 62 | UNKNOWN = 0; 63 | BOOL = 1; 64 | CHAR = 2; 65 | SHORT = 3; 66 | INT = 4; 67 | LONG = 5; 68 | FLOAT = 6; 69 | DOUBLE = 7; 70 | STRING = 8; 71 | BYTES = 9; 72 | INT_LIST = 10; 73 | LONG_LIST = 11; 74 | FLOAT_LIST = 12; 75 | DOUBLE_LIST = 13; 76 | STRING_LIST = 14; 77 | NULLVALUE = 15; 78 | UINT = 16; 79 | ULONG = 17; 80 | DYNAMIC = 18; 81 | } 82 | -------------------------------------------------------------------------------- /apps/networkx/python_bindings/fragment_builder.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef APPS_NETWORKX_PYTHON_BINDINGS_FRAGMENT_BUILDER_H_ 17 | #define APPS_NETWORKX_PYTHON_BINDINGS_FRAGMENT_BUILDER_H_ 18 | 19 | #include 20 | #include 21 | 22 | #include "basic/ds/arrow.h" 23 | #include "etcd/Client.hpp" 24 | #include "vineyard/common/util/json.h" 25 | 26 | #include "interfaces/fragment/gart_fragment.h" 27 | 28 | using GraphType = gart::GartFragment; 29 | 30 | class FragmentBuilder { 31 | public: 32 | FragmentBuilder(std::string etcd_endpoint, std::string meta_prefix, 33 | int read_epoch); 34 | ~FragmentBuilder(); 35 | 36 | std::shared_ptr get_graph() const; 37 | 38 | private: 39 | int value_; 40 | std::string etcd_endpoint_; 41 | std::string meta_prefix_; 42 | int read_epoch_; 43 | std::shared_ptr etcd_client_; 44 | vineyard::json graph_schema_; 45 | std::shared_ptr fragment_; 46 | }; 47 | 48 | #endif // APPS_NETWORKX_PYTHON_BINDINGS_FRAGMENT_BUILDER_H_ 49 | -------------------------------------------------------------------------------- /apps/networkx/server/graph_server.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #include 16 | #include 17 | #include 18 | 19 | #include "server/graph_reporter.h" 20 | #include "server/graph_server_flags.h" 21 | 22 | int main(int argc, char** argv) { 23 | /* parse arguments for whole system */ 24 | google::InitGoogleLogging(argv[0]); 25 | gflags::ParseCommandLineFlags(&argc, &argv, true); 26 | 27 | gart::QueryGraphServiceImpl service(FLAGS_etcd_endpoint, FLAGS_meta_prefix); 28 | 29 | grpc::ServerBuilder builder; 30 | builder.AddListeningPort(FLAGS_server_addr, 31 | grpc::InsecureServerCredentials()); 32 | // Increase the maximum send message size as 101MB, 1MB for header 33 | builder.SetMaxSendMessageSize(1024 * 1024 * 100 + 1024 * 1024 * 1); 34 | builder.RegisterService(&service); 35 | std::unique_ptr server(builder.BuildAndStart()); 36 | std::cout << "Server listening on " << FLAGS_server_addr << std::endl; 37 | server->Wait(); 38 | 39 | return 0; 40 | } 41 | -------------------------------------------------------------------------------- /apps/networkx/server/graph_server_flags.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include "server/graph_server_flags.h" 17 | 18 | DEFINE_string(etcd_endpoint, "http://127.0.0.1:23760", 19 | "etcd endpoint for schema."); 20 | DEFINE_string(meta_prefix, "gart_meta_", "meta prefix for etcd."); 21 | DEFINE_string(server_addr, "127.0.0.1:50051", "server address."); 22 | -------------------------------------------------------------------------------- /apps/networkx/server/graph_server_flags.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef APPS_NETWORKX_SERVER_GRAPH_SERVER_FLAGS_H_ 17 | #define APPS_NETWORKX_SERVER_GRAPH_SERVER_FLAGS_H_ 18 | 19 | #include 20 | #include 21 | 22 | DECLARE_string(etcd_endpoint); 23 | DECLARE_string(meta_prefix); 24 | DECLARE_string(server_addr); 25 | 26 | #endif // APPS_NETWORKX_SERVER_GRAPH_SERVER_FLAGS_H_ 27 | -------------------------------------------------------------------------------- /apps/networkx/server/utils/dynamic.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include "server/utils/dynamic.h" 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | /* static definition */ 24 | gart::dynamic::AllocatorT gart::dynamic::Value::allocator_{}; 25 | 26 | std::size_t gart::dynamic::Value::hash() const { 27 | switch (GetType()) { 28 | case rapidjson::kNullType: 29 | return 0xBAAAAAAD; 30 | case rapidjson::kArrayType: { 31 | std::size_t hash_value = 0; 32 | for (const auto& val : GetArray()) { 33 | if (val.IsString()) { 34 | hash_value += std::hash()(val.GetString()); 35 | } else if (val.IsInt64()) { 36 | hash_value += std::hash()(val.GetInt64()); 37 | } else if (val.IsDouble()) { 38 | hash_value += std::hash()(val.GetDouble()); 39 | } 40 | } 41 | return hash_value; 42 | } 43 | case rapidjson::kNumberType: { 44 | if (IsDouble()) { 45 | return std::hash()(GetDouble()); 46 | } else { 47 | return std::hash()(GetInt64()); 48 | } 49 | } 50 | case rapidjson::kTrueType: 51 | case rapidjson::kFalseType: 52 | return std::hash()(GetBool()); 53 | case rapidjson::kStringType: 54 | return std::hash()(GetString()); 55 | case rapidjson::kObjectType: 56 | throw std::runtime_error("Object value can't not be hashed."); 57 | } 58 | return rapidjson::kNullType; // avoid -Wreturn-type warnings 59 | } 60 | -------------------------------------------------------------------------------- /apps/pgx/Makefile: -------------------------------------------------------------------------------- 1 | EXTENSION = gart 2 | MODULE_big = gart 3 | DATA = gart--0.0.1.sql 4 | OBJS = gart.o utility.o 5 | 6 | PG_CONFIG = pg_config 7 | PGXS := $(shell $(PG_CONFIG) --pgxs) 8 | include $(PGXS) 9 | -------------------------------------------------------------------------------- /apps/pgx/compile_commands.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "arguments": [ 4 | "/usr/bin/clang-10", 5 | "-c", 6 | "-Wno-ignored-attributes", 7 | "-fno-strict-aliasing", 8 | "-fwrapv", 9 | "-Wno-unused-command-line-argument", 10 | "-O2", 11 | "-I.", 12 | "-I./", 13 | "-I/usr/include/postgresql/12/server", 14 | "-I/usr/include/postgresql/internal", 15 | "-Wdate-time", 16 | "-D_FORTIFY_SOURCE=2", 17 | "-D_GNU_SOURCE", 18 | "-I/usr/include/libxml2", 19 | "-I/usr/include/mit-krb5", 20 | "-flto=thin", 21 | "-emit-llvm", 22 | "-o", 23 | "log.bc", 24 | "log.c" 25 | ], 26 | "directory": "/opt/ssj/projects/gart/apps/pgx", 27 | "file": "log.c" 28 | }, 29 | { 30 | "arguments": [ 31 | "gcc", 32 | "-c", 33 | "-Wall", 34 | "-Wmissing-prototypes", 35 | "-Wpointer-arith", 36 | "-Wdeclaration-after-statement", 37 | "-Werror=vla", 38 | "-Wendif-labels", 39 | "-Wmissing-format-attribute", 40 | "-Wformat-security", 41 | "-fno-strict-aliasing", 42 | "-fwrapv", 43 | "-fexcess-precision=standard", 44 | "-Wno-format-truncation", 45 | "-Wno-stringop-truncation", 46 | "-g", 47 | "-g", 48 | "-O2", 49 | "-fstack-protector-strong", 50 | "-Wformat", 51 | "-Werror=format-security", 52 | "-fno-omit-frame-pointer", 53 | "-fPIC", 54 | "-I.", 55 | "-I./", 56 | "-I/usr/include/postgresql/12/server", 57 | "-I/usr/include/postgresql/internal", 58 | "-Wdate-time", 59 | "-D_FORTIFY_SOURCE=2", 60 | "-D_GNU_SOURCE", 61 | "-I/usr/include/libxml2", 62 | "-I/usr/include/mit-krb5", 63 | "-o", 64 | "log.o", 65 | "log.c" 66 | ], 67 | "directory": "/opt/ssj/projects/gart/apps/pgx", 68 | "file": "log.c" 69 | } 70 | ] -------------------------------------------------------------------------------- /apps/pgx/gart-pgx-config-template.ini: -------------------------------------------------------------------------------- 1 | [path] 2 | KAFKA_HOME=/opt/wanglei/kafka_2.13-3.7.0 3 | GART_HOME=/opt/ssj/projects/gart 4 | 5 | [log] 6 | log_path=/opt/postgresql/tmp.log 7 | real_time_log_path=/opt/postgresql/gart.log 8 | 9 | [gart] 10 | # GART db type: mysql, postgresql 11 | db-type=postgresql 12 | rgmapping-file=/opt/postgresql/rgmapping-ldbc.yaml 13 | v6d-sock=/opt/ssj/projects/gart/build/ldbc.sock 14 | etcd-endpoints=127.0.0.1:23760 15 | etcd-prefix=gart_meta_ 16 | subgraph-num=1 17 | enable-bulkload=1 18 | -------------------------------------------------------------------------------- /apps/pgx/gart.control: -------------------------------------------------------------------------------- 1 | comment = 'PostgreSQL Logger Utility' 2 | default_version = '0.0.1' 3 | relocatable = true 4 | module_pathname = '$libdir/gart' 5 | -------------------------------------------------------------------------------- /apps/pgx/utility.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef APPS_PGX_UTILITY_H_ 17 | #define APPS_PGX_UTILITY_H_ 18 | 19 | #include 20 | 21 | /* 22 | * Utility functions for basic I/O functions 23 | */ 24 | // Non-blocking reads using FILE* handles 25 | int non_blocking_fgets(char* buffer, int size, FILE* file_stream); 26 | 27 | /* 28 | * Utility functions for parsing ini files 29 | */ 30 | // parse the ini file 31 | void init_parse_ini(const char* file_name); 32 | 33 | // find the value of the key in the section 34 | // if the section is NULL, find the value of the key in the global section 35 | // the max length of the value is 1024 (MAX_VALUE) 36 | // return the length of the value, -1 if the key is not found 37 | int find_value(const char* section, const char* key, char* value); 38 | 39 | /* 40 | * Utility functions for NX server information 41 | */ 42 | 43 | int init_server_info(FILE* file); 44 | 45 | int get_next_server_id(void); 46 | 47 | // add server information to the file 48 | // return the server_id of the server 49 | int add_server_info(FILE* file, const char* hostname, int port); 50 | 51 | // find the server information in the file 52 | // return 0 if the server_id is not found, 1 if the server_id is found 53 | int get_server_info(FILE* file, int server_id, char* hostname, int* port); 54 | 55 | // delete the server information in the file 56 | // return 0 if the server_id is not found, 1 if the server_id is found 57 | int delete_server_info(FILE* file, int server_id); 58 | 59 | #endif // APPS_PGX_UTILITY_H_ 60 | -------------------------------------------------------------------------------- /charts/gart/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /charts/gart/Chart.yaml: -------------------------------------------------------------------------------- 1 | annotations: 2 | category: Database 3 | licenses: Apache-2.0 4 | apiVersion: v2 5 | appVersion: 1.16.0 6 | name: gart 7 | description: Graph Analysis on Relational Transactional Datasets 8 | home: https://github.com/graphscope/gart 9 | type: application 10 | version: 0.1.0 11 | maintainers: 12 | - name: GraphScope 13 | url: https://github.com/graphscope 14 | 15 | 16 | dependencies: 17 | - name: kafka 18 | repository: https://charts.bitnami.com/bitnami 19 | version: "*" 20 | - name: etcd 21 | repository: https://charts.bitnami.com/bitnami 22 | version: "*" -------------------------------------------------------------------------------- /charts/gart/templates/auth/role_and_binding.yaml: -------------------------------------------------------------------------------- 1 | kind: Role 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | metadata: 4 | namespace: {{ .Release.Namespace }} 5 | name: {{ include "gart.fullname" . }}-role 6 | rules: 7 | - apiGroups: [""] 8 | resources: ["pods"] 9 | verbs: ["get", "list", "watch"] 10 | - apiGroups: [""] 11 | resources: ["pods/exec"] 12 | verbs: ["create"] 13 | - apiGroups: [""] 14 | resources: ["services"] 15 | verbs: ["get", "list"] 16 | --- 17 | kind: RoleBinding 18 | apiVersion: rbac.authorization.k8s.io/v1 19 | metadata: 20 | name: {{ include "gart.fullname" . }}-role-binding 21 | namespace: {{ .Release.Namespace }} 22 | subjects: 23 | - kind: ServiceAccount 24 | name: default 25 | namespace: {{ .Release.Namespace }} 26 | roleRef: 27 | kind: Role 28 | name: {{ include "gart.fullname" . }}-role 29 | apiGroup: rbac.authorization.k8s.io 30 | -------------------------------------------------------------------------------- /charts/gart/templates/converter/svc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "gart.controller-service.fullname" . }} 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | app: converter 8 | spec: 9 | type: ClusterIP 10 | selector: 11 | app: converter 12 | ports: 13 | - protocol: TCP 14 | port: {{ .Values.controller.port }} 15 | targetPort: {{ .Values.controller.containerPort }} 16 | 17 | --- 18 | apiVersion: v1 19 | kind: Service 20 | metadata: 21 | name: {{ include "gart.coordinator-service.fullname" . }} 22 | namespace: {{ .Release.Namespace }} 23 | labels: 24 | app: converter 25 | spec: 26 | type: LoadBalancer 27 | selector: 28 | app: converter 29 | ports: 30 | - protocol: TCP 31 | port: {{ .Values.coordinator.port }} 32 | targetPort: {{ .Values.coordinator.containerPort }} 33 | name: coordinator-port 34 | -------------------------------------------------------------------------------- /charts/gart/templates/debezium/deployment.yaml: -------------------------------------------------------------------------------- 1 | {{- $kafka_service_name := .Values.kafka.fullnameOverride }} 2 | {{- $kafka_service_port := int .Values.kafka.service.ports.client }} 3 | {{- $kafka_service := printf "%s:%d" $kafka_service_name $kafka_service_port }} 4 | {{- $kafka_service := quote $kafka_service }} 5 | 6 | apiVersion: apps/v1 7 | kind: Deployment 8 | metadata: 9 | name: {{ include "gart.debezium.fullname" . }} 10 | namespace: {{ .Release.Namespace }} 11 | spec: 12 | replicas: {{ .Values.debezium.replicaCount }} 13 | selector: 14 | matchLabels: 15 | app: debezium-connect 16 | template: 17 | metadata: 18 | labels: 19 | app: debezium-connect 20 | spec: 21 | containers: 22 | - name: debezium-connect 23 | image: {{ include "gart.debezium.image" . }} 24 | imagePullPolicy: {{ .Values.debezium.image.pullPolicy | quote }} 25 | resources: {{ toYaml .Values.debezium.resources | nindent 10 }} 26 | env: 27 | - name: BOOTSTRAP_SERVERS 28 | value: {{ $kafka_service }} 29 | - name: GROUP_ID 30 | value: "1" 31 | - name: CONFIG_STORAGE_TOPIC 32 | value: "debezium-connect-configs" 33 | - name: OFFSET_STORAGE_TOPIC 34 | value: "debezium-connect-offsets" 35 | - name: STATUS_STORAGE_TOPIC 36 | value: "debezium-connect-status" 37 | ports: 38 | - containerPort: {{ .Values.debezium.containerPort }} 39 | 40 | -------------------------------------------------------------------------------- /charts/gart/templates/debezium/svc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "gart.debezium.fullname" . }} 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | app: debezium-connect 8 | spec: 9 | type: ClusterIP 10 | ports: 11 | - port: {{ .Values.debezium.containerPort }} 12 | targetPort: {{ .Values.debezium.containerPort }} 13 | selector: 14 | app: debezium-connect 15 | -------------------------------------------------------------------------------- /charts/gart/templates/gie_frontend/deployment.yaml: -------------------------------------------------------------------------------- 1 | {{- if not .Values.dataconfig.useGAE }} 2 | {{- $etcd_service_name := .Values.etcd.fullnameOverride }} 3 | {{- $etcd_service_port := int .Values.etcd.containerPorts.client }} 4 | {{- $etcd_service := printf "%s:%d" $etcd_service_name $etcd_service_port }} 5 | 6 | apiVersion: apps/v1 7 | kind: Deployment 8 | metadata: 9 | name: {{ include "gart.fullname" . }}-gie-frontend 10 | namespace: {{ .Release.Namespace }} 11 | spec: 12 | replicas: {{ .Values.gie_frontend.replicaCount }} 13 | selector: 14 | matchLabels: 15 | app: gie-frontend 16 | template: 17 | metadata: 18 | labels: 19 | app: gie-frontend 20 | spec: 21 | containers: 22 | - name: gie-frontend 23 | image: {{ include "gart.gie_frontend.image" . }} 24 | imagePullPolicy: {{ .Values.gie_frontend.image.pullPolicy | quote }} 25 | resources: {{ toYaml .Values.gie_frontend.resources | nindent 10 }} 26 | command: ["/bin/bash", "-c"] 27 | args: 28 | - | 29 | cd /home/graphscope/GART 30 | export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python 31 | ./scripts/update_gie_schema.py {{ $etcd_service }} {{ .Values.dataconfig.etcdPrefix }} {{ .Release.Namespace }} {{ include "gart.writer.fullname" . }} {{ .Values.gie_frontend.gremlinPort }} {{ .Values.dataconfig.subgraphNum}} {{ include "gart.fullname" . }}-gie-executor-service {{ .Values.gie_executor.GAIA_RPC_PORT}} {{ .Values.gie_frontend.cypherPort }} 32 | cd /home/graphscope/GraphScope/interactive_engine/compiler 33 | make run 34 | ports: 35 | - name: gremlin 36 | containerPort: {{ .Values.gie_frontend.gremlinPort }} 37 | - name: cypher 38 | containerPort: {{ .Values.gie_frontend.cypherPort }} 39 | envFrom: 40 | - configMapRef: 41 | name: {{ include "gart.configmapName" . }} 42 | 43 | 44 | 45 | {{- end }} -------------------------------------------------------------------------------- /charts/gart/templates/gie_frontend/svc.yaml: -------------------------------------------------------------------------------- 1 | {{- if not .Values.dataconfig.useGAE }} 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: {{ include "gart.fullname" . }}-gie-frontend-service 6 | namespace: {{ .Release.Namespace }} 7 | labels: 8 | app: gie-frontend 9 | spec: 10 | type: LoadBalancer 11 | selector: 12 | app: gie-frontend 13 | ports: 14 | - protocol: TCP 15 | port: {{ .Values.gie_frontend.gremlinPort }} 16 | targetPort: gremlin 17 | name: gremlin-port 18 | - protocol: TCP 19 | port: {{ .Values.gie_frontend.cypherPort }} 20 | targetPort: cypher 21 | name: cypher-port 22 | {{- end }} -------------------------------------------------------------------------------- /charts/gart/templates/writer/svc.yaml: -------------------------------------------------------------------------------- 1 | {{- if not .Values.dataconfig.useGAE }} 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: {{ include "gart.fullname" . }}-gie-executor-service 6 | spec: 7 | selector: 8 | app: writer 9 | ports: 10 | - name: gaia-rpc 11 | port: {{ .Values.gie_executor.GAIA_RPC_PORT }} 12 | targetPort: gaia-rpc 13 | - name: gaia-engine 14 | port: {{ .Values.gie_executor.ENGINE_PORT }} 15 | targetPort: gaia-engine 16 | - name: http-service 17 | port: {{ .Values.gie_executor.HTTP_SERVICE_PORT }} 18 | targetPort: {{ .Values.gie_executor.FLASK_PORT }} 19 | clusterIP: None 20 | 21 | {{- end }} -------------------------------------------------------------------------------- /converter/flags.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include "converter/flags.h" 17 | 18 | DEFINE_string(read_kafka_broker_list, "127.0.0.1:9092", 19 | "Kafka broker list for reading TxnLogs."); 20 | DEFINE_string(write_kafka_broker_list, "127.0.0.1:9092", 21 | "Kafka broker list for writing UnifiedLogs."); 22 | DEFINE_string(read_kafka_topic, "binlog", "Kafka topic for reading TxnLogs."); 23 | DEFINE_string(write_kafka_topic, "unified_log", 24 | "Kafka topic for writing UnifiedLogs."); 25 | 26 | DEFINE_int32(logs_per_epoch, 10000, "logs_per_epoch."); 27 | DEFINE_int32(seconds_per_epoch, 60, "seconds_per_epoch."); 28 | DEFINE_bool(use_logs_per_epoch, true, "use_logs_per_epoch."); 29 | 30 | DEFINE_string(etcd_endpoint, "127.0.0.1:2379", "etcd endpoint."); 31 | DEFINE_string(etcd_prefix, "", "etcd prefix."); 32 | 33 | DEFINE_int32(subgraph_num, 1, "Number of subgraphs."); 34 | 35 | DEFINE_int32(checkpoint_interval, 10, "Checkpoint interval in minutes."); 36 | DEFINE_string(checkpoint_dir, "/tmp/checkpoint", "Checkpoint directory."); 37 | 38 | DEFINE_bool(enable_bulkload, false, "Enable bulkload from existing data."); 39 | 40 | DEFINE_int32(num_threads, 4, "Number of threads for processing logs."); 41 | DEFINE_int32(max_queue_size, 20480, "Max queue size for logs."); 42 | -------------------------------------------------------------------------------- /converter/flags.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef CONVERTER_FLAGS_H_ 17 | #define CONVERTER_FLAGS_H_ 18 | 19 | #include 20 | #include 21 | 22 | DECLARE_string(read_kafka_broker_list); 23 | DECLARE_string(write_kafka_broker_list); 24 | DECLARE_string(read_kafka_topic); 25 | DECLARE_string(write_kafka_topic); 26 | 27 | DECLARE_int32(logs_per_epoch); 28 | DECLARE_int32(seconds_per_epoch); 29 | DECLARE_bool(use_logs_per_epoch); 30 | 31 | DECLARE_string(etcd_endpoint); 32 | DECLARE_string(etcd_prefix); 33 | 34 | DECLARE_int32(subgraph_num); 35 | DECLARE_bool(enable_bulkload); 36 | 37 | DECLARE_int32(checkpoint_interval); // in minutes 38 | DECLARE_string(checkpoint_dir); 39 | 40 | DECLARE_int32(num_threads); 41 | DECLARE_int32(max_queue_size); 42 | 43 | #endif // CONVERTER_FLAGS_H_ 44 | -------------------------------------------------------------------------------- /coordinator/.dockerignore: -------------------------------------------------------------------------------- 1 | .travis.yaml 2 | .openapi-generator-ignore 3 | README.md 4 | tox.ini 5 | git_push.sh 6 | test-requirements.txt 7 | setup.py 8 | 9 | # Byte-compiled / optimized / DLL files 10 | __pycache__/ 11 | *.py[cod] 12 | *$py.class 13 | 14 | # C extensions 15 | *.so 16 | 17 | # Distribution / packaging 18 | .Python 19 | env/ 20 | build/ 21 | develop-eggs/ 22 | dist/ 23 | downloads/ 24 | eggs/ 25 | .eggs/ 26 | lib/ 27 | lib64/ 28 | parts/ 29 | sdist/ 30 | var/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .coverage 49 | .coverage.* 50 | .cache 51 | nosetests.xml 52 | coverage.xml 53 | *,cover 54 | .hypothesis/ 55 | venv/ 56 | .python-version 57 | 58 | # Translations 59 | *.mo 60 | *.pot 61 | 62 | # Django stuff: 63 | *.log 64 | 65 | # Sphinx documentation 66 | docs/_build/ 67 | 68 | # PyBuilder 69 | target/ 70 | 71 | #Ipython Notebook 72 | .ipynb_checkpoints 73 | -------------------------------------------------------------------------------- /coordinator/.travis.yml: -------------------------------------------------------------------------------- 1 | # ref: https://docs.travis-ci.com/user/languages/python 2 | language: python 3 | python: 4 | - "3.2" 5 | - "3.3" 6 | - "3.4" 7 | - "3.5" 8 | - "3.6" 9 | - "3.7" 10 | - "3.8" 11 | # command to install dependencies 12 | install: "pip install -r requirements.txt" 13 | # command to run tests 14 | script: nosetests 15 | -------------------------------------------------------------------------------- /coordinator/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3-alpine 2 | 3 | RUN mkdir -p /usr/src/app 4 | WORKDIR /usr/src/app 5 | 6 | COPY requirements.txt /usr/src/app/ 7 | 8 | RUN pip3 install --no-cache-dir -r requirements.txt 9 | 10 | COPY . /usr/src/app 11 | 12 | ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION="python" 13 | # EXPOSE 8080 14 | 15 | CMD ["python3", "-m", "flex.server"] -------------------------------------------------------------------------------- /coordinator/README.md: -------------------------------------------------------------------------------- 1 | # OpenAPI generated server 2 | 3 | ## Overview 4 | This server was generated by the [OpenAPI Generator](https://openapi-generator.tech) project. By using the 5 | [OpenAPI-Spec](https://openapis.org) from a remote server, you can easily generate a server stub. This 6 | is an example of building a OpenAPI-enabled Flask server. 7 | 8 | This example uses the [Connexion](https://github.com/zalando/connexion) library on top of Flask. 9 | 10 | ## Requirements 11 | Python 3.5.2+ 12 | 13 | ## Usage 14 | To run the server, please execute the following from the root directory: 15 | 16 | ``` 17 | pip3 install -r requirements.txt 18 | python3 -m flex.server 19 | ``` 20 | 21 | and open your browser to here: 22 | 23 | ``` 24 | http://localhost:8080/ui/ 25 | ``` 26 | 27 | Your OpenAPI definition lives here: 28 | 29 | ``` 30 | http://localhost:8080/openapi.json 31 | ``` 32 | 33 | To launch the integration tests, use tox: 34 | ``` 35 | sudo pip install tox 36 | tox 37 | ``` 38 | 39 | ## Running with Docker 40 | 41 | To run the server on a Docker container, please execute the following from the root directory: 42 | 43 | ```bash 44 | # building the image 45 | docker build -t flex.server . 46 | 47 | # starting up a container 48 | docker run -p 8080:8080 flex.server 49 | ``` -------------------------------------------------------------------------------- /coordinator/flex/server/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GraphScope/GART/68767a870c9b3888015802660a09b5b83e7cd9a8/coordinator/flex/server/__init__.py -------------------------------------------------------------------------------- /coordinator/flex/server/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import connexion 4 | 5 | from flex.server import encoder 6 | from datetime import datetime 7 | 8 | 9 | def main(): 10 | current_timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S') 11 | with open("/tmp/cluster_create_time.txt", "w") as f: 12 | f.write(current_timestamp) 13 | app = connexion.App(__name__, specification_dir='./openapi/') 14 | app.app.json_encoder = encoder.JSONEncoder 15 | app.add_api('openapi.yaml', 16 | arguments={'title': 'GraphScope FLEX HTTP SERVICE API'}, 17 | pythonic_params=True) 18 | 19 | app.run(port=18080) 20 | 21 | 22 | if __name__ == '__main__': 23 | main() 24 | -------------------------------------------------------------------------------- /coordinator/flex/server/controllers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GraphScope/GART/68767a870c9b3888015802660a09b5b83e7cd9a8/coordinator/flex/server/controllers/__init__.py -------------------------------------------------------------------------------- /coordinator/flex/server/controllers/security_controller.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | -------------------------------------------------------------------------------- /coordinator/flex/server/controllers/utils_controller.py: -------------------------------------------------------------------------------- 1 | import connexion 2 | from typing import Dict 3 | from typing import Tuple 4 | from typing import Union 5 | 6 | from flex.server.models.error import Error # noqa: E501 7 | from flex.server.models.upload_file_response import UploadFileResponse # noqa: E501 8 | from flex.server import util 9 | 10 | 11 | def upload_file(filestorage=None): # noqa: E501 12 | """upload_file 13 | 14 | # noqa: E501 15 | 16 | :param filestorage: 17 | :type filestorage: str 18 | 19 | :rtype: Union[UploadFileResponse, Tuple[UploadFileResponse, int], Tuple[UploadFileResponse, int, Dict[str, str]] 20 | """ 21 | return 'do some magic!' 22 | -------------------------------------------------------------------------------- /coordinator/flex/server/encoder.py: -------------------------------------------------------------------------------- 1 | from connexion.apps.flask_app import FlaskJSONEncoder 2 | 3 | from flex.server.models.base_model import Model 4 | 5 | 6 | class JSONEncoder(FlaskJSONEncoder): 7 | include_nulls = False 8 | 9 | def default(self, o): 10 | if isinstance(o, Model): 11 | dikt = {} 12 | for attr in o.openapi_types: 13 | value = getattr(o, attr) 14 | if value is None and not self.include_nulls: 15 | continue 16 | attr = o.attribute_map[attr] 17 | dikt[attr] = value 18 | return dikt 19 | return FlaskJSONEncoder.default(self, o) 20 | -------------------------------------------------------------------------------- /coordinator/flex/server/models/base_model.py: -------------------------------------------------------------------------------- 1 | import pprint 2 | 3 | import typing 4 | 5 | from flex.server import util 6 | 7 | T = typing.TypeVar('T') 8 | 9 | 10 | class Model: 11 | # openapiTypes: The key is attribute name and the 12 | # value is attribute type. 13 | openapi_types: typing.Dict[str, type] = {} 14 | 15 | # attributeMap: The key is attribute name and the 16 | # value is json key in definition. 17 | attribute_map: typing.Dict[str, str] = {} 18 | 19 | @classmethod 20 | def from_dict(cls: typing.Type[T], dikt) -> T: 21 | """Returns the dict as a model""" 22 | return util.deserialize_model(dikt, cls) 23 | 24 | def to_dict(self): 25 | """Returns the model properties as a dict 26 | 27 | :rtype: dict 28 | """ 29 | result = {} 30 | 31 | for attr in self.openapi_types: 32 | value = getattr(self, attr) 33 | if isinstance(value, list): 34 | result[attr] = list(map( 35 | lambda x: x.to_dict() if hasattr(x, "to_dict") else x, 36 | value 37 | )) 38 | elif hasattr(value, "to_dict"): 39 | result[attr] = value.to_dict() 40 | elif isinstance(value, dict): 41 | result[attr] = dict(map( 42 | lambda item: (item[0], item[1].to_dict()) 43 | if hasattr(item[1], "to_dict") else item, 44 | value.items() 45 | )) 46 | else: 47 | result[attr] = value 48 | 49 | return result 50 | 51 | def to_str(self): 52 | """Returns the string representation of the model 53 | 54 | :rtype: str 55 | """ 56 | return pprint.pformat(self.to_dict()) 57 | 58 | def __repr__(self): 59 | """For `print` and `pprint`""" 60 | return self.to_str() 61 | 62 | def __eq__(self, other): 63 | """Returns true if both objects are equal""" 64 | return self.__dict__ == other.__dict__ 65 | 66 | def __ne__(self, other): 67 | """Returns true if both objects are not equal""" 68 | return not self == other 69 | -------------------------------------------------------------------------------- /coordinator/flex/server/models/base_vertex_type_x_csr_params.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime # noqa: F401 2 | 3 | from typing import List, Dict # noqa: F401 4 | 5 | from flex.server.models.base_model import Model 6 | from flex.server import util 7 | 8 | 9 | class BaseVertexTypeXCsrParams(Model): 10 | """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). 11 | 12 | Do not edit the class manually. 13 | """ 14 | 15 | def __init__(self, max_vertex_num=None): # noqa: E501 16 | """BaseVertexTypeXCsrParams - a model defined in OpenAPI 17 | 18 | :param max_vertex_num: The max_vertex_num of this BaseVertexTypeXCsrParams. # noqa: E501 19 | :type max_vertex_num: int 20 | """ 21 | self.openapi_types = { 22 | 'max_vertex_num': int 23 | } 24 | 25 | self.attribute_map = { 26 | 'max_vertex_num': 'max_vertex_num' 27 | } 28 | 29 | self._max_vertex_num = max_vertex_num 30 | 31 | @classmethod 32 | def from_dict(cls, dikt) -> 'BaseVertexTypeXCsrParams': 33 | """Returns the dict as a model 34 | 35 | :param dikt: A dict. 36 | :type: dict 37 | :return: The BaseVertexType_x_csr_params of this BaseVertexTypeXCsrParams. # noqa: E501 38 | :rtype: BaseVertexTypeXCsrParams 39 | """ 40 | return util.deserialize_model(dikt, cls) 41 | 42 | @property 43 | def max_vertex_num(self) -> int: 44 | """Gets the max_vertex_num of this BaseVertexTypeXCsrParams. 45 | 46 | 47 | :return: The max_vertex_num of this BaseVertexTypeXCsrParams. 48 | :rtype: int 49 | """ 50 | return self._max_vertex_num 51 | 52 | @max_vertex_num.setter 53 | def max_vertex_num(self, max_vertex_num: int): 54 | """Sets the max_vertex_num of this BaseVertexTypeXCsrParams. 55 | 56 | 57 | :param max_vertex_num: The max_vertex_num of this BaseVertexTypeXCsrParams. 58 | :type max_vertex_num: int 59 | """ 60 | 61 | self._max_vertex_num = max_vertex_num 62 | -------------------------------------------------------------------------------- /coordinator/flex/server/models/change_graph_version_request.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime # noqa: F401 2 | 3 | from typing import List, Dict # noqa: F401 4 | 5 | from flex.server.models.base_model import Model 6 | from flex.server import util 7 | 8 | 9 | class ChangeGraphVersionRequest(Model): 10 | """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). 11 | 12 | Do not edit the class manually. 13 | """ 14 | 15 | def __init__(self, version_id=None): # noqa: E501 16 | """ChangeGraphVersionRequest - a model defined in OpenAPI 17 | 18 | :param version_id: The version_id of this ChangeGraphVersionRequest. # noqa: E501 19 | :type version_id: str 20 | """ 21 | self.openapi_types = { 22 | 'version_id': str 23 | } 24 | 25 | self.attribute_map = { 26 | 'version_id': 'version_id' 27 | } 28 | 29 | self._version_id = version_id 30 | 31 | @classmethod 32 | def from_dict(cls, dikt) -> 'ChangeGraphVersionRequest': 33 | """Returns the dict as a model 34 | 35 | :param dikt: A dict. 36 | :type: dict 37 | :return: The ChangeGraphVersionRequest of this ChangeGraphVersionRequest. # noqa: E501 38 | :rtype: ChangeGraphVersionRequest 39 | """ 40 | return util.deserialize_model(dikt, cls) 41 | 42 | @property 43 | def version_id(self) -> str: 44 | """Gets the version_id of this ChangeGraphVersionRequest. 45 | 46 | 47 | :return: The version_id of this ChangeGraphVersionRequest. 48 | :rtype: str 49 | """ 50 | return self._version_id 51 | 52 | @version_id.setter 53 | def version_id(self, version_id: str): 54 | """Sets the version_id of this ChangeGraphVersionRequest. 55 | 56 | 57 | :param version_id: The version_id of this ChangeGraphVersionRequest. 58 | :type version_id: str 59 | """ 60 | if version_id is None: 61 | raise ValueError("Invalid value for `version_id`, must not be `None`") # noqa: E501 62 | 63 | self._version_id = version_id 64 | -------------------------------------------------------------------------------- /coordinator/flex/server/models/create_dataloading_job_response.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime # noqa: F401 2 | 3 | from typing import List, Dict # noqa: F401 4 | 5 | from flex.server.models.base_model import Model 6 | from flex.server import util 7 | 8 | 9 | class CreateDataloadingJobResponse(Model): 10 | """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). 11 | 12 | Do not edit the class manually. 13 | """ 14 | 15 | def __init__(self, job_id=None): # noqa: E501 16 | """CreateDataloadingJobResponse - a model defined in OpenAPI 17 | 18 | :param job_id: The job_id of this CreateDataloadingJobResponse. # noqa: E501 19 | :type job_id: str 20 | """ 21 | self.openapi_types = { 22 | 'job_id': str 23 | } 24 | 25 | self.attribute_map = { 26 | 'job_id': 'job_id' 27 | } 28 | 29 | self._job_id = job_id 30 | 31 | @classmethod 32 | def from_dict(cls, dikt) -> 'CreateDataloadingJobResponse': 33 | """Returns the dict as a model 34 | 35 | :param dikt: A dict. 36 | :type: dict 37 | :return: The CreateDataloadingJobResponse of this CreateDataloadingJobResponse. # noqa: E501 38 | :rtype: CreateDataloadingJobResponse 39 | """ 40 | return util.deserialize_model(dikt, cls) 41 | 42 | @property 43 | def job_id(self) -> str: 44 | """Gets the job_id of this CreateDataloadingJobResponse. 45 | 46 | 47 | :return: The job_id of this CreateDataloadingJobResponse. 48 | :rtype: str 49 | """ 50 | return self._job_id 51 | 52 | @job_id.setter 53 | def job_id(self, job_id: str): 54 | """Sets the job_id of this CreateDataloadingJobResponse. 55 | 56 | 57 | :param job_id: The job_id of this CreateDataloadingJobResponse. 58 | :type job_id: str 59 | """ 60 | if job_id is None: 61 | raise ValueError("Invalid value for `job_id`, must not be `None`") # noqa: E501 62 | 63 | self._job_id = job_id 64 | -------------------------------------------------------------------------------- /coordinator/flex/server/models/create_graph_response.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime # noqa: F401 2 | 3 | from typing import List, Dict # noqa: F401 4 | 5 | from flex.server.models.base_model import Model 6 | from flex.server import util 7 | 8 | 9 | class CreateGraphResponse(Model): 10 | """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). 11 | 12 | Do not edit the class manually. 13 | """ 14 | 15 | def __init__(self, graph_id=None): # noqa: E501 16 | """CreateGraphResponse - a model defined in OpenAPI 17 | 18 | :param graph_id: The graph_id of this CreateGraphResponse. # noqa: E501 19 | :type graph_id: str 20 | """ 21 | self.openapi_types = { 22 | 'graph_id': str 23 | } 24 | 25 | self.attribute_map = { 26 | 'graph_id': 'graph_id' 27 | } 28 | 29 | self._graph_id = graph_id 30 | 31 | @classmethod 32 | def from_dict(cls, dikt) -> 'CreateGraphResponse': 33 | """Returns the dict as a model 34 | 35 | :param dikt: A dict. 36 | :type: dict 37 | :return: The CreateGraphResponse of this CreateGraphResponse. # noqa: E501 38 | :rtype: CreateGraphResponse 39 | """ 40 | return util.deserialize_model(dikt, cls) 41 | 42 | @property 43 | def graph_id(self) -> str: 44 | """Gets the graph_id of this CreateGraphResponse. 45 | 46 | 47 | :return: The graph_id of this CreateGraphResponse. 48 | :rtype: str 49 | """ 50 | return self._graph_id 51 | 52 | @graph_id.setter 53 | def graph_id(self, graph_id: str): 54 | """Sets the graph_id of this CreateGraphResponse. 55 | 56 | 57 | :param graph_id: The graph_id of this CreateGraphResponse. 58 | :type graph_id: str 59 | """ 60 | if graph_id is None: 61 | raise ValueError("Invalid value for `graph_id`, must not be `None`") # noqa: E501 62 | 63 | self._graph_id = graph_id 64 | -------------------------------------------------------------------------------- /coordinator/flex/server/models/dataloading_job_config_vertices_inner.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime # noqa: F401 2 | 3 | from typing import List, Dict # noqa: F401 4 | 5 | from flex.server.models.base_model import Model 6 | from flex.server import util 7 | 8 | 9 | class DataloadingJobConfigVerticesInner(Model): 10 | """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). 11 | 12 | Do not edit the class manually. 13 | """ 14 | 15 | def __init__(self, type_name=None): # noqa: E501 16 | """DataloadingJobConfigVerticesInner - a model defined in OpenAPI 17 | 18 | :param type_name: The type_name of this DataloadingJobConfigVerticesInner. # noqa: E501 19 | :type type_name: str 20 | """ 21 | self.openapi_types = { 22 | 'type_name': str 23 | } 24 | 25 | self.attribute_map = { 26 | 'type_name': 'type_name' 27 | } 28 | 29 | self._type_name = type_name 30 | 31 | @classmethod 32 | def from_dict(cls, dikt) -> 'DataloadingJobConfigVerticesInner': 33 | """Returns the dict as a model 34 | 35 | :param dikt: A dict. 36 | :type: dict 37 | :return: The DataloadingJobConfig_vertices_inner of this DataloadingJobConfigVerticesInner. # noqa: E501 38 | :rtype: DataloadingJobConfigVerticesInner 39 | """ 40 | return util.deserialize_model(dikt, cls) 41 | 42 | @property 43 | def type_name(self) -> str: 44 | """Gets the type_name of this DataloadingJobConfigVerticesInner. 45 | 46 | 47 | :return: The type_name of this DataloadingJobConfigVerticesInner. 48 | :rtype: str 49 | """ 50 | return self._type_name 51 | 52 | @type_name.setter 53 | def type_name(self, type_name: str): 54 | """Sets the type_name of this DataloadingJobConfigVerticesInner. 55 | 56 | 57 | :param type_name: The type_name of this DataloadingJobConfigVerticesInner. 58 | :type type_name: str 59 | """ 60 | 61 | self._type_name = type_name 62 | -------------------------------------------------------------------------------- /coordinator/flex/server/models/dataloading_mr_job_config.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime # noqa: F401 2 | 3 | from typing import List, Dict # noqa: F401 4 | 5 | from flex.server.models.base_model import Model 6 | from flex.server import util 7 | 8 | 9 | class DataloadingMRJobConfig(Model): 10 | """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). 11 | 12 | Do not edit the class manually. 13 | """ 14 | 15 | def __init__(self, config=None): # noqa: E501 16 | """DataloadingMRJobConfig - a model defined in OpenAPI 17 | 18 | :param config: The config of this DataloadingMRJobConfig. # noqa: E501 19 | :type config: str 20 | """ 21 | self.openapi_types = { 22 | 'config': str 23 | } 24 | 25 | self.attribute_map = { 26 | 'config': 'config' 27 | } 28 | 29 | self._config = config 30 | 31 | @classmethod 32 | def from_dict(cls, dikt) -> 'DataloadingMRJobConfig': 33 | """Returns the dict as a model 34 | 35 | :param dikt: A dict. 36 | :type: dict 37 | :return: The DataloadingMRJobConfig of this DataloadingMRJobConfig. # noqa: E501 38 | :rtype: DataloadingMRJobConfig 39 | """ 40 | return util.deserialize_model(dikt, cls) 41 | 42 | @property 43 | def config(self) -> str: 44 | """Gets the config of this DataloadingMRJobConfig. 45 | 46 | 47 | :return: The config of this DataloadingMRJobConfig. 48 | :rtype: str 49 | """ 50 | return self._config 51 | 52 | @config.setter 53 | def config(self, config: str): 54 | """Sets the config of this DataloadingMRJobConfig. 55 | 56 | 57 | :param config: The config of this DataloadingMRJobConfig. 58 | :type config: str 59 | """ 60 | if config is None: 61 | raise ValueError("Invalid value for `config`, must not be `None`") # noqa: E501 62 | 63 | self._config = config 64 | -------------------------------------------------------------------------------- /coordinator/flex/server/models/date_type.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime # noqa: F401 2 | 3 | from typing import List, Dict # noqa: F401 4 | 5 | from flex.server.models.base_model import Model 6 | from flex.server import util 7 | 8 | 9 | class DateType(Model): 10 | """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). 11 | 12 | Do not edit the class manually. 13 | """ 14 | 15 | def __init__(self, date32=None): # noqa: E501 16 | """DateType - a model defined in OpenAPI 17 | 18 | :param date32: The date32 of this DateType. # noqa: E501 19 | :type date32: str 20 | """ 21 | self.openapi_types = { 22 | 'date32': str 23 | } 24 | 25 | self.attribute_map = { 26 | 'date32': 'date32' 27 | } 28 | 29 | self._date32 = date32 30 | 31 | @classmethod 32 | def from_dict(cls, dikt) -> 'DateType': 33 | """Returns the dict as a model 34 | 35 | :param dikt: A dict. 36 | :type: dict 37 | :return: The DateType of this DateType. # noqa: E501 38 | :rtype: DateType 39 | """ 40 | return util.deserialize_model(dikt, cls) 41 | 42 | @property 43 | def date32(self) -> str: 44 | """Gets the date32 of this DateType. 45 | 46 | 47 | :return: The date32 of this DateType. 48 | :rtype: str 49 | """ 50 | return self._date32 51 | 52 | @date32.setter 53 | def date32(self, date32: str): 54 | """Sets the date32 of this DateType. 55 | 56 | 57 | :param date32: The date32 of this DateType. 58 | :type date32: str 59 | """ 60 | if date32 is None: 61 | raise ValueError("Invalid value for `date32`, must not be `None`") # noqa: E501 62 | 63 | self._date32 = date32 64 | -------------------------------------------------------------------------------- /coordinator/flex/server/models/get_pod_log_response.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime # noqa: F401 2 | 3 | from typing import List, Dict # noqa: F401 4 | 5 | from flex.server.models.base_model import Model 6 | from flex.server import util 7 | 8 | 9 | class GetPodLogResponse(Model): 10 | """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). 11 | 12 | Do not edit the class manually. 13 | """ 14 | 15 | def __init__(self, log=None): # noqa: E501 16 | """GetPodLogResponse - a model defined in OpenAPI 17 | 18 | :param log: The log of this GetPodLogResponse. # noqa: E501 19 | :type log: Dict[str, object] 20 | """ 21 | self.openapi_types = { 22 | 'log': Dict[str, object] 23 | } 24 | 25 | self.attribute_map = { 26 | 'log': 'log' 27 | } 28 | 29 | self._log = log 30 | 31 | @classmethod 32 | def from_dict(cls, dikt) -> 'GetPodLogResponse': 33 | """Returns the dict as a model 34 | 35 | :param dikt: A dict. 36 | :type: dict 37 | :return: The GetPodLogResponse of this GetPodLogResponse. # noqa: E501 38 | :rtype: GetPodLogResponse 39 | """ 40 | return util.deserialize_model(dikt, cls) 41 | 42 | @property 43 | def log(self) -> Dict[str, object]: 44 | """Gets the log of this GetPodLogResponse. 45 | 46 | 47 | :return: The log of this GetPodLogResponse. 48 | :rtype: Dict[str, object] 49 | """ 50 | return self._log 51 | 52 | @log.setter 53 | def log(self, log: Dict[str, object]): 54 | """Sets the log of this GetPodLogResponse. 55 | 56 | 57 | :param log: The log of this GetPodLogResponse. 58 | :type log: Dict[str, object] 59 | """ 60 | if log is None: 61 | raise ValueError("Invalid value for `log`, must not be `None`") # noqa: E501 62 | 63 | self._log = log 64 | -------------------------------------------------------------------------------- /coordinator/flex/server/models/long_text.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime # noqa: F401 2 | 3 | from typing import List, Dict # noqa: F401 4 | 5 | from flex.server.models.base_model import Model 6 | from flex.server import util 7 | 8 | 9 | class LongText(Model): 10 | """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). 11 | 12 | Do not edit the class manually. 13 | """ 14 | 15 | def __init__(self, long_text=None): # noqa: E501 16 | """LongText - a model defined in OpenAPI 17 | 18 | :param long_text: The long_text of this LongText. # noqa: E501 19 | :type long_text: str 20 | """ 21 | self.openapi_types = { 22 | 'long_text': str 23 | } 24 | 25 | self.attribute_map = { 26 | 'long_text': 'long_text' 27 | } 28 | 29 | self._long_text = long_text 30 | 31 | @classmethod 32 | def from_dict(cls, dikt) -> 'LongText': 33 | """Returns the dict as a model 34 | 35 | :param dikt: A dict. 36 | :type: dict 37 | :return: The LongText of this LongText. # noqa: E501 38 | :rtype: LongText 39 | """ 40 | return util.deserialize_model(dikt, cls) 41 | 42 | @property 43 | def long_text(self) -> str: 44 | """Gets the long_text of this LongText. 45 | 46 | 47 | :return: The long_text of this LongText. 48 | :rtype: str 49 | """ 50 | return self._long_text 51 | 52 | @long_text.setter 53 | def long_text(self, long_text: str): 54 | """Sets the long_text of this LongText. 55 | 56 | 57 | :param long_text: The long_text of this LongText. 58 | :type long_text: str 59 | """ 60 | if long_text is None: 61 | raise ValueError("Invalid value for `long_text`, must not be `None`") # noqa: E501 62 | 63 | self._long_text = long_text 64 | -------------------------------------------------------------------------------- /coordinator/flex/server/models/start_service_request.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime # noqa: F401 2 | 3 | from typing import List, Dict # noqa: F401 4 | 5 | from flex.server.models.base_model import Model 6 | from flex.server import util 7 | 8 | 9 | class StartServiceRequest(Model): 10 | """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). 11 | 12 | Do not edit the class manually. 13 | """ 14 | 15 | def __init__(self, graph_id=None): # noqa: E501 16 | """StartServiceRequest - a model defined in OpenAPI 17 | 18 | :param graph_id: The graph_id of this StartServiceRequest. # noqa: E501 19 | :type graph_id: str 20 | """ 21 | self.openapi_types = { 22 | 'graph_id': str 23 | } 24 | 25 | self.attribute_map = { 26 | 'graph_id': 'graph_id' 27 | } 28 | 29 | self._graph_id = graph_id 30 | 31 | @classmethod 32 | def from_dict(cls, dikt) -> 'StartServiceRequest': 33 | """Returns the dict as a model 34 | 35 | :param dikt: A dict. 36 | :type: dict 37 | :return: The StartServiceRequest of this StartServiceRequest. # noqa: E501 38 | :rtype: StartServiceRequest 39 | """ 40 | return util.deserialize_model(dikt, cls) 41 | 42 | @property 43 | def graph_id(self) -> str: 44 | """Gets the graph_id of this StartServiceRequest. 45 | 46 | 47 | :return: The graph_id of this StartServiceRequest. 48 | :rtype: str 49 | """ 50 | return self._graph_id 51 | 52 | @graph_id.setter 53 | def graph_id(self, graph_id: str): 54 | """Sets the graph_id of this StartServiceRequest. 55 | 56 | 57 | :param graph_id: The graph_id of this StartServiceRequest. 58 | :type graph_id: str 59 | """ 60 | 61 | self._graph_id = graph_id 62 | -------------------------------------------------------------------------------- /coordinator/flex/server/models/string_type.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime # noqa: F401 2 | 3 | from typing import List, Dict # noqa: F401 4 | 5 | from flex.server.models.base_model import Model 6 | from flex.server.models.string_type_string import StringTypeString 7 | from flex.server import util 8 | 9 | from flex.server.models.string_type_string import StringTypeString # noqa: E501 10 | 11 | class StringType(Model): 12 | """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). 13 | 14 | Do not edit the class manually. 15 | """ 16 | 17 | def __init__(self, string=None): # noqa: E501 18 | """StringType - a model defined in OpenAPI 19 | 20 | :param string: The string of this StringType. # noqa: E501 21 | :type string: StringTypeString 22 | """ 23 | self.openapi_types = { 24 | 'string': StringTypeString 25 | } 26 | 27 | self.attribute_map = { 28 | 'string': 'string' 29 | } 30 | 31 | self._string = string 32 | 33 | @classmethod 34 | def from_dict(cls, dikt) -> 'StringType': 35 | """Returns the dict as a model 36 | 37 | :param dikt: A dict. 38 | :type: dict 39 | :return: The StringType of this StringType. # noqa: E501 40 | :rtype: StringType 41 | """ 42 | return util.deserialize_model(dikt, cls) 43 | 44 | @property 45 | def string(self) -> StringTypeString: 46 | """Gets the string of this StringType. 47 | 48 | 49 | :return: The string of this StringType. 50 | :rtype: StringTypeString 51 | """ 52 | return self._string 53 | 54 | @string.setter 55 | def string(self, string: StringTypeString): 56 | """Sets the string of this StringType. 57 | 58 | 59 | :param string: The string of this StringType. 60 | :type string: StringTypeString 61 | """ 62 | if string is None: 63 | raise ValueError("Invalid value for `string`, must not be `None`") # noqa: E501 64 | 65 | self._string = string 66 | -------------------------------------------------------------------------------- /coordinator/flex/server/models/string_type_string.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime # noqa: F401 2 | 3 | from typing import List, Dict # noqa: F401 4 | 5 | from flex.server.models.base_model import Model 6 | from flex.server.models.long_text import LongText 7 | from flex.server import util 8 | 9 | from flex.server.models.long_text import LongText # noqa: E501 10 | 11 | class StringTypeString(Model): 12 | """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). 13 | 14 | Do not edit the class manually. 15 | """ 16 | 17 | def __init__(self, long_text=None): # noqa: E501 18 | """StringTypeString - a model defined in OpenAPI 19 | 20 | :param long_text: The long_text of this StringTypeString. # noqa: E501 21 | :type long_text: str 22 | """ 23 | self.openapi_types = { 24 | 'long_text': str 25 | } 26 | 27 | self.attribute_map = { 28 | 'long_text': 'long_text' 29 | } 30 | 31 | self._long_text = long_text 32 | 33 | @classmethod 34 | def from_dict(cls, dikt) -> 'StringTypeString': 35 | """Returns the dict as a model 36 | 37 | :param dikt: A dict. 38 | :type: dict 39 | :return: The StringType_string of this StringTypeString. # noqa: E501 40 | :rtype: StringTypeString 41 | """ 42 | return util.deserialize_model(dikt, cls) 43 | 44 | @property 45 | def long_text(self) -> str: 46 | """Gets the long_text of this StringTypeString. 47 | 48 | 49 | :return: The long_text of this StringTypeString. 50 | :rtype: str 51 | """ 52 | return self._long_text 53 | 54 | @long_text.setter 55 | def long_text(self, long_text: str): 56 | """Sets the long_text of this StringTypeString. 57 | 58 | 59 | :param long_text: The long_text of this StringTypeString. 60 | :type long_text: str 61 | """ 62 | if long_text is None: 63 | raise ValueError("Invalid value for `long_text`, must not be `None`") # noqa: E501 64 | 65 | self._long_text = long_text 66 | -------------------------------------------------------------------------------- /coordinator/flex/server/models/temporal_type.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime # noqa: F401 2 | 3 | from typing import List, Dict # noqa: F401 4 | 5 | from flex.server.models.base_model import Model 6 | from flex.server.models.temporal_type_temporal import TemporalTypeTemporal 7 | from flex.server import util 8 | 9 | from flex.server.models.temporal_type_temporal import TemporalTypeTemporal # noqa: E501 10 | 11 | class TemporalType(Model): 12 | """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). 13 | 14 | Do not edit the class manually. 15 | """ 16 | 17 | def __init__(self, temporal=None): # noqa: E501 18 | """TemporalType - a model defined in OpenAPI 19 | 20 | :param temporal: The temporal of this TemporalType. # noqa: E501 21 | :type temporal: TemporalTypeTemporal 22 | """ 23 | self.openapi_types = { 24 | 'temporal': TemporalTypeTemporal 25 | } 26 | 27 | self.attribute_map = { 28 | 'temporal': 'temporal' 29 | } 30 | 31 | self._temporal = temporal 32 | 33 | @classmethod 34 | def from_dict(cls, dikt) -> 'TemporalType': 35 | """Returns the dict as a model 36 | 37 | :param dikt: A dict. 38 | :type: dict 39 | :return: The TemporalType of this TemporalType. # noqa: E501 40 | :rtype: TemporalType 41 | """ 42 | return util.deserialize_model(dikt, cls) 43 | 44 | @property 45 | def temporal(self) -> TemporalTypeTemporal: 46 | """Gets the temporal of this TemporalType. 47 | 48 | 49 | :return: The temporal of this TemporalType. 50 | :rtype: TemporalTypeTemporal 51 | """ 52 | return self._temporal 53 | 54 | @temporal.setter 55 | def temporal(self, temporal: TemporalTypeTemporal): 56 | """Sets the temporal of this TemporalType. 57 | 58 | 59 | :param temporal: The temporal of this TemporalType. 60 | :type temporal: TemporalTypeTemporal 61 | """ 62 | if temporal is None: 63 | raise ValueError("Invalid value for `temporal`, must not be `None`") # noqa: E501 64 | 65 | self._temporal = temporal 66 | -------------------------------------------------------------------------------- /coordinator/flex/server/models/time_stamp_type.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime # noqa: F401 2 | 3 | from typing import List, Dict # noqa: F401 4 | 5 | from flex.server.models.base_model import Model 6 | from flex.server import util 7 | 8 | 9 | class TimeStampType(Model): 10 | """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). 11 | 12 | Do not edit the class manually. 13 | """ 14 | 15 | def __init__(self, timestamp=None): # noqa: E501 16 | """TimeStampType - a model defined in OpenAPI 17 | 18 | :param timestamp: The timestamp of this TimeStampType. # noqa: E501 19 | :type timestamp: str 20 | """ 21 | self.openapi_types = { 22 | 'timestamp': str 23 | } 24 | 25 | self.attribute_map = { 26 | 'timestamp': 'timestamp' 27 | } 28 | 29 | self._timestamp = timestamp 30 | 31 | @classmethod 32 | def from_dict(cls, dikt) -> 'TimeStampType': 33 | """Returns the dict as a model 34 | 35 | :param dikt: A dict. 36 | :type: dict 37 | :return: The TimeStampType of this TimeStampType. # noqa: E501 38 | :rtype: TimeStampType 39 | """ 40 | return util.deserialize_model(dikt, cls) 41 | 42 | @property 43 | def timestamp(self) -> str: 44 | """Gets the timestamp of this TimeStampType. 45 | 46 | 47 | :return: The timestamp of this TimeStampType. 48 | :rtype: str 49 | """ 50 | return self._timestamp 51 | 52 | @timestamp.setter 53 | def timestamp(self, timestamp: str): 54 | """Sets the timestamp of this TimeStampType. 55 | 56 | 57 | :param timestamp: The timestamp of this TimeStampType. 58 | :type timestamp: str 59 | """ 60 | if timestamp is None: 61 | raise ValueError("Invalid value for `timestamp`, must not be `None`") # noqa: E501 62 | 63 | self._timestamp = timestamp 64 | -------------------------------------------------------------------------------- /coordinator/flex/server/models/update_stored_proc_request.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime # noqa: F401 2 | 3 | from typing import List, Dict # noqa: F401 4 | 5 | from flex.server.models.base_model import Model 6 | from flex.server import util 7 | 8 | 9 | class UpdateStoredProcRequest(Model): 10 | """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). 11 | 12 | Do not edit the class manually. 13 | """ 14 | 15 | def __init__(self, description=None): # noqa: E501 16 | """UpdateStoredProcRequest - a model defined in OpenAPI 17 | 18 | :param description: The description of this UpdateStoredProcRequest. # noqa: E501 19 | :type description: str 20 | """ 21 | self.openapi_types = { 22 | 'description': str 23 | } 24 | 25 | self.attribute_map = { 26 | 'description': 'description' 27 | } 28 | 29 | self._description = description 30 | 31 | @classmethod 32 | def from_dict(cls, dikt) -> 'UpdateStoredProcRequest': 33 | """Returns the dict as a model 34 | 35 | :param dikt: A dict. 36 | :type: dict 37 | :return: The UpdateStoredProcRequest of this UpdateStoredProcRequest. # noqa: E501 38 | :rtype: UpdateStoredProcRequest 39 | """ 40 | return util.deserialize_model(dikt, cls) 41 | 42 | @property 43 | def description(self) -> str: 44 | """Gets the description of this UpdateStoredProcRequest. 45 | 46 | 47 | :return: The description of this UpdateStoredProcRequest. 48 | :rtype: str 49 | """ 50 | return self._description 51 | 52 | @description.setter 53 | def description(self, description: str): 54 | """Sets the description of this UpdateStoredProcRequest. 55 | 56 | 57 | :param description: The description of this UpdateStoredProcRequest. 58 | :type description: str 59 | """ 60 | if description is None: 61 | raise ValueError("Invalid value for `description`, must not be `None`") # noqa: E501 62 | 63 | self._description = description 64 | -------------------------------------------------------------------------------- /coordinator/flex/server/test/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import connexion 4 | from flask_testing import TestCase 5 | 6 | from flex.server.encoder import JSONEncoder 7 | 8 | 9 | class BaseTestCase(TestCase): 10 | 11 | def create_app(self): 12 | logging.getLogger('connexion.operation').setLevel('ERROR') 13 | app = connexion.App(__name__, specification_dir='../openapi/') 14 | app.app.json_encoder = JSONEncoder 15 | app.add_api('openapi.yaml', pythonic_params=True) 16 | return app.app 17 | -------------------------------------------------------------------------------- /coordinator/flex/server/test/test_utils_controller.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from flask import json 4 | 5 | from flex.server.models.error import Error # noqa: E501 6 | from flex.server.models.upload_file_response import UploadFileResponse # noqa: E501 7 | from flex.server.test import BaseTestCase 8 | 9 | 10 | class TestUtilsController(BaseTestCase): 11 | """UtilsController integration test stubs""" 12 | 13 | @unittest.skip("multipart/form-data not supported by Connexion") 14 | def test_upload_file(self): 15 | """Test case for upload_file 16 | 17 | 18 | """ 19 | headers = { 20 | 'Accept': 'application/json', 21 | 'Content-Type': 'multipart/form-data', 22 | } 23 | data = dict(filestorage='/path/to/file') 24 | response = self.client.open( 25 | '/api/v1/file/uploading', 26 | method='POST', 27 | headers=headers, 28 | data=data, 29 | content_type='multipart/form-data') 30 | self.assert200(response, 31 | 'Response body is : ' + response.data.decode('utf-8')) 32 | 33 | 34 | if __name__ == '__main__': 35 | unittest.main() 36 | -------------------------------------------------------------------------------- /coordinator/flex/server/typing_utils.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | if sys.version_info < (3, 7): 4 | import typing 5 | 6 | def is_generic(klass): 7 | """ Determine whether klass is a generic class """ 8 | return type(klass) == typing.GenericMeta 9 | 10 | def is_dict(klass): 11 | """ Determine whether klass is a Dict """ 12 | return klass.__extra__ == dict 13 | 14 | def is_list(klass): 15 | """ Determine whether klass is a List """ 16 | return klass.__extra__ == list 17 | 18 | else: 19 | 20 | def is_generic(klass): 21 | """ Determine whether klass is a generic class """ 22 | return hasattr(klass, '__origin__') 23 | 24 | def is_dict(klass): 25 | """ Determine whether klass is a Dict """ 26 | return klass.__origin__ == dict 27 | 28 | def is_list(klass): 29 | """ Determine whether klass is a List """ 30 | return klass.__origin__ == list 31 | -------------------------------------------------------------------------------- /coordinator/requirements.txt: -------------------------------------------------------------------------------- 1 | connexion[swagger-ui] >= 2.6.0; python_version>="3.6" 2 | # 2.3 is the last version that supports python 3.4-3.5 3 | connexion[swagger-ui] <= 2.3.0; python_version=="3.5" or python_version=="3.4" 4 | # prevent breaking dependencies from advent of connexion>=3.0 5 | connexion[swagger-ui] <= 2.14.2; python_version>"3.4" 6 | # connexion requires werkzeug but connexion < 2.4.0 does not install werkzeug 7 | # we must peg werkzeug versions below to fix connexion 8 | # https://github.com/zalando/connexion/pull/1044 9 | werkzeug == 0.16.1; python_version=="3.5" or python_version=="3.4" 10 | swagger-ui-bundle >= 0.0.2 11 | python_dateutil >= 2.6.0 12 | setuptools >= 21.0.0 13 | Flask == 2.1.1 14 | etcd3 15 | kubernetes -------------------------------------------------------------------------------- /coordinator/setup.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from setuptools import setup, find_packages 3 | 4 | NAME = "flex.server" 5 | VERSION = "1.0.0" 6 | 7 | # To install the library, run the following 8 | # 9 | # python setup.py install 10 | # 11 | # prerequisite: setuptools 12 | # http://pypi.python.org/pypi/setuptools 13 | 14 | REQUIRES = [ 15 | "connexion>=2.0.2", 16 | "swagger-ui-bundle>=0.0.2", 17 | "python_dateutil>=2.6.0" 18 | ] 19 | 20 | setup( 21 | name=NAME, 22 | version=VERSION, 23 | description="GraphScope FLEX HTTP SERVICE API", 24 | author_email="graphscope@alibaba-inc.com", 25 | url="", 26 | keywords=["OpenAPI", "GraphScope FLEX HTTP SERVICE API"], 27 | install_requires=REQUIRES, 28 | packages=find_packages(), 29 | package_data={'': ['openapi/openapi.yaml']}, 30 | include_package_data=True, 31 | entry_points={ 32 | 'console_scripts': ['flex.server=flex.server.__main__:main']}, 33 | long_description="""\ 34 | This is a specification for GraphScope FLEX HTTP service based on the OpenAPI 3.0 specification. You can find out more details about specification at [doc](https://swagger.io/specification/v3/). Some useful links: - [GraphScope Repository](https://github.com/alibaba/GraphScope) - [The Source API definition for GraphScope Interactive](https://github.com/GraphScope/portal/tree/main/httpservice) 35 | """ 36 | ) 37 | 38 | -------------------------------------------------------------------------------- /coordinator/test-requirements.txt: -------------------------------------------------------------------------------- 1 | pytest~=7.1.0 2 | pytest-cov>=2.8.1 3 | pytest-randomly>=1.2.3 4 | Flask-Testing==0.8.1 5 | -------------------------------------------------------------------------------- /coordinator/tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py3 3 | skipsdist=True 4 | 5 | [testenv] 6 | deps=-r{toxinidir}/requirements.txt 7 | -r{toxinidir}/test-requirements.txt 8 | {toxinidir} 9 | 10 | commands= 11 | pytest --cov=flex.server 12 | -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GraphScope/GART/68767a870c9b3888015802660a09b5b83e7cd9a8/docs/.nojekyll -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile doxygen 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile doxygen 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | 22 | doxygen: 23 | @mkdir -p _build 24 | # @doxygen Doxyfile || true 25 | -------------------------------------------------------------------------------- /docs/_static/css/custom.css: -------------------------------------------------------------------------------- 1 | .fa.fa-2x { 2 | font-size: 36px; 3 | } 4 | 5 | /* Decrease the padding of button block in panels */ 6 | div.card-body.card-body-less-padding { 7 | padding: 0.25em; 8 | } 9 | 10 | /* Disable theme toggle */ 11 | div.theme-toggle-container.theme-toggle-content { 12 | display: none; 13 | } 14 | 15 | /* Proper distance at the bottom of TOC tree */ 16 | .sidebar-container > .sidebar-sticky > .sidebar-scroll > .sidebar-tree { 17 | padding-bottom: 1em; 18 | } 19 | 20 | /* Admonition for drop-down like details component */ 21 | 22 | .admonition.admonition-details { 23 | padding: 0 0 0 0; 24 | margin: 0 auto; 25 | } 26 | 27 | .admonition.admonition-details.active > :not(.admonition-title) { 28 | display: inherit; 29 | } 30 | 31 | .admonition.admonition-details > :not(.admonition-title) { 32 | display: none; 33 | } 34 | 35 | .admonition.admonition-details > p.admonition-title, p.topic-title { 36 | margin: 0 0 0 0; 37 | } 38 | 39 | .admonition.admonition-details > blockquote { 40 | padding: 0; 41 | } 42 | 43 | .admonition.admonition-details > blockquote > div > div[class^=highlight-] { 44 | margin: 0; 45 | } 46 | 47 | .admonition.admonition-details > blockquote > div > div[class^=table-wrapper] { 48 | margin-top: 0; 49 | margin-bottom: 0; 50 | padding: 0 0 0 0; 51 | } 52 | 53 | .admonition.admonition-details > blockquote > div > div[class^=table-wrapper] > table { 54 | min-width: 100%; 55 | } 56 | -------------------------------------------------------------------------------- /docs/_static/webfonts/fa-brands-400.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GraphScope/GART/68767a870c9b3888015802660a09b5b83e7cd9a8/docs/_static/webfonts/fa-brands-400.ttf -------------------------------------------------------------------------------- /docs/_static/webfonts/fa-brands-400.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GraphScope/GART/68767a870c9b3888015802660a09b5b83e7cd9a8/docs/_static/webfonts/fa-brands-400.woff2 -------------------------------------------------------------------------------- /docs/_templates/base.html: -------------------------------------------------------------------------------- 1 | {% extends "!base.html" %} 2 | 3 | {% block body %} 4 | 7 | 8 | 48 | {% endblock %} 49 | -------------------------------------------------------------------------------- /docs/_templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 16 | GART: Graph Analysis on Relational Transactional Datasets 17 | 18 | 19 | 20 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /docs/_templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends "!layout.html" %} 2 | 3 | {%- block extrahead %} 4 | 5 | {{ super() }} 6 | 7 | {% endblock %} 8 | -------------------------------------------------------------------------------- /docs/documentation/deployment/deploy-k8s.rst: -------------------------------------------------------------------------------- 1 | .. _deploy-k8s: 2 | 3 | Deployment on Kubernetes 4 | ======================================== 5 | 6 | GART also supports deployment on Kubernetes. The following steps will guide you through the process of deploying GART on Kubernetes. 7 | 8 | -------------------------------------------------------------------------------- /docs/documentation/deployment/env-setup.rst: -------------------------------------------------------------------------------- 1 | Environment Setup 2 | ==================== 3 | 4 | This section describes how to deploy GART locally. 5 | 6 | Requirements 7 | ------------ 8 | 9 | - `glog`_ 10 | - `gflags`_ 11 | - `etcd-cpp-apiv3`_ 12 | - `TBB`_ 13 | - `librdkafka`_ 14 | - `Vineyard`_ 15 | - `Apache Kafka`_ 16 | - `Debezium`_ 17 | 18 | 19 | Install Dependencies 20 | -------------------- 21 | 22 | To simplify installing dependencies, we provide a script (`scripts/install-deps.sh`_) for installing GART's dependency environment. 23 | 24 | .. code-block:: bash 25 | :linenos: 26 | 27 | $ git clone https://github.com/GraphScope/GART.git gart 28 | $ cd gart 29 | $ ./scripts/install-deps.sh /path/to/your/installation # install dependencies 30 | 31 | Compile and Install GART 32 | ------------------------ 33 | 34 | .. code-block:: bash 35 | :linenos: 36 | 37 | $ mkdir -p build; cd build 38 | $ cmake .. -DCMAKE_BUILD_TYPE=Release 39 | $ make -j 40 | $ sudo make install 41 | 42 | .. _glog: https://github.com/google/glog 43 | .. _gflags: https://github.com/gflags/gflags 44 | .. _etcd-cpp-apiv3: https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3 45 | .. _TBB: https://github.com/oneapi-src/oneTBB 46 | .. _librdkafka: https://github.com/confluentinc/librdkafka 47 | .. _Vineyard: https://github.com/v6d-io/v6d 48 | .. _Apache Kafka: https://kafka.apache.org/quickstart 49 | .. _Debezium: https://github.com/debezium/debezium 50 | .. _scripts/install-deps.sh: https://github.com/GraphScope/GART/blob/main/scripts/install-deps.sh 51 | -------------------------------------------------------------------------------- /docs/documentation/design/graph-storage.rst: -------------------------------------------------------------------------------- 1 | .. _gstore: 2 | 3 | Dynamic Graph Storage 4 | ============================ 5 | 6 | Graph Representation 7 | -------------------- 8 | 9 | To enable dynamic graph data analysis of data that is changing, GART provides a dynamic graph store. The execution engine can read the latest data **snapshot** on this storage for real-time data analysis. 10 | 11 | To balance performance, GART's graph store is implemented based on snapshots. Conceptually, the user can assume that the system will generate a complete snapshot of the graph data at regular intervals (default is 50 milliseconds). Note that the optimization techniques provided by GART do not create a full copy of the data every time. 12 | 13 | Users can think of the graph data stored in GART as a multi-version *compressed sparse row* (CSR) representation. 14 | Specific details of the design can be found in `our paper `_. 15 | In GART, we refer to the time interval between snapshots as the **epoch**, and each snapshot version will be identified by the epoch number. 16 | 17 | Distributed Store 18 | -------------------- 19 | 20 | The graph data is stored in `Vineyard`_ in a distributed manner. 21 | Vineyard (v6d) is an in-memory immutable data manager that offers out-of-the-box high-level abstraction and zero-copy sharing for distributed data in big data tasks. 22 | 23 | Graph data is partitioned into multiple data partitions, called sub-graph, and each data partition is stored in a machine. A machine can hold multiple data partitions. 24 | We can use the machine ID and data partition ID to identify a specific data partition. 25 | 26 | Metadata Management 27 | -------------------- 28 | 29 | GART manages all the metadata stored in `etcd`_, a distributed key-value store. 30 | The metadata includes the epoch number, machine ID, and data partition ID for each snapshot. 31 | Users can retrieve the metadata by querying the etcd store. 32 | 33 | .. code:: bash 34 | 35 | # Identifies the latest readable epoch number 36 | # For which the machine ID is 0 and the data partition is 1 37 | etcdctl get --prefix "gart_meta_gart_blob_m0_p1_e" --keys-only 38 | 39 | 40 | .. _Vineyard: https://v6d.io/ 41 | .. _etcd: https://etcd.io/ 42 | -------------------------------------------------------------------------------- /docs/documentation/getting-started/introduction.rst: -------------------------------------------------------------------------------- 1 | Introduction 2 | =============== 3 | 4 | .. include:: intro-content.rst 5 | -------------------------------------------------------------------------------- /docs/documentation/tutorials/data-source-config.rst: -------------------------------------------------------------------------------- 1 | Data Source Configuration 2 | ============================== 3 | 4 | .. toctree:: 5 | :maxdepth: 1 6 | :caption: TOC 7 | :hidden: 8 | 9 | data-source-config/postgresql 10 | data-source-config/mysql 11 | data-source-config/storage 12 | 13 | 14 | GART supports different types of data sources for data model transformation to provide real-time graph analytic processing. Because of the need to keep track of data updates, some permissions need to be configured on the data sources. 15 | 16 | In this section, we will introduce the configuration of the data sources supported by GART: PostgreSQL and MySQL. At the same time, GART also supports arbitrary data streams, which only require the user to manually convert the data updates to the specified format. 17 | 18 | .. panels:: 19 | :header: text-center 20 | :column: col-lg-12 p-2 21 | 22 | .. link-button:: data-source-config/postgresql 23 | :type: ref 24 | :text: PostgreSQL 25 | :classes: btn-block stretched-link 26 | ^^^^^^^^^^^^ 27 | PostgreSQL configuration guide. 28 | 29 | --- 30 | 31 | .. link-button:: data-source-config/mysql 32 | :type: ref 33 | :text: MySQL 34 | :classes: btn-block stretched-link 35 | ^^^^^^^^^^^^ 36 | MySQL configuration guide. 37 | 38 | --- 39 | 40 | .. link-button:: data-source-config/storage 41 | :type: ref 42 | :text: GART Storage 43 | :classes: btn-block stretched-link 44 | ^^^^^^^^^^^^ 45 | Guidelines for the use of GART's dynamic graph storage. 46 | -------------------------------------------------------------------------------- /docs/documentation/tutorials/data-source-config/mysql.rst: -------------------------------------------------------------------------------- 1 | MySQL 2 | ================ 3 | 4 | Modify Configuration Files 5 | -------------------------- 6 | 7 | MySQL configuration file ``/etc/mysql/my.cnf``: 8 | 9 | .. code:: ini 10 | :linenos: 11 | 12 | [mysqld] 13 | # Prefix of the binlogs 14 | log-bin=mysql-bin 15 | 16 | # Binlog Format: row-based logging, only Maxwell needs binlog_format=row 17 | binlog_format=row 18 | binlog_row_image=full 19 | 20 | # Enable GTID for consistency 21 | gtid_mode=ON 22 | enforce_gtid_consistency=ON 23 | 24 | # The databases captured. GART will capture all databases if not specified. 25 | binlog-do-db=ldbc # change the name to your database 26 | binlog-do-db=... # change the name to your database 27 | 28 | Create MySQL User 29 | ------------------ 30 | 31 | Create a MySQL user for the log capture (`Debezium`_): 32 | 33 | .. code:: mysql 34 | :linenos: 35 | 36 | # Create a user call "dbuser" with password "123456" 37 | # The hostname part of the account name, if omitted, defaults to '%'. 38 | CREATE USER 'dbuser'@'localhost' IDENTIFIED BY '123456'; 39 | 40 | # Grant necesarry privileges 41 | # PrivilegesRELOAD and SHOW DATABASES are only used for Debezium 42 | GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'dbuser'@'localhost'; 43 | 44 | # Grant privileges on the database "dbuser", only used for Maxwell 45 | GRANT ALL ON maxwell.* TO 'dbuser'@'localhost'; 46 | 47 | .. _Debezium: https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-creating-user 48 | -------------------------------------------------------------------------------- /docs/documentation/tutorials/data-source-config/postgresql.rst: -------------------------------------------------------------------------------- 1 | PostgreSQL 2 | ================ 3 | 4 | 5 | To implement the conversion of relational data to graph data using GART, the data source needs to be configured with the necessary permissions. Here is an example of PostgreSQL. 6 | 7 | The PostgreSQL configuration file is in the directory ``/etc/postgresql/$PSQL_VERSION/main/``. In our docker image, the PostgreSQL version is 16, so the configuration file is in the directory ``/etc/postgresql/16/main/``. 8 | 9 | 1. Modify the configuration file ``postgresql.conf`` to enable WAL as follows: 10 | 11 | .. code:: ini 12 | 13 | wal_level = logical 14 | max_replication_slots = 10 # larger than 0, default is 10 15 | max_wal_senders = 10 # larger than 0, default is 10 16 | 17 | 2. Create a PostgreSQL user (``dbuser``) for the log capture Debezium: 18 | 19 | .. code:: postgresql 20 | :linenos: 21 | 22 | CREATE USER dbuser WITH PASSWORD '123456'; 23 | ALTER USER dbuser REPLICATION; 24 | ALTER USER dbuser LOGIN; 25 | GRANT pg_read_server_files TO dbuser; -- For loading CSV files 26 | 27 | CREATE DATABASE ldbc; 28 | GRANT ALL ON DATABASE ldbc TO dbuser; 29 | 30 | 3. Switch to the database ``ldbc`` and create the schema ``public``: 31 | 32 | .. code:: 33 | 34 | \c ldbc 35 | 36 | .. code:: postgresql 37 | 38 | GRANT ALL ON SCHEMA public TO dbuser; 39 | 40 | 4. Modify the configuration file ``/etc/postgresql/$PSQL_VERSION/main/pg_hba.conf`` to `trust the user`_ ``dbuser``: 41 | 42 | .. code:: none 43 | 44 | local replication dbuser trust 45 | host replication dbuser 127.0.0.1/32 trust 46 | host replication dbuser ::1/128 trust 47 | 48 | 5. Restart PostgreSQL: 49 | 50 | .. code:: bash 51 | 52 | gart-env$ sudo /etc/init.d/postgresql restart 53 | 54 | .. _trust the user: https://debezium.io/documentation/reference/stable/postgres-plugins.html#:~:text=pg_hba.conf%20%2C%20configuration%20file%20parameters%20settings 55 | -------------------------------------------------------------------------------- /docs/documentation/tutorials/run-gap.rst: -------------------------------------------------------------------------------- 1 | Run Graph Analysis 2 | ========================= 3 | 4 | GART supports different approaches for graph analysis. The following sections describe how to run the different analysis methods. 5 | 6 | Graph Analysis with GAE 7 | ------------------------ 8 | 9 | `GAE `_ is a graph analysis framework that provides a set of graph algorithms for analyzing large-scale graphs. GAE is implemented in C++ and provides a Python interface for running graph algorithms on graphs stored in the GART format. 10 | 11 | After running the GART server, you can run graph analysis using the GAE library by ``mpirun`` command. 12 | 13 | .. code:: bash 14 | 15 | gart-env$ cd /workspace/gart/ 16 | gart-env$ mpirun -n 1 ./apps/run_gart_app --read_epoch 0 --app_name sssp --sssp_source_label organisation --sssp_source_oid 0 --sssp_weight_name wa_work_from 17 | 18 | Other Alternatives 19 | ------------------ 20 | 21 | Please refer to the `PostgreSQL Extension `_. The PostgreSQL extension provides a set of graph algorithms that can be run on graphs stored in the GART format. 22 | -------------------------------------------------------------------------------- /docs/images/arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GraphScope/GART/68767a870c9b3888015802660a09b5b83e7cd9a8/docs/images/arch.png -------------------------------------------------------------------------------- /docs/images/graph-arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GraphScope/GART/68767a870c9b3888015802660a09b5b83e7cd9a8/docs/images/graph-arch.png -------------------------------------------------------------------------------- /docs/images/ldbc-snb.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GraphScope/GART/68767a870c9b3888015802660a09b5b83e7cd9a8/docs/images/ldbc-snb.jpg -------------------------------------------------------------------------------- /docs/images/rgmapping.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GraphScope/GART/68767a870c9b3888015802660a09b5b83e7cd9a8/docs/images/rgmapping.png -------------------------------------------------------------------------------- /docs/images/use-case-fr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GraphScope/GART/68767a870c9b3888015802660a09b5b83e7cd9a8/docs/images/use-case-fr.png -------------------------------------------------------------------------------- /interfaces/fragment/gart_fragment_traits.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef INTERFACES_FRAGMENT_GART_FRAGMENT_TRAITS_H_ 17 | #define INTERFACES_FRAGMENT_GART_FRAGMENT_TRAITS_H_ 18 | 19 | #include "interfaces/fragment/gart_fragment.h" 20 | 21 | namespace gart { 22 | template 23 | class GartFragment; 24 | } // namespace gart 25 | 26 | namespace vineyard { 27 | template 28 | struct is_property_fragment> { 29 | using type = std::true_type; 30 | static constexpr bool value = true; 31 | }; 32 | } // namespace vineyard 33 | 34 | #endif // INTERFACES_FRAGMENT_GART_FRAGMENT_TRAITS_H_ 35 | -------------------------------------------------------------------------------- /interfaces/fragment/types.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef INTERFACES_FRAGMENT_TYPES_H_ 17 | #define INTERFACES_FRAGMENT_TYPES_H_ 18 | 19 | #include "vineyard/graph/fragment/property_graph_types.h" 20 | 21 | namespace gart { 22 | using vid_t = uint64_t; 23 | using fid_t = grape::fid_t; 24 | 25 | using order_t = uint8_t; 26 | using segid_t = uint64_t; 27 | using timestamp_t = int64_t; 28 | using label_id_t = vineyard::property_graph_types::LABEL_ID_TYPE; 29 | using vertex_t = grape::Vertex; 30 | 31 | enum PropertyType { 32 | INVALID = 0, 33 | BOOL = 1, 34 | CHAR = 2, 35 | SHORT = 3, 36 | INT = 4, 37 | LONG = 5, 38 | FLOAT = 6, 39 | DOUBLE = 7, 40 | STRING = 8, 41 | BYTES = 9, 42 | INT_LIST = 10, 43 | LONG_LIST = 11, 44 | FLOAT_LIST = 12, 45 | DOUBLE_LIST = 13, 46 | STRING_LIST = 14, 47 | DATE = 15, 48 | DATETIME = 16, 49 | TIME = 17, 50 | TIMESTAMP = 18 51 | }; 52 | 53 | #define VERTEX_PER_SEG 4096 54 | } // namespace gart 55 | 56 | #endif // INTERFACES_FRAGMENT_TYPES_H_ 57 | -------------------------------------------------------------------------------- /interfaces/grin/rust/gart_all.h: -------------------------------------------------------------------------------- 1 | #include "../predefine.h" 2 | 3 | #include "../include/include/common/error.h" 4 | #include "../include/include/index/label.h" 5 | #include "../include/include/index/order.h" 6 | #include "../include/include/partition/partition.h" 7 | #include "../include/include/partition/reference.h" 8 | #include "../include/include/partition/topology.h" 9 | #include "../include/include/property/partition.h" 10 | #include "../include/include/property/primarykey.h" 11 | #include "../include/include/property/property.h" 12 | #include "../include/include/property/propertylist.h" 13 | #include "../include/include/property/row.h" 14 | #include "../include/include/property/topology.h" 15 | #include "../include/include/property/type.h" 16 | #include "../include/include/topology/adjacentlist.h" 17 | #include "../include/include/topology/edgelist.h" 18 | #include "../include/include/topology/structure.h" 19 | #include "../include/include/topology/vertexlist.h" 20 | 21 | /// RUST_KEEP pub const GRIN_NULL_DATATYPE: GrinDatatype = GRIN_DATATYPE_UNDEFINED; 22 | /// RUST_KEEP pub const GRIN_NULL_GRAPH: GrinGraph = std::ptr::null_mut(); 23 | /// RUST_KEEP pub const GRIN_NULL_VERTEX: GrinVertex = u64::MAX; 24 | /// RUST_KEEP pub const GRIN_NULL_EDGE: GrinEdge = std::ptr::null_mut(); 25 | /// RUST_KEEP pub const GRIN_NULL_LIST: *mut ::std::os::raw::c_void = std::ptr::null_mut(); 26 | /// RUST_KEEP pub const GRIN_NULL_LIST_ITERATOR: *mut ::std::os::raw::c_void = std::ptr::null_mut(); 27 | /// RUST_KEEP pub const GRIN_NULL_PARTITION: GrinPartition = u32::MAX; 28 | /// RUST_KEEP pub const GRIN_NULL_VERTEX_REF: GrinVertexRef = -1; 29 | /// RUST_KEEP pub const GRIN_NULL_VERTEX_TYPE: GrinVertexType = u32::MAX; 30 | /// RUST_KEEP pub const GRIN_NULL_EDGE_TYPE: GrinEdgeType = u32::MAX; 31 | /// RUST_KEEP pub const GRIN_NULL_VERTEX_PROPERTY: GrinVertexProperty = u64::MAX; 32 | /// RUST_KEEP pub const GRIN_NULL_EDGE_PROPERTY: GrinEdgeProperty = u64::MAX; 33 | /// RUST_KEEP pub const GRIN_NULL_ROW: GrinRow = std::ptr::null_mut(); 34 | /// RUST_KEEP pub const GRIN_NULL_NATURAL_ID: u32 = u32::MAX; 35 | /// RUST_KEEP pub const GRIN_NULL_SIZE: u32 = u32::MAX; 36 | int ending; -------------------------------------------------------------------------------- /interfaces/grin/src/common/error.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020 Alibaba Group Holding Limited. 2 | Licensed under the Apache License, Version 2.0 (the "License"); 3 | you may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software 7 | distributed under the License is distributed on an "AS IS" BASIS, 8 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | See the License for the specific language governing permissions and 10 | limitations under the License. 11 | */ 12 | 13 | /** 14 | @file error.h 15 | @brief Define the error code related APIs 16 | */ 17 | 18 | #include "grin/src/predefine.h" 19 | 20 | #include "grin/include/include/common/error.h" 21 | 22 | __thread GRIN_ERROR_CODE grin_error_code = GRIN_ERROR_CODE::NO_ERROR; 23 | 24 | GRIN_ERROR_CODE grin_get_last_error_code() { return grin_error_code; } -------------------------------------------------------------------------------- /interfaces/grin/src/common/message.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020 Alibaba Group Holding Limited. 2 | Licensed under the Apache License, Version 2.0 (the "License"); 3 | you may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software 7 | distributed under the License is distributed on an "AS IS" BASIS, 8 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | See the License for the specific language governing permissions and 10 | limitations under the License. 11 | */ 12 | 13 | #include "grin/include/include/common/message.h" 14 | #include "grin/src/predefine.h" 15 | 16 | #ifdef GRIN_ENABLE_SCHEMA 17 | void grin_destroy_graph_schema_msg(const char* msg); 18 | 19 | const char* grin_get_graph_schema_msg(const char* uri); 20 | #endif -------------------------------------------------------------------------------- /interfaces/grin/src/property/topology.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020 Alibaba Group Holding Limited. 2 | Licensed under the Apache License, Version 2.0 (the "License"); 3 | you may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software 7 | distributed under the License is distributed on an "AS IS" BASIS, 8 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | See the License for the specific language governing permissions and 10 | limitations under the License. 11 | */ 12 | 13 | #include "grin/src/predefine.h" 14 | 15 | #include "grin/include/include/property/topology.h" 16 | 17 | #ifdef GRIN_ENABLE_SCHEMA 18 | size_t grin_get_vertex_num_by_type(GRIN_GRAPH g, GRIN_VERTEX_TYPE vt) { 19 | auto _g = static_cast(g)->frag; 20 | return _g->GetVerticesNum(vt); 21 | } 22 | 23 | size_t grin_get_edge_num_by_type(GRIN_GRAPH g, GRIN_EDGE_TYPE et) { 24 | auto _g = static_cast(g)->frag; 25 | return _g->GetEdgeNum(et); 26 | } 27 | #endif 28 | 29 | #if defined(GRIN_ENABLE_VERTEX_LIST) && defined(GRIN_ENABLE_SCHEMA) 30 | GRIN_VERTEX_LIST grin_get_vertex_list_by_type(GRIN_GRAPH g, 31 | GRIN_VERTEX_TYPE vt) { 32 | auto vl = new GRIN_VERTEX_LIST_T(); 33 | vl->vtype = vt; 34 | vl->all_master_mirror = 0; 35 | return vl; 36 | } 37 | #endif 38 | 39 | #if defined(GRIN_ENABLE_EDGE_LIST) && defined(GRIN_ENABLE_SCHEMA) 40 | GRIN_EDGE_LIST grin_get_edge_list_by_type(GRIN_GRAPH, GRIN_EDGE_TYPE); 41 | #endif 42 | 43 | #if defined(GRIN_ENABLE_ADJACENT_LIST) && defined(GRIN_ENABLE_SCHEMA) 44 | GRIN_ADJACENT_LIST grin_get_adjacent_list_by_edge_type(GRIN_GRAPH g, 45 | GRIN_DIRECTION dir, 46 | GRIN_VERTEX v, 47 | GRIN_EDGE_TYPE et) { 48 | GRIN_ADJACENT_LIST adj_list; 49 | adj_list.v = v; 50 | adj_list.dir = dir; 51 | adj_list.etype = et; 52 | return adj_list; 53 | } 54 | #endif -------------------------------------------------------------------------------- /k8s/converter-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: converter-deployment 5 | namespace: gart 6 | spec: 7 | replicas: 1 8 | selector: 9 | matchLabels: 10 | app: converter 11 | template: 12 | metadata: 13 | labels: 14 | app: converter 15 | spec: 16 | containers: 17 | - name: converter 18 | image: gart-converter:latest 19 | imagePullPolicy: IfNotPresent 20 | command: ["/bin/bash", "-c"] 21 | args: 22 | - | 23 | cd /workspace/gart/build && 24 | ./gart \ 25 | --db-host ${DB_HOST} \ 26 | --db-port ${DB_PORT} \ 27 | --db-name ${DB_NAME} \ 28 | --db-type ${DB_TYPE} \ 29 | --db-user ${DB_USER} \ 30 | --db-password ${DB_PASSWORD} \ 31 | --v6d-sock ${V6D_SOCKET} \ 32 | --v6d-size ${V6D_SIZE} \ 33 | --etcd-endpoint etcd-service:2379 \ 34 | --etcd-prefix ${ETCD_PREFIX} \ 35 | --kafka-server kafka-service:9092 \ 36 | --subgraph-num ${SUBGRAPH_NUM} \ 37 | --enable-bulkload ${ENABLE_BULKLOAD} \ 38 | --rg-from-etcd true \ 39 | --k8s-mode true \ 40 | --role converter && 41 | sleep infinity 42 | envFrom: 43 | - configMapRef: 44 | name: gart-config 45 | 46 | 47 | -------------------------------------------------------------------------------- /k8s/debezium-config-template.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: debezium-config 5 | namespace: gart 6 | data: 7 | mysql-connector.json: |- 8 | { 9 | "name": "debezium-connector-mysql", 10 | "config": { 11 | "connector.class": "io.debezium.connector.mysql.MySqlConnector", 12 | "database.hostname": "127.0.0.1", 13 | "database.port": "3306", 14 | "database.user": "dbuser", 15 | "database.password": "123456", 16 | "database.include.list": "ldbc", 17 | "table.include.list": "ldbc.organisation,ldbc.place,ldbc.tag,ldbc.tagclass,ldbc.person,ldbc.comment,ldbc.post,ldbc.forum,ldbc.org_islocationin,ldbc.ispartof,ldbc.issubclassof,ldbc.hastype,ldbc.comment_hascreator,ldbc.comment_hastag,ldbc.comment_islocationin,ldbc.replyof_comment,ldbc.replyof_post,ldbc.post_hascreator,ldbc.post_hastag,ldbc.post_islocationin,ldbc.forum_containerof,ldbc.forum_hasmoderator,ldbc.forum_hastag,ldbc.person_hasinterest,ldbc.person_islocationin,ldbc.forum_hasmember,ldbc.knows,ldbc.likes_comment,ldbc.likes_post,ldbc.studyat,ldbc.workat", 18 | "snapshot.mode": "initial", 19 | "database.server.id": "1", 20 | "include.schema.changes": "false", 21 | "tombstones.on.delete": "false", 22 | "topic.prefix": "fullfillment", 23 | "schema.history.internal.kafka.topic": "schemahistory.fullfillment", 24 | "transforms": "Combine,ReplaceField", 25 | "transforms.Combine.type": "io.debezium.transforms.ByLogicalTableRouter", 26 | "transforms.Combine.topic.regex": "(.*)", 27 | "transforms.Combine.topic.replacement": "binlog", 28 | "transforms.ReplaceField.type": "org.apache.kafka.connect.transforms.ReplaceField$Value", 29 | "transforms.ReplaceField.exclude": "ts_ms,transaction", 30 | "key.converter": "org.apache.kafka.connect.json.JsonConverter", 31 | "value.converter": "org.apache.kafka.connect.json.JsonConverter", 32 | "key.converter.schemas.enable": "false", 33 | "value.converter.schemas.enable": "false", 34 | "database.history.kafka.bootstrap.servers": "kafka-service:9092", 35 | "schema.history.internal.kafka.bootstrap.servers": "kafka-service:9092" 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /k8s/debezium-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: debezium-connect 5 | namespace: gart 6 | spec: 7 | replicas: 1 8 | selector: 9 | matchLabels: 10 | app: debezium-connect 11 | template: 12 | metadata: 13 | labels: 14 | app: debezium-connect 15 | spec: 16 | containers: 17 | - name: debezium-connect 18 | image: debezium/connect:latest 19 | imagePullPolicy: IfNotPresent 20 | env: 21 | - name: BOOTSTRAP_SERVERS 22 | value: "kafka-service:9092" 23 | - name: GROUP_ID 24 | value: "1" 25 | - name: CONFIG_STORAGE_TOPIC 26 | value: "debezium-connect-configs" 27 | - name: OFFSET_STORAGE_TOPIC 28 | value: "debezium-connect-offsets" 29 | - name: STATUS_STORAGE_TOPIC 30 | value: "debezium-connect-status" 31 | ports: 32 | - containerPort: 8083 33 | 34 | --- 35 | apiVersion: v1 36 | kind: Service 37 | metadata: 38 | name: debezium-connect-service 39 | namespace: gart 40 | labels: 41 | app: debezium-connect 42 | spec: 43 | type: ClusterIP 44 | ports: 45 | - port: 8083 46 | targetPort: 8083 47 | selector: 48 | app: debezium-connect 49 | -------------------------------------------------------------------------------- /k8s/debezium-job.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: Job 3 | metadata: 4 | name: debezium-connector-launch 5 | namespace: gart 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: debezium-connector-launch 11 | image: curlimages/curl:latest 12 | imagePullPolicy: IfNotPresent 13 | command: ["/bin/sh"] 14 | args: 15 | - "-c" 16 | - > 17 | echo "Applying Debezium connector configuration..."; 18 | CONFIG=$(cat /etc/debezium-connector-config/mysql-connector.json); 19 | curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" 20 | --data "$CONFIG" http://debezium-connect-service:8083/connectors/; 21 | echo "Configuration applied."; 22 | volumeMounts: 23 | - name: config-volume 24 | mountPath: "/etc/debezium-connector-config" 25 | restartPolicy: OnFailure 26 | volumes: 27 | - name: config-volume 28 | configMap: 29 | name: debezium-config 30 | -------------------------------------------------------------------------------- /k8s/etcd-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | labels: 5 | app: etcd 6 | name: etcd-server 7 | namespace: gart 8 | spec: 9 | containers: 10 | - command: 11 | - /usr/local/bin/etcd 12 | - --name 13 | - etcd-server 14 | - --initial-advertise-peer-urls 15 | - http://127.0.0.1:2380 16 | - --listen-peer-urls 17 | - http://0.0.0.0:2380 18 | - --listen-client-urls 19 | - http://0.0.0.0:2379 20 | - --advertise-client-urls 21 | - http://127.0.0.1:2379 22 | - --initial-cluster 23 | - etcd-server=http://127.0.0.1:2380 24 | - --initial-cluster-state 25 | - new 26 | image: quay.io/coreos/etcd:latest 27 | name: etcd-server 28 | ports: 29 | - containerPort: 2379 30 | name: client 31 | protocol: TCP 32 | - containerPort: 2380 33 | name: server 34 | protocol: TCP 35 | restartPolicy: Always 36 | 37 | --- 38 | apiVersion: v1 39 | kind: Service 40 | metadata: 41 | name: etcd-service 42 | namespace: gart 43 | spec: 44 | type: ClusterIP 45 | ports: 46 | - name: etcd-service-port 47 | port: 2379 48 | #nodePort: 22379 49 | #protocol: TCP 50 | targetPort: 2379 51 | selector: 52 | app: etcd -------------------------------------------------------------------------------- /k8s/gart-config-template.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: gart-config 5 | namespace: gart 6 | data: 7 | DB_HOST: "127.0.0.1" 8 | DB_PORT: "3306" 9 | DB_USER: "dbuser" 10 | DB_PASSWORD: "123456" 11 | DB_TYPE: "mysql" 12 | DB_NAME: "ldbc" 13 | V6D_SOCKET: "/tmp/v6d.sock" 14 | V6D_SIZE: "750G" 15 | ETCD_PREFIX: "gart_meta_" 16 | ENABLE_BULKLOAD: "1" 17 | SUBGRAPH_NUM: "2" 18 | -------------------------------------------------------------------------------- /k8s/gart-namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: gart -------------------------------------------------------------------------------- /k8s/kafka-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: kafka-broker1 5 | namespace: gart 6 | spec: 7 | selector: 8 | matchLabels: 9 | app: kafka 10 | template: 11 | metadata: 12 | labels: 13 | app: kafka 14 | id: "1" 15 | spec: 16 | containers: 17 | - name: kafka 18 | image: bitnami/kafka:latest 19 | imagePullPolicy: IfNotPresent 20 | ports: 21 | - containerPort: 9092 22 | env: 23 | - name: KAFKA_CFG_ADVERTISED_LISTENERS 24 | value: "PLAINTEXT://kafka-service:9092" 25 | - name: KAFKA_CFG_NODE_ID 26 | value: "1" 27 | - name: KAFKA_CFG_ZOOKEEPER_CONNECT 28 | value: "zoo1:2181" 29 | - name: ALLOW_PLAINTEXT_LISTENER 30 | value: "yes" 31 | - name: KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE 32 | value: "true" 33 | 34 | --- 35 | apiVersion: v1 36 | kind: Service 37 | metadata: 38 | name: kafka-service 39 | namespace: gart 40 | labels: 41 | name: kafka 42 | spec: 43 | ports: 44 | - port: 9092 45 | name: kafka-port 46 | protocol: TCP 47 | targetPort: 9092 48 | selector: 49 | app: kafka 50 | type: ClusterIP -------------------------------------------------------------------------------- /k8s/metallb-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: metallb.io/v1beta1 2 | kind: IPAddressPool 3 | metadata: 4 | name: first-pool 5 | namespace: metallb-system 6 | spec: 7 | addresses: 8 | - 172.18.255.200-172.18.255.250 9 | --- 10 | apiVersion: metallb.io/v1beta1 11 | kind: L2Advertisement 12 | metadata: 13 | name: example 14 | namespace: metallb-system 15 | spec: 16 | ipAddressPools: 17 | - first-pool 18 | -------------------------------------------------------------------------------- /k8s/writer-deployment-template.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: writer-statefulset 5 | namespace: gart 6 | spec: 7 | serviceName: "writer" 8 | replicas: 2 9 | selector: 10 | matchLabels: 11 | app: writer 12 | template: 13 | metadata: 14 | labels: 15 | app: writer 16 | spec: 17 | containers: 18 | - name: writer 19 | image: gart-writer:latest 20 | imagePullPolicy: IfNotPresent 21 | command: ["/bin/bash", "-c"] 22 | args: 23 | - | 24 | cd /workspace/gart/build && 25 | ./gart \ 26 | --db-host ${DB_HOST} \ 27 | --db-port ${DB_PORT} \ 28 | --db-name ${DB_NAME} \ 29 | --db-type ${DB_TYPE} \ 30 | --db-user ${DB_USER} \ 31 | --db-password ${DB_PASSWORD} \ 32 | --v6d-sock ${V6D_SOCKET} \ 33 | --v6d-size ${V6D_SIZE} \ 34 | --etcd-endpoint etcd-service:2379 \ 35 | --etcd-prefix ${ETCD_PREFIX} \ 36 | --kafka-server kafka-service:9092 \ 37 | --subgraph-num ${SUBGRAPH_NUM} \ 38 | --subgraph-id $((${HOSTNAME##*-} + 0)) \ 39 | --enable-bulkload ${ENABLE_BULKLOAD} \ 40 | --rg-from-etcd true \ 41 | --k8s-mode true \ 42 | --role writer && 43 | sleep infinity 44 | envFrom: 45 | - configMapRef: 46 | name: gart-config 47 | 48 | 49 | -------------------------------------------------------------------------------- /k8s/zk-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: zookeeper-deployment-1 5 | namespace: gart 6 | spec: 7 | selector: 8 | matchLabels: 9 | app: zookeeper-1 10 | template: 11 | metadata: 12 | labels: 13 | app: zookeeper-1 14 | spec: 15 | containers: 16 | - name: zoo1 17 | image: bitnami/zookeeper:latest 18 | imagePullPolicy: IfNotPresent 19 | ports: 20 | - containerPort: 2181 21 | env: 22 | - name: ZOOKEEPER_ID 23 | value: "1" 24 | - name: ZOOKEEPER_SERVER_1 25 | value: zoo1 26 | - name: ALLOW_ANONYMOUS_LOGIN 27 | value: "yes" 28 | 29 | --- 30 | apiVersion: v1 31 | kind: Service 32 | metadata: 33 | name: zoo1 34 | namespace: gart 35 | labels: 36 | app: zookeeper-1 37 | spec: 38 | type: ClusterIP 39 | ports: 40 | - name: zk-port 41 | port: 2181 42 | targetPort: 2181 43 | selector: 44 | app: zookeeper-1 -------------------------------------------------------------------------------- /pgql/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.2...3.5) 2 | 3 | set(GART_PGQL_MAJOR_VERSION 0) 4 | set(GART_PGQL_MINOR_VERSION 1) 5 | set(GART_PGQL_VERSION ${GART_PGQL_MAJOR_VERSION}.${GART_PGQL_MINOR_VERSION}) 6 | 7 | project(gart_pgql VERSION ${GART_PGQL_VERSION}) 8 | 9 | ### Forbid in-source builds ### 10 | if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}") 11 | message(FATAL_ERROR "In-source builds are not allowed.") 12 | endif("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}") 13 | 14 | file(GLOB_RECURSE JAVA_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.java") 15 | file(GLOB_RECURSE POM_FILE "pom.xml") 16 | 17 | add_custom_target(pgql 18 | COMMAND mvn -f ${POM_FILE} clean package 19 | DEPENDS ${JAVA_FILES} ${POM_FILE} 20 | WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} 21 | VERBATIM 22 | ) 23 | -------------------------------------------------------------------------------- /pgql/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.alibaba.graphscope 7 | gart-rg 8 | 1.0.0 9 | 10 | 11 | 12 | 13 | oracle.pg 14 | pgql-lang 15 | 0.0.0-SNAPSHOT 16 | compile 17 | 18 | 19 | 20 | 21 | org.yaml 22 | snakeyaml 23 | 2.1 24 | 25 | 26 | 27 | 28 | 29 | 1.8 30 | 1.8 31 | 32 | 33 | -------------------------------------------------------------------------------- /pgql/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$#" -lt 2 ]; then 4 | echo "Usage: $0 " 5 | exit 1 6 | fi 7 | 8 | type=$1 # sql2yaml or yaml2sql 9 | input=$2 10 | output=$3 11 | 12 | case "$type" in 13 | "sql2yaml"|"yaml2sql"|"sqlStr2yaml"|"yaml-check"|"yaml-format") 14 | # If $type is one of the expected values, do nothing 15 | ;; 16 | *) 17 | # If $type is not a recognized type, output an error message and exit 18 | echo "Invalid type: $type" 19 | exit 1 20 | ;; 21 | esac 22 | 23 | export MAVEN_OPTS="-Xms512m -Xmx1024m -Xss16m $MAVEN_OPTS" 24 | 25 | mvn clean package exec:java -Dexec.mainClass="gart.pgql.Main" -Dexec.cleanupDaemonThreads=false -Dexec.args="$type \"$input\" $output" 26 | -------------------------------------------------------------------------------- /pgql/src/main/java/gart/pgql/CustomRepresenter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020-2023 Alibaba Group Holding Limited. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package gart.pgql; 18 | 19 | import java.util.LinkedHashSet; 20 | import java.util.Set; 21 | import java.util.stream.Collectors; 22 | 23 | import org.yaml.snakeyaml.DumperOptions; 24 | import org.yaml.snakeyaml.introspector.*; 25 | import org.yaml.snakeyaml.representer.Representer; 26 | 27 | // https://stackoverflow.com/questions/62956776/how-to-order-nodes-in-yaml-with-snakeyaml 28 | public class CustomRepresenter extends Representer { 29 | 30 | public CustomRepresenter(DumperOptions options) { 31 | super(options); 32 | PropertyUtils propUtil = new PropertyUtils() { 33 | @Override 34 | protected Set createPropertySet(Class type, BeanAccess bAccess) { 35 | return getPropertiesMap(type, bAccess).values().stream().sequential() 36 | .filter(prop -> prop.isReadable() && (isAllowReadOnlyProperties() || prop.isWritable())) 37 | .collect(Collectors.toCollection(LinkedHashSet::new)); 38 | } 39 | }; 40 | setPropertyUtils(propUtil); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /pgql/src/main/java/gart/pgql/CustomYaml.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020-2023 Alibaba Group Holding Limited. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package gart.pgql; 18 | 19 | import org.yaml.snakeyaml.DumperOptions; 20 | import org.yaml.snakeyaml.LoaderOptions; 21 | import org.yaml.snakeyaml.Yaml; 22 | import org.yaml.snakeyaml.constructor.Constructor; 23 | import org.yaml.snakeyaml.inspector.TagInspector; 24 | import org.yaml.snakeyaml.representer.Representer; 25 | 26 | public class CustomYaml { 27 | public static Yaml getReadYaml() { 28 | // ignore extra properties in a YAML file 29 | Representer representer = new CustomRepresenter(new DumperOptions()); 30 | representer.getPropertyUtils().setSkipMissingProperties(true); 31 | TagInspector tagInspector = tag -> tag.getClassName().equals(GSchema.class.getName()); 32 | LoaderOptions loaderOptions = new LoaderOptions(); 33 | loaderOptions.setTagInspector(tagInspector); 34 | Yaml yaml = new Yaml(new Constructor(GSchema.class, loaderOptions), representer); 35 | 36 | return yaml; 37 | } 38 | 39 | public static Yaml getWriteYaml() { 40 | // initialize yaml 41 | DumperOptions options = new DumperOptions(); 42 | options.setIndent(2); 43 | options.setPrettyFlow(true); 44 | options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); 45 | Representer representer = new CustomRepresenter(options); 46 | Yaml yaml = new Yaml(representer, options); 47 | 48 | return yaml; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /pgql/src/main/java/gart/pgql/PgqlConverter.java: -------------------------------------------------------------------------------- 1 | package gart.pgql; 2 | 3 | import java.io.FileReader; 4 | import java.util.List; 5 | 6 | import org.yaml.snakeyaml.Yaml; 7 | 8 | import oracle.pgql.lang.ddl.propertygraph.CreatePropertyGraph; 9 | import oracle.pgql.lang.ddl.propertygraph.EdgeTable; 10 | import oracle.pgql.lang.ddl.propertygraph.VertexTable; 11 | import oracle.pgql.lang.ir.SchemaQualifiedName; 12 | 13 | // Converter PGQL YAML to DDL 14 | public class PgqlConverter { 15 | 16 | public PgqlConverter(FileReader reader) { 17 | this.reader = reader; 18 | } 19 | 20 | public CreatePropertyGraph convert() { 21 | // test input 22 | GSchema gSchema = null; 23 | try { 24 | Yaml yaml = CustomYaml.getReadYaml(); 25 | gSchema = yaml.load(reader); 26 | gSchema.format(); 27 | } catch (Exception e) { 28 | System.out.println(e.getMessage()); 29 | } 30 | 31 | SchemaQualifiedName graphName = new SchemaQualifiedName("", gSchema.graph); 32 | List vertexTables = gSchema.getVertexTables(); 33 | List edgeTables = gSchema.getEdgeTables(vertexTables); 34 | 35 | CreatePropertyGraph ddl = new CreatePropertyGraph(graphName, vertexTables, edgeTables); 36 | return ddl; 37 | } 38 | 39 | private FileReader reader; 40 | } 41 | -------------------------------------------------------------------------------- /pgql/src/main/java/gart/pgql/YamlConverter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020-2023 Alibaba Group Holding Limited. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package gart.pgql; 18 | 19 | import java.util.List; 20 | import java.io.FileWriter; 21 | 22 | import oracle.pgql.lang.ddl.propertygraph.CreatePropertyGraph; 23 | import oracle.pgql.lang.ddl.propertygraph.EdgeTable; 24 | import oracle.pgql.lang.ddl.propertygraph.VertexTable; 25 | 26 | // Converter PGQL DDL to YAML 27 | public class YamlConverter { 28 | public YamlConverter(FileWriter writer, CreatePropertyGraph ddlStatement) { 29 | this.writer = writer; 30 | this.ddlStatement = ddlStatement; 31 | } 32 | 33 | public void convert() { 34 | GSchema gSchema = new GSchema(); 35 | gSchema.graph = ddlStatement.getGraphName().toString(); 36 | 37 | gSchema.loadingConfig = new LoadingConfig(); 38 | gSchema.loadingConfig.dataSource = "rdbms"; 39 | gSchema.loadingConfig.database = "ldbc"; 40 | 41 | List vertexTables = ddlStatement.getVertexTables(); 42 | List edgeTables = ddlStatement.getEdgeTables(); 43 | 44 | gSchema.vertexMappings = new VertexMappings(vertexTables); 45 | gSchema.edgeMappings = new EdgeMappings(edgeTables, vertexTables); 46 | 47 | gSchema.format(); 48 | 49 | // output yaml 50 | CustomYaml.getWriteYaml().dump(gSchema, writer); 51 | // String str = yaml.dump(schema); 52 | // System.out.println(str); 53 | } 54 | 55 | final private CreatePropertyGraph ddlStatement; 56 | final private FileWriter writer; 57 | } 58 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | black>=22.3.0 2 | breathe 3 | docutils==0.20.1 4 | flake8>=4.0.1 5 | flake8-comprehensions 6 | flake8-logging-format 7 | furo # sphinx theme 8 | isort>=5.10.1 9 | jinja2>=3.1.0 10 | libclang 11 | linkify-it-py 12 | myst-parser>=0.13.0 13 | nbsphinx 14 | pygments>=2.4.1 15 | pytest<8.0.0 16 | pytest-benchmark 17 | pytest-cases 18 | pytest-datafiles 19 | pytest-timeout 20 | sphinx>=7.1.2 21 | sphinx-copybutton 22 | sphinx-panels 23 | sphinxcontrib-video 24 | sphinxemoji 25 | sphinxext-opengraph 26 | -------------------------------------------------------------------------------- /scripts/check_process.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Define the list of pids with process names 4 | 5 | # $1 format: declare -A pids=([key2]="value2" [key1]="value1" ) 6 | 7 | v6d_sock=$1 8 | 9 | eval "$2" 10 | 11 | for process_name in ${!pids[@]}; do 12 | process_pid=${pids[$process_name]} 13 | if [[ -z $process_pid || ! $process_pid =~ ^[0-9]+$ ]]; then 14 | echo "Invalid pid $process_pid for process $process_name" 15 | unset pids[$process_name] 16 | continue 17 | fi 18 | done 19 | 20 | echo "Check Process with v6d_sock: $v6d_sock" 21 | 22 | if [[ ${#pids[@]} -eq 0 ]]; then 23 | echo "No valid pids found" 24 | exit -1 25 | fi 26 | 27 | while true; do 28 | # Loop through the list of pids and process names 29 | for process_name in ${!pids[@]}; do 30 | process_pid=${pids[$process_name]} 31 | 32 | # echo "Checking $process_name with pid $process_pid" 33 | 34 | # Check if the process is running 35 | if ! ps -p "$process_pid" > /dev/null; then 36 | echo "$process_name is not running" 37 | ./stop-gart --kill-v6d-sock $v6d_sock --is-abnormal 38 | exit 1 39 | fi 40 | done 41 | 42 | # Wait for 10 seconds and repeat the process 43 | sleep 10 44 | done 45 | -------------------------------------------------------------------------------- /scripts/install-mysql.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sudo DEBIAN_FRONTEND=noninteractive apt install -y mysql-server 4 | pip3 install pymysql cryptography 5 | 6 | sudo tee -a /etc/mysql/my.cnf << EOT 7 | 8 | [mysqld] 9 | # Prefix of the binlogs 10 | log-bin=mysql-bin 11 | 12 | # Binlog Format: row-based logging, maxwell needs binlog_format=row 13 | binlog_format=row 14 | binlog_row_image=full 15 | 16 | # Enable GTID for consistency 17 | gtid_mode=ON 18 | enforce_gtid_consistency=ON 19 | 20 | # The databases captured. GART will capture all databases if not specified. 21 | binlog-do-db=ldbc # change the name to your database 22 | 23 | EOT 24 | 25 | # For some reason, the mysql user's home directory is not set correctly 26 | # see https://stackoverflow.com/questions/62987154/mysql-wont-start-error-su-warning-cannot-change-directory-to-nonexistent 27 | sudo usermod -d /var/lib/mysql/ mysql 28 | 29 | # Restart mysql to apply the changes 30 | sudo service mysql restart 31 | -------------------------------------------------------------------------------- /scripts/install-psql.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Reference: https://www.postgresql.org/download/linux/ubuntu/ 4 | 5 | # Create the file repository configuration: 6 | sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' 7 | 8 | # Import the repository signing key: 9 | wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - 10 | 11 | # Update the package lists: 12 | sudo apt-get update 13 | 14 | export PSQL_VERSION=16 15 | 16 | # Install the specific version of PostgreSQL. 17 | # If you want a latest version, use 'postgresql' instead of 'postgresql-$PSQL_VERSION': 18 | sudo DEBIAN_FRONTEND=noninteractive apt-get -y install postgresql-$PSQL_VERSION 19 | 20 | export PSQL_CONFIG=/etc/postgresql/$PSQL_VERSION/main/postgresql.conf 21 | 22 | sudo tee -a $PSQL_CONFIG << EOT 23 | 24 | wal_level = logical 25 | max_replication_slots = 10 # larger than 0, default is 10 26 | max_wal_senders = 10 # larger than 0, default is 10 27 | 28 | EOT 29 | 30 | # Install the development package for the PostgreSQL Extension: 31 | sudo apt-get install -y postgresql-server-dev-$PSQL_VERSION 32 | 33 | sudo /etc/init.d/postgresql restart 34 | 35 | # pg_ctlcluster 16 main start 36 | # use: sudo -u postgres -i psql 37 | -------------------------------------------------------------------------------- /scripts/kube_ssh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | POD_NAME=$1 3 | shift 4 | kubectl exec ${POD_NAME} -c analyzer -- /bin/sh -c "cat /tmp/hosts_of_nodes | sudo tee -a /etc/hosts && $*" -------------------------------------------------------------------------------- /scripts/launch_gie_executor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | etcd_endpoint=$1 4 | etcd_prefix=$2 5 | read_epoch=$3 6 | 7 | export READ_EPOCH=$read_epoch 8 | export ETCD_SERVER=$etcd_endpoint 9 | export ETCD_PREFIX=$etcd_prefix 10 | 11 | APP_NAME="grin_executor" 12 | 13 | # Start the GIE executor 14 | cd /home/graphscope/GraphScope/interactive_engine/executor/assembly/grin_gart 15 | # run backgorund and redirect stdout and stderr to /home/graphscope/gie_executor.log 16 | ./target/release/gart_executor ../../../assembly/src/conf/graphscope/log4rs.yml /home/graphscope/gie-executor-config.properties > /home/graphscope/gie_executor.log 2>&1 & -------------------------------------------------------------------------------- /scripts/pause_resume_data_processing.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | op=$1 4 | 5 | check_env_vars() { 6 | if [[ -z $DEBEZIUM_SERVICE ]] || [[ -z $DEBEZIUM_CONNECTOR_NAME ]]; then 7 | echo "Debezium service URL or connector name not set." 8 | exit 1 9 | fi 10 | } 11 | 12 | if [[ $op == "pause" ]]; then 13 | echo "Pausing data processing" 14 | check_env_vars 15 | response=$(curl -s -o /dev/null -w "%{http_code}" -X PUT "http://${DEBEZIUM_SERVICE}/connectors/${DEBEZIUM_CONNECTOR_NAME}/pause") 16 | if [[ $response != 202 ]]; then 17 | echo "Failed to pause connector" 18 | exit 1 19 | else 20 | echo "Connector paused successfully" 21 | exit 0 22 | fi 23 | elif [[ $op == "resume" ]]; then 24 | echo "Resuming data processing" 25 | check_env_vars 26 | response=$(curl -s -o /dev/null -w "%{http_code}" -X PUT "http://${DEBEZIUM_SERVICE}/connectors/${DEBEZIUM_CONNECTOR_NAME}/resume") 27 | if [[ $response != 202 ]]; then 28 | echo "Failed to resume connector" 29 | exit 1 30 | else 31 | echo "Connector resumed successfully" 32 | exit 0 33 | fi 34 | else 35 | echo "Invalid operation $op" 36 | exit 1 37 | fi 38 | -------------------------------------------------------------------------------- /vegito/cmake/FindGFlags.cmake: -------------------------------------------------------------------------------- 1 | # This file is used to find gflags library in CMake script, based on code 2 | # from 3 | # 4 | # https://github.com/BVLC/caffe/blob/master/cmake/Modules/FindGFlags.cmake 5 | # 6 | # which is licensed under the 3-Clause BSD License. 7 | # 8 | # - Try to find GFLAGS 9 | # 10 | # The following variables are optionally searched for defaults 11 | # GFLAGS_ROOT_DIR: Base directory where all GFLAGS components are found 12 | # 13 | # The following are set after configuration is done: 14 | # GFLAGS_FOUND 15 | # GFLAGS_INCLUDE_DIRS 16 | # GFLAGS_LIBRARIES 17 | # GFLAGS_LIBRARYRARY_DIRS 18 | 19 | include(FindPackageHandleStandardArgs) 20 | 21 | set(GFLAGS_ROOT_DIR "" CACHE PATH "Folder contains Gflags") 22 | 23 | # We are testing only a couple of files in the include directories 24 | if(WIN32) 25 | find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h 26 | PATHS ${GFLAGS_ROOT_DIR}/src/windows) 27 | else() 28 | find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h 29 | PATHS ${GFLAGS_ROOT_DIR}) 30 | endif() 31 | 32 | if(MSVC) 33 | find_library(GFLAGS_LIBRARY_RELEASE 34 | NAMES libgflags 35 | PATHS ${GFLAGS_ROOT_DIR} 36 | PATH_SUFFIXES Release) 37 | 38 | find_library(GFLAGS_LIBRARY_DEBUG 39 | NAMES libgflags-debug 40 | PATHS ${GFLAGS_ROOT_DIR} 41 | PATH_SUFFIXES Debug) 42 | 43 | set(GFLAGS_LIBRARY optimized ${GFLAGS_LIBRARY_RELEASE} debug ${GFLAGS_LIBRARY_DEBUG}) 44 | else() 45 | find_library(GFLAGS_LIBRARY gflags) 46 | endif() 47 | 48 | find_package_handle_standard_args(GFlags DEFAULT_MSG GFLAGS_INCLUDE_DIR GFLAGS_LIBRARY) 49 | 50 | 51 | if(GFLAGS_FOUND) 52 | set(GFLAGS_INCLUDE_DIRS ${GFLAGS_INCLUDE_DIR}) 53 | set(GFLAGS_LIBRARIES ${GFLAGS_LIBRARY}) 54 | message(STATUS "Found gflags (include: ${GFLAGS_INCLUDE_DIR}, library: ${GFLAGS_LIBRARY})") 55 | mark_as_advanced(GFLAGS_LIBRARY_DEBUG GFLAGS_LIBRARY_RELEASE 56 | GFLAGS_LIBRARY GFLAGS_INCLUDE_DIR GFLAGS_ROOT_DIR) 57 | endif() 58 | -------------------------------------------------------------------------------- /vegito/cmake/FindGlog.cmake: -------------------------------------------------------------------------------- 1 | # This file is used to find glog library in CMake script, based on code 2 | # from 3 | # 4 | # https://github.com/BVLC/caffe/blob/master/cmake/Modules/FindGlog.cmake 5 | # 6 | # which is licensed under the 2-Clause BSD License. 7 | # 8 | # - Try to find Glog 9 | # 10 | # The following variables are optionally searched for defaults 11 | # GLOG_ROOT_DIR: Base directory where all GLOG components are found 12 | # 13 | # The following are set after configuration is done: 14 | # GLOG_FOUND 15 | # GLOG_INCLUDE_DIRS 16 | # GLOG_LIBRARIES 17 | # GLOG_LIBRARYRARY_DIRS 18 | 19 | include(FindPackageHandleStandardArgs) 20 | 21 | set(GLOG_ROOT_DIR "" CACHE PATH "Folder contains Google glog") 22 | 23 | if(WIN32) 24 | find_path(GLOG_INCLUDE_DIR glog/logging.h 25 | PATHS ${GLOG_ROOT_DIR}/src/windows) 26 | else() 27 | find_path(GLOG_INCLUDE_DIR glog/logging.h 28 | PATHS ${GLOG_ROOT_DIR}) 29 | endif() 30 | 31 | if(MSVC) 32 | find_library(GLOG_LIBRARY_RELEASE libglog_static 33 | PATHS ${GLOG_ROOT_DIR} 34 | PATH_SUFFIXES Release) 35 | 36 | find_library(GLOG_LIBRARY_DEBUG libglog_static 37 | PATHS ${GLOG_ROOT_DIR} 38 | PATH_SUFFIXES Debug) 39 | 40 | set(GLOG_LIBRARY optimized ${GLOG_LIBRARY_RELEASE} debug ${GLOG_LIBRARY_DEBUG}) 41 | else() 42 | find_library(GLOG_LIBRARY glog 43 | PATHS ${GLOG_ROOT_DIR} 44 | PATH_SUFFIXES lib lib64) 45 | endif() 46 | 47 | find_package_handle_standard_args(Glog DEFAULT_MSG GLOG_INCLUDE_DIR GLOG_LIBRARY) 48 | 49 | if(GLOG_FOUND) 50 | set(GLOG_INCLUDE_DIRS ${GLOG_INCLUDE_DIR}) 51 | set(GLOG_LIBRARIES ${GLOG_LIBRARY}) 52 | message(STATUS "Found glog (include: ${GLOG_INCLUDE_DIR}, library: ${GLOG_LIBRARY})") 53 | mark_as_advanced(GLOG_ROOT_DIR GLOG_LIBRARY_RELEASE GLOG_LIBRARY_DEBUG 54 | GLOG_LIBRARY GLOG_INCLUDE_DIR) 55 | endif() 56 | -------------------------------------------------------------------------------- /vegito/cmake/FindRdkafka.cmake: -------------------------------------------------------------------------------- 1 | # This file is used to find librdkafka library in CMake script, modifeid from the 2 | # code from 3 | # 4 | # https://github.com/BVLC/caffe/blob/master/cmake/Modules/FindGlog.cmake 5 | # 6 | # which is licensed under the 2-Clause BSD License. 7 | # 8 | # - Try to find librdkafka 9 | # 10 | # The following variables are optionally searched for defaults 11 | # RDKAFKA_ROOT_DIR: Base directory where all RDKAFKA components are found 12 | # 13 | # The following are set after configuration is done: 14 | # RDKAFKA_FOUND 15 | # RDKAFKA_INCLUDE_DIRS 16 | # RDKAFKA_LIBRARIES 17 | # RDKAFKA_LIBRARY_DIRS 18 | 19 | include(FindPackageHandleStandardArgs) 20 | 21 | set(RDKAFKA_ROOT_DIR "" CACHE PATH "Folder contains librdkafka") 22 | 23 | # We are testing only a couple of files in the include directories 24 | find_path(RDKAFKA_INCLUDE_DIR librdkafka PATHS ${RDKAFKA_ROOT_DIR}/include) 25 | 26 | find_library(RDKAFKA_LIBRARY rdkafka PATHS ${RDKAFKA_ROOT_DIR}/lib) 27 | find_library(RDKAFKA++_LIBRARY rdkafka++ PATHS ${RDKAFKA_ROOT_DIR}/lib) 28 | 29 | find_package_handle_standard_args(RDKAFKA DEFAULT_MSG RDKAFKA_INCLUDE_DIR RDKAFKA_LIBRARY) 30 | 31 | 32 | if(RDKAFKA_FOUND) 33 | set(RDKAFKA_INCLUDE_DIRS ${RDKAFKA_INCLUDE_DIR}) 34 | # The RDKAFKA_LIBRARY comes later, since it is depended by the former. 35 | set(RDKAFKA_LIBRARIES ${RDKAFKA++_LIBRARY} ${RDKAFKA_LIBRARY}) 36 | message(STATUS "Found rdkafka (include: ${RDKAFKA_INCLUDE_DIRS}, library: ${RDKAFKA_LIBRARIES})") 37 | mark_as_advanced(RDKAFKA_LIBRARY_DEBUG RDKAFKA_LIBRARY_RELEASE 38 | RDKAFKA_LIBRARY RDKAFKA_INCLUDE_DIR RDKAFKA_ROOT_DIR) 39 | endif() 40 | -------------------------------------------------------------------------------- /vegito/include/seggraph/seggraph.hpp: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #pragma once 17 | 18 | #include "seggraph/blocks.hpp" 19 | #include "seggraph/edge_iterator.hpp" 20 | #include "seggraph/epoch_graph_reader.hpp" 21 | #include "seggraph/epoch_graph_writer.hpp" 22 | #include "seggraph/segment_graph.hpp" 23 | -------------------------------------------------------------------------------- /vegito/include/seggraph/types.hpp: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #pragma once 17 | 18 | #include 19 | #include 20 | 21 | namespace seggraph { 22 | using label_t = uint16_t; 23 | using vertex_t = uint64_t; 24 | using segid_t = uint64_t; 25 | using order_t = uint8_t; 26 | using timestamp_t = int64_t; 27 | 28 | #define VERTEX_PER_SEG 4096 29 | 30 | } // namespace seggraph 31 | -------------------------------------------------------------------------------- /vegito/include/seggraph/utils.hpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2020 Guanyu Feng, Tsinghua University 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #pragma once 17 | 18 | #include "seggraph/types.hpp" 19 | 20 | namespace seggraph { 21 | inline void compiler_fence() { asm volatile("" ::: "memory"); } 22 | 23 | inline order_t size_to_order(size_t size) { 24 | order_t order = (order_t)((size & (size - 1)) != 0); 25 | while (size > 1) { 26 | order += 1; 27 | size >>= 1; 28 | } 29 | return order; 30 | } 31 | 32 | inline int cmp_timestamp(const timestamp_t* xp, timestamp_t y) { // y > 0 33 | timestamp_t x = *xp; 34 | if (x < 0) 35 | return 1; 36 | if (x < y) 37 | return -1; 38 | if (x == y) 39 | return 0; 40 | return 1; 41 | } 42 | 43 | inline int cmp_timestamp(const timestamp_t* xp, timestamp_t y, 44 | timestamp_t local_txn_id) { // y > 0 45 | timestamp_t x = *xp; 46 | if (-x == local_txn_id) 47 | return 0; 48 | if (x < 0) 49 | return 1; 50 | if (x < y) 51 | return -1; 52 | if (x == y) 53 | return 0; 54 | return 1; 55 | } 56 | 57 | } // namespace seggraph 58 | -------------------------------------------------------------------------------- /vegito/include/util/bitset.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef VEGITO_INCLUDE_UTIL_BITSET_H_ 17 | #define VEGITO_INCLUDE_UTIL_BITSET_H_ 18 | 19 | #include 20 | 21 | #define BYTE_SIZE(n) (((n) + 7ul) >> 3) 22 | 23 | #define BYTE_INDEX(i) ((i) >> 3) 24 | #define BIT_OFFSET(i) ((i) &0x7) 25 | 26 | inline const bool get_bit(uint8_t* data, uint64_t idx) { 27 | return data[BYTE_INDEX(idx)] & (((uint8_t) 1) << BIT_OFFSET(idx)); 28 | } 29 | 30 | inline void set_bit(uint8_t* data, uint64_t idx) { 31 | __sync_fetch_and_or(data + BYTE_INDEX(idx), ((uint8_t) 1) << BIT_OFFSET(idx)); 32 | } 33 | 34 | inline void reset_bit(uint8_t* data, uint64_t idx) { 35 | __sync_fetch_and_and(data + BYTE_INDEX(idx), 36 | ~(((uint8_t) 1) << BIT_OFFSET(idx))); 37 | } 38 | 39 | #endif // VEGITO_INCLUDE_UTIL_BITSET_H_ 40 | -------------------------------------------------------------------------------- /vegito/include/util/spinlock.h: -------------------------------------------------------------------------------- 1 | /* 2 | * The code is a part of our project called VEGITO, which retrofits 3 | * high availability mechanism to tame hybrid transaction/analytical 4 | * processing. 5 | * 6 | * Copyright (c) 2021 Shanghai Jiao Tong University. 7 | * All rights reserved. 8 | * 9 | * Licensed under the Apache License, Version 2.0 (the "License"); 10 | * you may not use this file except in compliance with the License. 11 | * You may obtain a copy of the License at 12 | * 13 | * http://www.apache.org/licenses/LICENSE-2.0 14 | * 15 | * Unless required by applicable law or agreed to in writing, 16 | * software distributed under the License is distributed on an "AS 17 | * IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 18 | * express or implied. See the License for the specific language 19 | * governing permissions and limitations under the License. 20 | * 21 | * For more about this software visit: 22 | * 23 | * http://ipads.se.sjtu.edu.cn/projects/vegito 24 | * 25 | */ 26 | 27 | #ifndef VEGITO_INCLUDE_UTIL_SPINLOCK_H_ 28 | #define VEGITO_INCLUDE_UTIL_SPINLOCK_H_ 29 | 30 | #include "atomic.h" // NOLINT(build/include_subdir) 31 | 32 | /* The counter should be initialized to be 0. */ 33 | class SpinLock { 34 | public: 35 | // 0: free, 1: busy 36 | // occupy an exclusive cache line 37 | volatile uint8_t padding1[32]; 38 | volatile uint16_t lock; 39 | volatile uint8_t padding2[32]; 40 | 41 | public: 42 | SpinLock() { lock = 0; } 43 | 44 | inline void Lock() { 45 | while (1) { 46 | uint16_t* p = const_cast( 47 | reinterpret_cast(&lock)); 48 | if (!xchg16(p, 1)) 49 | return; 50 | 51 | while (lock) 52 | cpu_relax(); 53 | } 54 | } 55 | 56 | inline void Unlock() { 57 | barrier(); 58 | lock = 0; 59 | } 60 | 61 | inline uint16_t Trylock() { 62 | uint16_t* p = const_cast( 63 | reinterpret_cast(&lock)); 64 | return xchg16(p, 1); 65 | } 66 | 67 | inline uint16_t IsLocked() { return lock; } 68 | }; 69 | 70 | #endif // VEGITO_INCLUDE_UTIL_SPINLOCK_H_ 71 | -------------------------------------------------------------------------------- /vegito/src/framework/config.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include "config.h" // NOLINT(build/include_subdir) 17 | #include "system_flags.h" // NOLINT(build/include_subdir) 18 | 19 | namespace gart { 20 | namespace framework { 21 | 22 | Config config; 23 | 24 | void Config::parse_sys_args(int argc, char** argv) { 25 | ipc_socket_ = FLAGS_v6d_ipc_socket; 26 | num_servers_ = FLAGS_subgraph_num; 27 | server_id_ = FLAGS_subgraph_id; 28 | } 29 | 30 | void Config::printConfig() const {} 31 | 32 | } // namespace framework 33 | } // namespace gart 34 | -------------------------------------------------------------------------------- /vegito/src/framework/config.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef VEGITO_SRC_FRAMEWORK_CONFIG_H_ 17 | #define VEGITO_SRC_FRAMEWORK_CONFIG_H_ 18 | 19 | #include 20 | 21 | namespace gart { 22 | namespace framework { 23 | 24 | // config for whole system 25 | class Config { 26 | public: 27 | // 1. Environments 28 | inline const std::string& getExeName() const { return exe_name_; } 29 | 30 | // 2. server config 31 | inline int getNumServers() const { return num_servers_; } // #machines 32 | inline int getServerID() const { return server_id_; } // machine id 33 | 34 | // 3. Property Store 35 | inline int getPropertyType() const { return property_type_; } 36 | 37 | inline std::string getIPCScoket() const { return ipc_socket_; } 38 | 39 | void parse_sys_args(int argc, char** argv); 40 | void printConfig() const; 41 | 42 | private: 43 | std::string exe_name_; 44 | int num_servers_ = 1; 45 | int server_id_ = 0; 46 | 47 | // 0 for kv-store, 1 for row-store, 2 for column-store, 3 for naive col 48 | // FIXME: fix this hard code according PropertyStoreType 49 | int property_type_ = 2; 50 | 51 | std::string ipc_socket_; 52 | }; // class Config 53 | 54 | extern Config config; 55 | 56 | } // namespace framework 57 | } // namespace gart 58 | 59 | #endif // VEGITO_SRC_FRAMEWORK_CONFIG_H_ 60 | -------------------------------------------------------------------------------- /vegito/src/graph/graph_ops.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef VEGITO_SRC_GRAPH_GRAPH_OPS_H_ 17 | #define VEGITO_SRC_GRAPH_GRAPH_OPS_H_ 18 | 19 | #include 20 | 21 | #include "graph/graph_store.h" 22 | #include "graph/type_def.h" 23 | 24 | namespace gart { 25 | namespace graph { 26 | 27 | void process_add_vertex(const property::StringViewList& cmd, 28 | graph::GraphStore* graph_store); 29 | void process_add_edge(const property::StringViewList& cmd, 30 | graph::GraphStore* graph_store); 31 | void process_del_vertex(const property::StringViewList& cmd, 32 | graph::GraphStore* graph_store); 33 | void process_del_edge(const property::StringViewList& cmd, 34 | graph::GraphStore* graph_store); 35 | void process_update_vertex(const property::StringViewList& cmd, 36 | graph::GraphStore* graph_store); 37 | } // namespace graph 38 | } // namespace gart 39 | 40 | #endif // VEGITO_SRC_GRAPH_GRAPH_OPS_H_ 41 | -------------------------------------------------------------------------------- /vegito/src/graph/graph_ops/add_vertex.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include 17 | 18 | #include "graph/graph_ops.h" 19 | #include "graph/type_def.h" 20 | #include "property/property.h" 21 | 22 | using std::string; 23 | 24 | using gart::property::StringViewList; 25 | 26 | namespace { 27 | 28 | template 29 | inline void assign(void* ptr, T val) { 30 | *reinterpret_cast(ptr) = val; 31 | } 32 | 33 | } // namespace 34 | 35 | namespace gart { 36 | namespace graph { 37 | 38 | void process_add_vertex(const StringViewList& cmd, 39 | graph::GraphStore* graph_store) { 40 | int write_epoch = stoi(string(cmd[0])); 41 | uint64_t vid = static_cast(stoll(string(cmd[1]))); 42 | std::string external_id = string(cmd[2]); 43 | assert(!external_id.empty()); 44 | StringViewList props(cmd.begin() + 3, cmd.end()); 45 | graph_store->insert_inner_vertex(write_epoch, vid, external_id, props); 46 | } 47 | 48 | } // namespace graph 49 | } // namespace gart 50 | -------------------------------------------------------------------------------- /vegito/src/graph/graph_ops/update_vertex.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include 17 | 18 | #include "graph/graph_ops.h" 19 | #include "graph/type_def.h" 20 | 21 | using std::string; 22 | 23 | using gart::property::StringViewList; 24 | 25 | namespace gart { 26 | namespace graph { 27 | 28 | void process_update_vertex(const StringViewList& cmd, 29 | graph::GraphStore* graph_store) { 30 | int write_epoch = stoi(string(cmd[0])); 31 | uint64_t vid = static_cast(stoll(string(cmd[1]))); 32 | StringViewList props(cmd.begin() + 3, cmd.end()); 33 | graph_store->update_inner_vertex(write_epoch, vid, props); 34 | } 35 | 36 | } // namespace graph 37 | } // namespace gart 38 | -------------------------------------------------------------------------------- /vegito/src/graph/type_def.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef VEGITO_SRC_GRAPH_TYPE_DEF_H_ 17 | #define VEGITO_SRC_GRAPH_TYPE_DEF_H_ 18 | 19 | #include "util/inline_str.h" 20 | 21 | namespace gart { 22 | namespace graph { 23 | 24 | // data types from LDBC SNB 25 | 26 | typedef uint64_t ID; 27 | typedef uint32_t Uint; 28 | 29 | typedef int Date; 30 | typedef int64_t DateTime; 31 | typedef int64_t Time; 32 | typedef inline_str_fixed<22> TimeStamp; 33 | // format for debezium: 2010-12-30T17:15:00.0Z 34 | 35 | } // namespace graph 36 | } // namespace gart 37 | 38 | #endif // VEGITO_SRC_GRAPH_TYPE_DEF_H_ 39 | -------------------------------------------------------------------------------- /vegito/src/system_flags.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include "system_flags.h" // NOLINT(build/include_subdir) 17 | 18 | DEFINE_string(kafka_broker_list, "127.0.0.1:9092", 19 | "Kafka broker list for GART writer."); 20 | DEFINE_string(kafka_unified_log_topic, "unified_log", 21 | "Kafka topic for unified logs."); 22 | DEFINE_string(kafka_unified_log_file, "test_graph.txt", 23 | "The file to record unified logs."); // for tests 24 | 25 | DEFINE_string(etcd_endpoint, "http://127.0.0.1:2379", 26 | "etcd endpoint for schema."); 27 | DEFINE_string(meta_prefix, "gart_meta_", "meta prefix for etcd."); 28 | DEFINE_string(v6d_ipc_socket, "/var/run/vineyard.sock", "Vineyard IPC socket."); 29 | 30 | DEFINE_int32(subgraph_num, 1, "total subgraph number."); 31 | DEFINE_int32(subgraph_id, 0, "subgraph id."); 32 | 33 | DEFINE_int64(default_max_vertex_number, 1 * (1ul << 26), 34 | "default max vertex number."); 35 | DEFINE_int64(default_max_memory_usage_for_each_type_vertex, 10 * (1ul << 30), 36 | "default max memory usage for each type vertex."); // in bytes 37 | DEFINE_string(customized_vertex_number_memory_usage_config, 38 | "", // format: "type1:100:10000,type2:100:10000" 39 | "customized vertex number memory usage config."); 40 | 41 | DEFINE_int32(num_threads, 2, "number of threads"); -------------------------------------------------------------------------------- /vegito/src/system_flags.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef VEGITO_SRC_SYSTEM_FLAGS_H_ 17 | #define VEGITO_SRC_SYSTEM_FLAGS_H_ 18 | 19 | #include 20 | 21 | DECLARE_string(kafka_broker_list); 22 | DECLARE_string(kafka_unified_log_topic); 23 | DECLARE_string(kafka_unified_log_file); // for tests 24 | 25 | DECLARE_string(etcd_endpoint); 26 | DECLARE_string(meta_prefix); 27 | DECLARE_string(v6d_ipc_socket); 28 | 29 | DECLARE_int32(subgraph_num); 30 | DECLARE_int32(subgraph_id); 31 | 32 | DECLARE_int64(default_max_vertex_number); 33 | DECLARE_int64(default_max_memory_usage_for_each_type_vertex); // in bytes 34 | DECLARE_string( 35 | customized_vertex_number_memory_usage_config); // format: 36 | // "type1:100:10000,type2:100:10000" 37 | 38 | DECLARE_int32(num_threads); // number of threads 39 | #endif // VEGITO_SRC_SYSTEM_FLAGS_H_ 40 | -------------------------------------------------------------------------------- /vegito/test/config/connect-debezium-mysql.properties: -------------------------------------------------------------------------------- 1 | name=test-connector 2 | connector.class=io.debezium.connector.mysql.MySqlConnector 3 | database.hostname=127.0.0.1 4 | database.port=3306 5 | database.user= 6 | database.password= 7 | database.server.id=1 8 | database.server.name=mysql 9 | database.include.list=ldbc 10 | table.include.list=ldbc.organisation,ldbc.place,ldbc.tag,ldbc.tagclass,ldbc.person,ldbc.comment,ldbc.post,ldbc.forum,ldbc.org_islocationin,ldbc.ispartof,ldbc.issubclassof,ldbc.hastype,ldbc.comment_hascreator,ldbc.comment_hastag,ldbc.comment_islocationin,ldbc.replyof_comment,ldbc.replyof_post,ldbc.post_hascreator,ldbc.post_hastag,ldbc.post_islocationin,ldbc.forum_containerof,ldbc.forum_hasmoderator,ldbc.forum_hastag,ldbc.person_hasinterest,ldbc.person_islocationin,ldbc.forum_hasmember,ldbc.knows,ldbc.likes_comment,ldbc.likes_post,ldbc.studyat,ldbc.workat 11 | database.history.kafka.bootstrap.servers=localhost:9092 12 | schema.history.internal.kafka.bootstrap.servers=localhost:9092 13 | database.history.kafka.topic=history 14 | include.schema.changes=false 15 | tombstones.on.delete=false 16 | snapshot.mode=initial 17 | topic.prefix=fullfillment 18 | schema.history.internal.kafka.topic=schemahistory.fullfillment 19 | transforms=Combine,ReplaceField 20 | transforms.Combine.type=io.debezium.transforms.ByLogicalTableRouter 21 | transforms.Combine.topic.regex=(.*) 22 | transforms.Combine.topic.replacement=binlog 23 | transforms.ReplaceField.type=org.apache.kafka.connect.transforms.ReplaceField$Value 24 | transforms.ReplaceField.exclude=ts_ms,transaction 25 | key.converter=org.apache.kafka.connect.json.JsonConverter 26 | value.converter=org.apache.kafka.connect.json.JsonConverter 27 | key.converter.schemas.enable=false 28 | value.converter.schemas.enable=false 29 | -------------------------------------------------------------------------------- /vegito/test/config/connect-debezium-postgresql.properties: -------------------------------------------------------------------------------- 1 | name=test-connector 2 | connector.class=io.debezium.connector.postgresql.PostgresConnector 3 | database.hostname=127.0.0.1 4 | database.port=5432 5 | database.user= 6 | database.password= 7 | database.dbname=ldbc 8 | table.include.list=public.organisation,public.place,public.tag,public.tagclass,public.person,public.comment,public.post,public.forum,public.org_islocationin,public.ispartof,public.issubclassof,public.hastype,public.comment_hascreator,public.comment_hastag,public.comment_islocationin,public.replyof_comment,public.replyof_post,public.post_hascreator,public.post_hastag,public.post_islocationin,public.forum_containerof,public.forum_hasmoderator,public.forum_hastag,public.person_hasinterest,public.person_islocationin,public.forum_hasmember,public.knows,public.likes_comment,public.likes_post,public.studyat,public.workat 9 | database.history.kafka.bootstrap.servers=localhost:9092 10 | schema.history.internal.kafka.bootstrap.servers=localhost:9092 11 | plugin.name=pgoutput 12 | snapshot.mode=always 13 | database.server.id=1 14 | include.schema.changes=false 15 | tombstones.on.delete=false 16 | publication.autocreate.mode=filtered 17 | topic.prefix=fullfillment 18 | schema.history.internal.kafka.topic=schemahistory.fullfillment 19 | transforms=Combine,ReplaceField 20 | transforms.Combine.type=io.debezium.transforms.ByLogicalTableRouter 21 | transforms.Combine.topic.regex=(.*) 22 | transforms.Combine.topic.replacement=binlog 23 | transforms.ReplaceField.type=org.apache.kafka.connect.transforms.ReplaceField$Value 24 | transforms.ReplaceField.exclude=ts_ms,transaction 25 | key.converter=org.apache.kafka.connect.json.JsonConverter 26 | value.converter=org.apache.kafka.connect.json.JsonConverter 27 | key.converter.schemas.enable=false 28 | value.converter.schemas.enable=false 29 | -------------------------------------------------------------------------------- /vegito/test/load_graph_test.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include 17 | 18 | #include "framework/bench_runner.h" 19 | #include "util/spinlock.h" 20 | 21 | namespace { 22 | // some helper functions 23 | 24 | SpinLock exit_lock; // race between two signals 25 | 26 | void printTraceExit(int sig) {} 27 | 28 | void sigsegv_handler(int sig) { 29 | exit_lock.Lock(); 30 | fprintf(stderr, "Meet a segmentation fault!\n"); 31 | // printTraceExit(sig); 32 | exit(-1); 33 | } 34 | 35 | void sigint_handler(int sig) { 36 | exit_lock.Lock(); 37 | fprintf(stderr, "Meet an interrupt!\n"); 38 | // printTraceExit(sig); 39 | exit(-1); 40 | } 41 | 42 | void sigabrt_handler(int sig) { 43 | exit_lock.Lock(); 44 | fprintf(stderr, "Meet an assertion failure!\n"); 45 | // printTraceExit(sig); 46 | exit(-1); 47 | } 48 | 49 | } // anonymous namespace 50 | 51 | int main(int argc, char** argv) { 52 | /* install the event handler if necessary */ 53 | signal(SIGSEGV, sigsegv_handler); 54 | signal(SIGABRT, sigabrt_handler); 55 | signal(SIGINT, sigint_handler); 56 | 57 | /* parse arguments for whole system */ 58 | gflags::ParseCommandLineFlags(&argc, &argv, true); 59 | 60 | gart::framework::config.parse_sys_args(argc, argv); 61 | 62 | gart::framework::Runner* runner = new gart::framework::Runner(); 63 | runner->run(); 64 | return 0; 65 | } 66 | -------------------------------------------------------------------------------- /vegito/test/run_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ../build/load_graph_test --kafka_unified_log_file ./data/test_graph.txt --v6d_ipc_socket /opt/tmp/tmp.sock 4 | -------------------------------------------------------------------------------- /vegito/test/schema/distributed_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_host": "127.0.0.1", 3 | "db_port": 5432, 4 | "db_type": "postgresql", 5 | "db_name": "ldbc", 6 | "db_user": "dbuser", 7 | "db_password": "123456", 8 | "rgmapping_file": "/path/to/rgmapping.yaml", 9 | "v6d_socket": "/tmp/ldbc.sock", 10 | "v6d_size": "750G", 11 | "etcd_endpoint": "http://127.0.0.1:23760", 12 | "etcd_prefix": "gart_meta_", 13 | "kafka_server": "127.0.0.1:9092", 14 | "total_subgraph_num": 2, 15 | "gart_bin_path": "/path/to/GART/build/", 16 | "enable_bulkload": 1, 17 | "capturer_host": "127.0.0.1", 18 | "converter_host": "127.0.0.1", 19 | "kafka_path": "/path/to/kafka", 20 | "writer_hosts": [ 21 | { 22 | "subgraph_id": 0, 23 | "host": "127.0.0.1" 24 | }, 25 | { 26 | "subgraph_id": 1, 27 | "host": "127.0.0.1" 28 | } 29 | ] 30 | } -------------------------------------------------------------------------------- /vegito/test/schema/financial_transactions.sql: -------------------------------------------------------------------------------- 1 | CREATE DATABASE IF NOT EXISTS financial_transactions; 2 | USE financial_transactions; 3 | 4 | CREATE TABLE Companies ( 5 | id BIGINT NOT NULL, 6 | name VARCHAR(255), 7 | PRIMARY KEY (id) 8 | ); 9 | 10 | CREATE TABLE Persons ( 11 | id BIGINT NOT NULL, 12 | name VARCHAR(255), 13 | company_id BIGINT, 14 | PRIMARY KEY (id), 15 | FOREIGN KEY (company_id) REFERENCES Companies(id) 16 | ); 17 | 18 | CREATE TABLE Accounts ( 19 | number BIGINT NOT NULL, 20 | account_type VARCHAR(255), 21 | person_id BIGINT, 22 | company_id BIGINT, 23 | PRIMARY KEY (number), 24 | FOREIGN KEY (person_id) REFERENCES Persons(id), 25 | FOREIGN KEY (company_id) REFERENCES Companies(id) 26 | ); 27 | 28 | CREATE TABLE Transactions ( 29 | id BIGINT NOT NULL, 30 | from_account BIGINT, 31 | to_account BIGINT, 32 | date VARCHAR(255), 33 | amount DOUBLE, 34 | PRIMARY KEY (id), 35 | FOREIGN KEY (from_account) REFERENCES Accounts(number), 36 | FOREIGN KEY (to_account) REFERENCES Accounts(number) 37 | ); -------------------------------------------------------------------------------- /vegito/test/schema/k8s_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_host": "127.0.0.1", 3 | "db_port": 3306, 4 | "db_type": "mysql", 5 | "db_name": "ldbc", 6 | "db_user": "dbuser", 7 | "db_password": "123456", 8 | "rgmapping_file": "/path/to/rgmapping-ldbc.yaml", 9 | "v6d_socket": "/tmp/ldbc.sock", 10 | "v6d_size": "750G", 11 | "etcd_prefix": "gart_meta_", 12 | "total_subgraph_num": 2, 13 | "enable_bulkload": 1 14 | } -------------------------------------------------------------------------------- /vegito/test/schema/rgmapping-smallbank.sql: -------------------------------------------------------------------------------- 1 | CREATE PROPERTY GRAPH smallbank 2 | VERTEX TABLES ( 3 | Accounts 4 | KEY ( custid ) 5 | LABEL Account PROPERTIES ( custid, name), 6 | Savings 7 | KEY ( custid ) 8 | LABEL Saving PROPERTIES ( custid, bal), 9 | Checking 10 | KEY ( custid ) 11 | LABEL Checking PROPERTIES ( custid, bal) 12 | ) 13 | EDGE TABLES ( 14 | Savings 15 | SOURCE KEY ( custid ) REFERENCES Accounts 16 | DESTINATION KEY ( custid ) REFERENCES Savings 17 | LABEL saving_account NO PROPERTIES, 18 | Checking 19 | SOURCE KEY ( custid ) REFERENCES Accounts 20 | DESTINATION KEY ( custid ) REFERENCES Checking 21 | LABEL checking_account NO PROPERTIES 22 | ) 23 | -------------------------------------------------------------------------------- /vegito/test/schema/rgmapping-smallbank.yaml: -------------------------------------------------------------------------------- 1 | !!gart.pgql.GSchema 2 | graph: smallbank 3 | loadingConfig: 4 | dataSource: rdbms 5 | database: ldbc 6 | method: append 7 | enableRowStore: false 8 | vertexMappings: 9 | vertex_types: 10 | - type_name: account 11 | dataSourceName: accounts 12 | idFieldName: custid 13 | mappings: 14 | - property: custid 15 | dataField: 16 | name: custid 17 | - property: name 18 | dataField: 19 | name: name 20 | - type_name: saving 21 | dataSourceName: savings 22 | idFieldName: custid 23 | mappings: 24 | - property: custid 25 | dataField: 26 | name: custid 27 | - property: bal 28 | dataField: 29 | name: bal 30 | - type_name: checking 31 | dataSourceName: checking 32 | idFieldName: custid 33 | mappings: 34 | - property: custid 35 | dataField: 36 | name: custid 37 | - property: bal 38 | dataField: 39 | name: bal 40 | edgeMappings: 41 | edge_types: 42 | - type_pair: 43 | edge: saving_account 44 | source_vertex: account 45 | destination_vertex: saving 46 | dataSourceName: savings 47 | sourceVertexMappings: 48 | - dataField: 49 | name: custid 50 | destinationVertexMappings: 51 | - dataField: 52 | name: custid 53 | dataFieldMappings: [ 54 | ] 55 | - type_pair: 56 | edge: checking_account 57 | source_vertex: account 58 | destination_vertex: checking 59 | dataSourceName: checking 60 | sourceVertexMappings: 61 | - dataField: 62 | name: custid 63 | destinationVertexMappings: 64 | - dataField: 65 | name: custid 66 | dataFieldMappings: [ 67 | ] 68 | --------------------------------------------------------------------------------