├── .dockerignore ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── build-issue.md │ └── feature_request.md └── PULL_REQUEST_TEMPLATE ├── .gitignore ├── CMakeLists.txt ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── build.sh ├── cleanup.sh ├── cmake-modules ├── BuildOptions.cmake ├── Dependencies.cmake ├── FindAnt.cmake ├── FindMkDocs.cmake └── Functions.cmake ├── conf ├── confluo-env.sh ├── confluo.conf └── hosts ├── docker ├── Dockerfile.ubuntu └── README.md ├── docs ├── CMakeLists.txt ├── DoxyfileCpp.in ├── DoxyfileCppClient.in ├── DoxyfileJavaClient.in ├── DoxyfilePyClient.in ├── README.md ├── material-theme │ ├── 404.html │ ├── __init__.py │ ├── assets │ │ ├── fonts │ │ │ ├── font-awesome.css │ │ │ ├── material-icons.css │ │ │ └── specimen │ │ │ │ ├── FontAwesome.ttf │ │ │ │ ├── FontAwesome.woff │ │ │ │ ├── FontAwesome.woff2 │ │ │ │ ├── MaterialIcons-Regular.ttf │ │ │ │ ├── MaterialIcons-Regular.woff │ │ │ │ └── MaterialIcons-Regular.woff2 │ │ ├── images │ │ │ ├── favicon.png │ │ │ └── icons │ │ │ │ ├── bitbucket.1b09e088.svg │ │ │ │ ├── github.f0b8504a.svg │ │ │ │ └── gitlab.6dd19c00.svg │ │ ├── javascripts │ │ │ ├── application.905597d0.js │ │ │ ├── lunr │ │ │ │ ├── lunr.da.js │ │ │ │ ├── lunr.de.js │ │ │ │ ├── lunr.du.js │ │ │ │ ├── lunr.es.js │ │ │ │ ├── lunr.fi.js │ │ │ │ ├── lunr.fr.js │ │ │ │ ├── lunr.hu.js │ │ │ │ ├── lunr.it.js │ │ │ │ ├── lunr.jp.js │ │ │ │ ├── lunr.multi.js │ │ │ │ ├── lunr.no.js │ │ │ │ ├── lunr.pt.js │ │ │ │ ├── lunr.ro.js │ │ │ │ ├── lunr.ru.js │ │ │ │ ├── lunr.stemmer.support.js │ │ │ │ ├── lunr.sv.js │ │ │ │ ├── lunr.tr.js │ │ │ │ └── tinyseg.js │ │ │ └── modernizr.20ef595d.js │ │ └── stylesheets │ │ │ ├── application-palette.22915126.css │ │ │ └── application.11e41852.css │ ├── base.html │ ├── main.html │ ├── manifest.json │ ├── mkdocs_theme.yml │ └── partials │ │ ├── footer.html │ │ ├── header.html │ │ ├── hero.html │ │ ├── integrations │ │ ├── analytics.html │ │ └── disqus.html │ │ ├── language.html │ │ ├── language │ │ ├── ar.html │ │ ├── ca.html │ │ ├── cs.html │ │ ├── da.html │ │ ├── de.html │ │ ├── en.html │ │ ├── es.html │ │ ├── fa.html │ │ ├── fi.html │ │ ├── fr.html │ │ ├── gl.html │ │ ├── he.html │ │ ├── hi.html │ │ ├── hr.html │ │ ├── hu.html │ │ ├── id.html │ │ ├── it.html │ │ ├── ja.html │ │ ├── kr.html │ │ ├── nl.html │ │ ├── no.html │ │ ├── pl.html │ │ ├── pt.html │ │ ├── ru.html │ │ ├── sh.html │ │ ├── sk.html │ │ ├── sr.html │ │ ├── sv.html │ │ ├── tr.html │ │ ├── uk.html │ │ ├── vi.html │ │ ├── zh-Hant.html │ │ ├── zh-TW.html │ │ └── zh.html │ │ ├── nav-item.html │ │ ├── nav.html │ │ ├── palette.html │ │ ├── search.html │ │ ├── social.html │ │ ├── source.html │ │ ├── tabs-item.html │ │ ├── tabs.html │ │ ├── toc-item.html │ │ └── toc.html ├── mkdocs.yml.in ├── pyclient │ └── source │ │ ├── conf.py │ │ ├── index.rst │ │ ├── modules.rst │ │ └── rpc.rst └── src │ ├── api.md │ ├── applications.md │ ├── client_api.md │ ├── contact.md │ ├── contributing.md │ ├── cpp_api.md │ ├── cpp_client_api.md │ ├── data_archival.md │ ├── img │ ├── logo.png │ ├── pub_sub_perf.png │ └── tsdb_perf.png │ ├── index.md │ ├── install.md │ ├── java_client_api.md │ ├── loading_data.md │ ├── modes_of_operation.md │ ├── network_monitoring.md │ ├── offline_queries.md │ ├── online_queries.md │ ├── pub_sub.md │ ├── python_client_api.md │ ├── queries.md │ ├── quick_start.md │ ├── research.md │ ├── timeseries_db.md │ └── type_system.md ├── ec2 └── setup.sh ├── examples ├── libstreaming │ ├── CMakeLists.txt │ ├── streaming │ │ ├── stream_consumer.h │ │ └── stream_producer.h │ └── test │ │ ├── stream_test.h │ │ ├── streaming_test_utils.h │ │ └── test_main.cc └── libtimeseries │ ├── CMakeLists.txt │ ├── test │ ├── test_main.cc │ └── timeseries_db_test.h │ └── timeseries │ └── timeseries_db.h ├── install.sh ├── javaclient ├── CMakeLists.txt ├── build.xml.in ├── pom.xml └── src │ ├── main │ └── java │ │ └── confluo │ │ └── rpc │ │ ├── AlertStream.java │ │ ├── Column.java │ │ ├── ConfigurationParams.java │ │ ├── DataParser.java │ │ ├── DataType.java │ │ ├── DataTypes.java │ │ ├── Field.java │ │ ├── Record.java │ │ ├── RecordBatchBuilder.java │ │ ├── RecordStream.java │ │ ├── RpcClient.java │ │ ├── Schema.java │ │ ├── SchemaBuilder.java │ │ ├── StorageMode.java │ │ ├── TypeConversions.java │ │ ├── TypeId.java │ │ ├── rpc_atomic_multilog_info.java │ │ ├── rpc_column.java │ │ ├── rpc_data_type.java │ │ ├── rpc_invalid_operation.java │ │ ├── rpc_iterator_descriptor.java │ │ ├── rpc_iterator_handle.java │ │ ├── rpc_iterator_type.java │ │ ├── rpc_management_exception.java │ │ ├── rpc_record_batch.java │ │ ├── rpc_record_block.java │ │ ├── rpc_service.java │ │ └── rpc_storage_mode.java │ └── test │ └── java │ └── TestRpcClient.java ├── libconfluo ├── CMakeLists.txt ├── confluo │ ├── aggregate │ │ ├── aggregate.h │ │ ├── aggregate_info.h │ │ ├── aggregate_manager.h │ │ └── aggregate_ops.h │ ├── aggregated_reflog.h │ ├── alert.h │ ├── alert_index.h │ ├── archival │ │ ├── archival_actions.h │ │ ├── archival_metadata.h │ │ ├── archival_mode.h │ │ ├── archival_utils.h │ │ ├── archiver.h │ │ ├── atomic_multilog_archiver.h │ │ ├── filter_archiver.h │ │ ├── filter_log_archiver.h │ │ ├── index_archiver.h │ │ ├── index_log_archiver.h │ │ ├── io │ │ │ ├── incremental_file_offset.h │ │ │ ├── incremental_file_reader.h │ │ │ ├── incremental_file_stream.h │ │ │ └── incremental_file_writer.h │ │ ├── load_utils.h │ │ └── monolog_linear_archiver.h │ ├── atomic_multilog.h │ ├── atomic_multilog_metadata.h │ ├── compression │ │ ├── confluo_encoder.h │ │ ├── delta_decoder.h │ │ ├── delta_encoder.h │ │ ├── lz4_decoder.h │ │ └── lz4_encoder.h │ ├── conf │ │ ├── configuration_params.h │ │ ├── configuration_parser.h │ │ └── defaults.h │ ├── confluo_store.h │ ├── container │ │ ├── bitmap │ │ │ ├── bitmap.h │ │ │ ├── bitmap_array.h │ │ │ └── delta_encoded_array.h │ │ ├── cursor │ │ │ ├── alert_cursor.h │ │ │ ├── batched_cursor.h │ │ │ ├── offset_cursors.h │ │ │ └── record_cursors.h │ │ ├── data_log.h │ │ ├── flatten.h │ │ ├── lazy │ │ │ └── stream.h │ │ ├── monolog │ │ │ ├── monolog.h │ │ │ ├── monolog_exp2.h │ │ │ ├── monolog_exp2_linear.h │ │ │ ├── monolog_iterator.h │ │ │ ├── monolog_linear.h │ │ │ └── monolog_linear_bucket.h │ │ ├── radix_tree.h │ │ ├── record_offset_range.h │ │ ├── reflog.h │ │ ├── sketch │ │ │ ├── count_min_sketch.h │ │ │ ├── count_sketch.h │ │ │ ├── frequency_functions.h │ │ │ ├── hash_manager.h │ │ │ ├── priority_queue.h │ │ │ ├── sketch_utils.h │ │ │ └── universal_sketch.h │ │ ├── string_map.h │ │ └── unique_byte_array.h │ ├── exceptions.h │ ├── filter.h │ ├── filter_log.h │ ├── index_log.h │ ├── parser │ │ ├── aggregate_parser.h │ │ ├── expression_compiler.h │ │ ├── expression_parser.h │ │ ├── schema_parser.h │ │ └── trigger_parser.h │ ├── planner │ │ ├── query_ops.h │ │ ├── query_plan.h │ │ └── query_planner.h │ ├── read_tail.h │ ├── schema │ │ ├── column.h │ │ ├── column_snapshot.h │ │ ├── field.h │ │ ├── index_state.h │ │ ├── record.h │ │ ├── record_batch.h │ │ ├── schema.h │ │ └── schema_snapshot.h │ ├── storage │ │ ├── allocator.h │ │ ├── encoded_ptr.h │ │ ├── memory_stat.h │ │ ├── ptr_aux_block.h │ │ ├── ptr_metadata.h │ │ ├── reference_counts.h │ │ ├── storage.h │ │ ├── storage_allocator.h │ │ ├── storage_utils.h │ │ ├── swappable_encoded_ptr.h │ │ └── swappable_ptr.h │ ├── threads │ │ ├── periodic_task.h │ │ ├── task_pool.h │ │ └── thread_manager.h │ ├── trigger.h │ ├── trigger_log.h │ ├── types │ │ ├── arithmetic_ops.h │ │ ├── byte_string.h │ │ ├── data_type.h │ │ ├── immutable_value.h │ │ ├── key_ops.h │ │ ├── mutable_value.h │ │ ├── numeric.h │ │ ├── primitive_types.h │ │ ├── raw_data.h │ │ ├── relational_ops.h │ │ ├── serde_ops.h │ │ ├── string_ops.h │ │ ├── type_manager.h │ │ └── type_properties.h │ └── univ_sketch_log.h ├── src │ ├── aggregate │ │ ├── aggregate.cc │ │ ├── aggregate_info.cc │ │ ├── aggregate_manager.cc │ │ └── aggregate_ops.cc │ ├── aggregated_reflog.cc │ ├── alert.cc │ ├── alert_index.cc │ ├── archival │ │ ├── archival_actions.cc │ │ ├── archival_metadata.cc │ │ ├── archival_utils.cc │ │ ├── atomic_multilog_archiver.cc │ │ ├── filter_archiver.cc │ │ ├── filter_log_archiver.cc │ │ ├── index_archiver.cc │ │ ├── index_log_archiver.cc │ │ ├── io │ │ │ ├── incremental_file_offset.cc │ │ │ ├── incremental_file_reader.cc │ │ │ ├── incremental_file_stream.cc │ │ │ └── incremental_file_writer.cc │ │ └── load_utils.cc │ ├── atomic_multilog.cc │ ├── atomic_multilog_metadata.cc │ ├── compression │ │ └── confluo_encoder.cc │ ├── confluo_store.cc │ ├── container │ │ ├── cursor │ │ │ ├── alert_cursor.cc │ │ │ ├── offset_cursors.cc │ │ │ └── record_cursors.cc │ │ ├── data_log.cc │ │ ├── reflog.cc │ │ └── sketch │ │ │ ├── hash_manager.cc │ │ │ └── universal_sketch.cc │ ├── filter.cc │ ├── parser │ │ ├── aggregate_parser.cc │ │ ├── expression_compiler.cc │ │ ├── expression_parser.cc │ │ ├── schema_parser.cc │ │ └── trigger_parser.cc │ ├── planner │ │ ├── query_ops.cc │ │ ├── query_plan.cc │ │ └── query_planner.cc │ ├── read_tail.cc │ ├── schema │ │ ├── column.cc │ │ ├── field.cc │ │ ├── index_state.cc │ │ ├── record.cc │ │ ├── record_batch.cc │ │ ├── schema.cc │ │ └── schema_snapshot.cc │ ├── storage │ │ ├── memory_stat.cc │ │ ├── ptr_aux_block.cc │ │ ├── ptr_metadata.cc │ │ ├── reference_counts.cc │ │ ├── storage.cc │ │ └── storage_allocator.cc │ ├── threads │ │ ├── periodic_task.cc │ │ ├── task_pool.cc │ │ └── thread_manager.cc │ ├── trigger.cc │ └── types │ │ ├── byte_string.cc │ │ ├── data_type.cc │ │ ├── immutable_value.cc │ │ ├── mutable_value.cc │ │ ├── numeric.cc │ │ ├── primitive_types.cc │ │ ├── raw_data.cc │ │ ├── type_manager.cc │ │ └── type_properties.cc └── test │ ├── aggregate │ └── aggregate_test.h │ ├── aggregated_reflog_test.h │ ├── archival │ ├── filter_archival_test.h │ ├── filter_load_test.h │ ├── index_archival_test.h │ ├── index_load_test.h │ ├── monolog_linear_archival_test.h │ └── monolog_linear_load_test.h │ ├── atomic_multilog_metadata_test.h │ ├── atomic_multilog_test.h │ ├── compression │ ├── delta_encode_test.h │ └── lz4_encode_test.h │ ├── confluo_store_test.h │ ├── container │ ├── bitmap │ │ ├── bitmap_array_test.h │ │ ├── bitmap_test.h │ │ └── delta_encoded_array_test.h │ ├── cursor │ │ └── batched_cursor_test.h │ ├── flatten_test.h │ ├── monolog │ │ └── monolog_test.h │ ├── radix_tree_test.h │ ├── sketch │ │ ├── count_sketch_test.h │ │ ├── priority_queue_test.h │ │ ├── sketch_test_utils.h │ │ └── universal_sketch_test.h │ ├── stream_test.h │ └── string_map_test.h │ ├── filter_test.h │ ├── parser │ ├── aggregate_parser_test.h │ ├── expression_compiler_test.h │ ├── expression_parser_test.h │ ├── schema_parser_test.h │ └── trigger_parser_test.h │ ├── schema │ ├── column_test.h │ ├── index_state_test.h │ ├── record_batch_test.h │ └── schema_test.h │ ├── storage │ ├── memory_stat_test.h │ ├── ptr_test.h │ └── storage_allocator_test.h │ ├── test_main.cc │ ├── test_utils.h │ ├── threads │ ├── periodic_task_test.h │ ├── task_test.h │ └── thread_manager_test.h │ └── types │ ├── byte_string_test.h │ ├── data_types_test.h │ ├── immutable_value_test.h │ ├── ip_address.h │ ├── mutable_value_test.h │ ├── size_type.h │ └── type_manager_test.h ├── librpc ├── CMakeLists.txt ├── rpc │ ├── rpc_alert_stream.h │ ├── rpc_client.h │ ├── rpc_configuration_params.h │ ├── rpc_constants.h │ ├── rpc_defaults.h │ ├── rpc_handler_registry.h │ ├── rpc_record_batch_builder.h │ ├── rpc_record_stream.h │ ├── rpc_server.h │ ├── rpc_service.h │ ├── rpc_service.tcc │ ├── rpc_thread_factory.h │ ├── rpc_type_conversions.h │ ├── rpc_types.h │ └── rpc_types.tcc ├── src │ ├── confluo_server.cc │ ├── rpc_alert_stream.cc │ ├── rpc_client.cc │ ├── rpc_constants.cc │ ├── rpc_handler_registry.cc │ ├── rpc_record_batch_builder.cc │ ├── rpc_record_stream.cc │ ├── rpc_server.cc │ ├── rpc_service.cc │ ├── rpc_thread_factory.cc │ ├── rpc_type_conversions.cc │ └── rpc_types.cc └── test │ ├── client_connection_test.h │ ├── client_read_ops_test.h │ ├── client_write_ops_test.h │ ├── rpc_test_utils.h │ └── test_main.cc ├── libutils ├── CMakeLists.txt ├── src │ └── utils │ │ ├── assertions.cc │ │ ├── cmd_parse.cc │ │ ├── config_utils.cc │ │ ├── error_handling.cc │ │ ├── file_utils.cc │ │ ├── io_utils.cc │ │ ├── logger.cc │ │ ├── math_utils.cc │ │ ├── mmap_utils.cc │ │ ├── rand_utils.cc │ │ ├── string_utils.cc │ │ └── time_utils.cc └── utils │ ├── assertions.h │ ├── atomic.h │ ├── bit_utils.h │ ├── byte_utils.h │ ├── cmd_parse.h │ ├── config_utils.h │ ├── debug_utils.h │ ├── error_handling.h │ ├── file_utils.h │ ├── function_traits.h │ ├── io_utils.h │ ├── logger.h │ ├── math_utils.h │ ├── mmap_utils.h │ ├── optional.h │ ├── rand_utils.h │ ├── string_utils.h │ ├── thread_utils.h │ └── time_utils.h ├── pyclient ├── CMakeLists.txt ├── confluo │ ├── __init__.py │ └── rpc │ │ ├── __init__.py │ │ ├── batch.py │ │ ├── client.py │ │ ├── configuration_params.py │ │ ├── constants.py │ │ ├── data_types.py │ │ ├── rpc_service.py │ │ ├── schema.py │ │ ├── storage.py │ │ ├── stream.py │ │ ├── ttypes.py │ │ └── type_conversions.py ├── preamble.py ├── setup.cfg ├── setup.py.in └── test │ └── test_rpc_client.py ├── sbin ├── confluo-config.sh ├── hosts.sh ├── load-confluo-env.sh ├── start-confluo.sh ├── stop-confluo.sh └── sync └── thrift ├── genall.sh └── rpc.thrift /.dockerignore: -------------------------------------------------------------------------------- 1 | .* 2 | Dockerfile* 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[BUG]" 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ### Describe the bug 11 | (A clear and concise description of what the bug is.) 12 | 13 | ### How to reproduce the bug? 14 | (Provide clear steps to reproduce the bug.) 15 | 16 | ### Expected behavior 17 | (A clear and concise description of what you expected to happen.) 18 | 19 | ### Detailed logs 20 | (Provide logs generated by Confluo, if applicable.) 21 | 22 | ### Platform Details 23 | (Please provide information regarding the OS, including type and version.) 24 | 25 | ### Additional context 26 | (Add any other context about the problem here.) 27 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/build-issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Build Issue 3 | about: Create a report to help us improve Confluo's build system 4 | title: "[BUILD]" 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ### What OS are you using? 11 | (Provide details regarding your OS type and version.) 12 | 13 | ### What compiler are you using? What version? 14 | (Provide details regarding the type and version of your C/C++ compiler, Java compiler (only for Java client build issues) and Python interpreter (only for Python client build issues).) 15 | 16 | ### What's your CMake arguments? 17 | (Provide details regarding the CMake arguments used for Confluo's build.) 18 | 19 | ### What is the output of the CMake and make commands on a fresh build? (Please sanitize any sensitive information) 20 | (Provide the verbatim logs for CMake and make commands on a fresh build. To obtain a fresh build, remove the contents of the out-of-source build directory before running `cmake` and `make`.) 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "[FEATURE]" 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ### Describe the problem your feature request aims to solve. 11 | (A clear and concise description of the problem.) 12 | 13 | ### Describe your solution 14 | (A clear and concise description of how the solution resolves the problem.) 15 | 16 | ### Describe alternatives you've considered 17 | (A clear and concise description of any alternative solutions or features considered.) 18 | 19 | ### Additional context 20 | (Add any other context or screenshots about the feature request here.) 21 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE: -------------------------------------------------------------------------------- 1 | ## What changes were proposed in this pull request? 2 | 3 | (Please fill in changes proposed in this fix) 4 | 5 | ## How was this patch tested? 6 | 7 | (Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests) 8 | 9 | Please review https://ucbrise.github.io/confluo/contributing/ before opening a pull request. 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | *.obj 6 | 7 | # Precompiled Headers 8 | *.gch 9 | *.pch 10 | 11 | # Compiled Dynamic libraries 12 | *.so 13 | *.dylib 14 | *.dll 15 | 16 | # Fortran module files 17 | *.mod 18 | 19 | # Compiled Static libraries 20 | *.lai 21 | *.la 22 | *.a 23 | *.lib 24 | lib/* 25 | 26 | # Executables 27 | *.exe 28 | *.out 29 | *.app 30 | bin/* 31 | 32 | # Directories 33 | external/thrift-0.10.0/lib/py/dist/ 34 | external/thrift-0.10.0/lib/py/thrift.egg-info/ 35 | external/thrift-0.10.0/lib/py/build/ 36 | build 37 | target 38 | log 39 | thrift-gen.files 40 | .eggs 41 | pyclient/confluo.egg-info 42 | pyclient/dist 43 | 44 | # Eclipse files 45 | .settings 46 | debug 47 | 48 | # Netbeans files 49 | nbproject 50 | 51 | # Idea files 52 | .idea 53 | *.iml 54 | 55 | # CLion Files 56 | cmake-build-debug* 57 | cmake-build-release* 58 | 59 | # generic ignores 60 | *.deps 61 | *.dirstamp 62 | *.libs 63 | *.log 64 | *.trs 65 | *.suo 66 | *.pyc 67 | *.cache 68 | *.user 69 | *.ipch 70 | *.sdf 71 | *.jar 72 | *.exe 73 | *.dll 74 | *_ReSharper* 75 | *.opensdf 76 | *.swp 77 | *.hi 78 | *~ 79 | *tmp* 80 | .*project 81 | junit*.properties 82 | gen-* 83 | aclocal.m4 84 | acinclude.m4 85 | autom4te.cache 86 | node_modules 87 | test-driver 88 | .sonar 89 | .DS_Store 90 | /Debug/ 91 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | When contributing to this repository, please first discuss the change you wish to make via issue, 4 | email, or any other method with the owners of this repository before making a change. 5 | 6 | Please note we have a code of conduct, please follow it in all your interactions with the project. 7 | 8 | ## Pull Request Process 9 | 10 | 1. Ensure any new install or build dependencies are added as a part of the CMake build process. 11 | 2. Update the documentation with details of changes to the interface, this includes new environment 12 | variables, exposed ports, useful file locations and code examples. 13 | 3. If needed, increase the version numbers in the project's root CMakeLists.txt to the new version that this 14 | Pull Request would represent. The versioning scheme we use is [SemVer](http://semver.org/). 15 | 4. You may merge the Pull Request in once you have the sign-off of two other developers, or if you 16 | do not have permission to do that, you may request the second reviewer to merge it for you. 17 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | mkdir -p build 5 | cd build 6 | cmake "$@" .. 7 | 8 | START=$(date +%s) 9 | make 10 | make test ARGS="-VV" 11 | END=$(date +%s) 12 | echo "Total build time (real) = $(( $END - $START )) seconds" 13 | -------------------------------------------------------------------------------- /cleanup.sh: -------------------------------------------------------------------------------- 1 | rm -rf build/ 2 | find . -name "thrift-gen.files" -type d -delete 3 | -------------------------------------------------------------------------------- /cmake-modules/BuildOptions.cmake: -------------------------------------------------------------------------------- 1 | include(CMakeDependentOption) 2 | 3 | # Components to build 4 | option(BUILD_TESTS "Build with unittests" ON) 5 | option(BUILD_STRESS_TESTS "Build with stress tests" OFF) 6 | option(BUILD_RPC "Build RPC framework" ON) 7 | option(BUILD_DOC "Build documentation" OFF) 8 | option(BUILD_EXAMPLES "Build examples" ON) 9 | CMAKE_DEPENDENT_OPTION(WITH_PY_CLIENT "Build python client" ON "BUILD_RPC" OFF) 10 | CMAKE_DEPENDENT_OPTION(WITH_JAVA_CLIENT "Build java client" ON "BUILD_RPC" OFF) 11 | 12 | message(STATUS "----------------------------------------------------------") 13 | message(STATUS "Confluo version: ${CONFLUO_VERSION}") 14 | message(STATUS "Build configuration Summary") 15 | message(STATUS " Build RPC Framework: ${BUILD_RPC}") 16 | message(STATUS " Build python client libraries: ${WITH_PY_CLIENT}") 17 | message(STATUS " Build java client libraries: ${WITH_JAVA_CLIENT}") 18 | message(STATUS " Build with unit tests: ${BUILD_TESTS}") 19 | message(STATUS " Build documentation: ${BUILD_DOC}") 20 | message(STATUS " Build examples: ${BUILD_EXAMPLES}") 21 | message(STATUS "----------------------------------------------------------") 22 | -------------------------------------------------------------------------------- /cmake-modules/FindAnt.cmake: -------------------------------------------------------------------------------- 1 | # ANT_FOUND - system has Ant 2 | # Ant_EXECUTABLE - the Ant executable 3 | # 4 | # It will search the environment variable ANT_HOME if it is set 5 | 6 | include(FindPackageHandleStandardArgs) 7 | 8 | find_program(Ant_EXECUTABLE NAMES ant PATHS $ENV{ANT_HOME}/bin) 9 | find_package_handle_standard_args(Ant DEFAULT_MSG Ant_EXECUTABLE) 10 | mark_as_advanced(Ant_EXECUTABLE) 11 | -------------------------------------------------------------------------------- /cmake-modules/FindMkDocs.cmake: -------------------------------------------------------------------------------- 1 | # FindMkDocs 2 | # ----------- 3 | # 4 | # This module looks for MkDocs and is based on the FindDoxygen module 5 | # 6 | # MkDocs is a static site generator that's geared towards building project documentation. 7 | # http://www.mkdocs.org 8 | # 9 | # This modules defines the following variables: 10 | # 11 | # :: 12 | # 13 | # MKDOCS_EXECUTABLE = The path to the MkDocs command. 14 | # MKDOCS_FOUND = Was MkDocs found or not? 15 | # MKDOCS_VERSION = The version reported by mkdocs --version 16 | # 17 | 18 | # 19 | # Find MkDocs... 20 | # 21 | 22 | find_program(MKDOCS_EXECUTABLE 23 | NAMES mkdocs 24 | PATHS 25 | /usr/local/bin 26 | DOC "MkDocs documentation generation tool (http://www.mkdocs.org)" 27 | ) 28 | 29 | if(MKDOCS_EXECUTABLE) 30 | execute_process(COMMAND ${MKDOCS_EXECUTABLE} "--version" OUTPUT_VARIABLE MKDOCS_VERSION_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE) 31 | string(REGEX MATCH "[0-9]+\\.*" MKDOCS_VERSION ${MKDOCS_VERSION_OUTPUT}) 32 | endif() 33 | 34 | include(FindPackageHandleStandardArgs) 35 | find_package_handle_standard_args(MkDocs DEFAULT_MSG MKDOCS_EXECUTABLE MKDOCS_VERSION) 36 | 37 | mark_as_advanced(MKDOCS_EXECUTABLE) 38 | -------------------------------------------------------------------------------- /conf/confluo-env.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ucbrise/confluo/e9b8621e99f56b1adf5675c4296c006d89e6e582/conf/confluo-env.sh -------------------------------------------------------------------------------- /conf/confluo.conf: -------------------------------------------------------------------------------- 1 | ## Maximum concurrency 2 | # max_concurrency: 32 3 | 4 | # Index block size 5 | index_block_size: 1.0 6 | 7 | # Time resolution in nanoseconds 8 | time_resolution_ns: 1000000 9 | 10 | # Montor window in milliseconds 11 | monitor_window_ms: 10 12 | 13 | # Monitor periodicity in milliseconds 14 | monitor_periodicity_ms: 1 15 | -------------------------------------------------------------------------------- /conf/hosts: -------------------------------------------------------------------------------- 1 | localhost 2 | -------------------------------------------------------------------------------- /docker/Dockerfile.ubuntu: -------------------------------------------------------------------------------- 1 | FROM ubuntu:latest 2 | 3 | EXPOSE 9090 4 | 5 | RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends \ 6 | build-essential cmake libboost-all-dev 7 | 8 | RUN mkdir -p /opt/confluo 9 | COPY . /opt/confluo 10 | 11 | WORKDIR /opt/confluo 12 | RUN mkdir build \ 13 | && cd build \ 14 | && cmake -DBUILD_TESTS=OFF -DWITH_PY_CLIENT=OFF -DWITH_JAVA_CLIENT=OFF .. \ 15 | && make -j8 install 16 | 17 | RUN mkdir -p /var/db 18 | VOLUME /var/db 19 | 20 | ENTRYPOINT ["confluod"] 21 | CMD ["--address=0.0.0.0", "--port=9090", "--data-path=/var/db"] 22 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | # Dockerfiles for the Confluo project 2 | 3 | ## Build docker image 4 | *Run this in root confluo directory* 5 | ``` 6 | docker build -f docker/Dockerfile.ubuntu -t confluo:ubuntu . 7 | ``` 8 | 9 | ## Run image 10 | ``` 11 | docker run -p 9090:9090 -v :/var/db confluo:ubuntu 12 | ``` 13 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Confluo Documentation 2 | 3 | Confluo uses: 4 | 5 | * MkDocs to generate documentation html 6 | * Doxygen to generate API documentation 7 | 8 | as a part of its build. 9 | 10 | If the `BUILD_DOCS` variable is set to `ON` while configuring the 11 | build using CMake, the documentation is automatically built with the source. 12 | 13 | Confluo documentation is spread across text files formatted using markdown. You 14 | can read these text files directly, with [`index.md`](src/index.md) as your 15 | starting point. 16 | -------------------------------------------------------------------------------- /docs/material-theme/404.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block content %} 3 |

404 - Not found

4 | {% endblock %} 5 | -------------------------------------------------------------------------------- /docs/material-theme/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ucbrise/confluo/e9b8621e99f56b1adf5675c4296c006d89e6e582/docs/material-theme/__init__.py -------------------------------------------------------------------------------- /docs/material-theme/assets/fonts/material-icons.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 3 | * use this file except in compliance with the License. You may obtain a copy 4 | * of the License at: 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING, SOFTWARE 9 | * DISTRIBUTED UNDER THE LICENSE IS DISTRIBUTED ON AN "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. 11 | * SEE THE LICENSE FOR THE SPECIFIC LANGUAGE GOVERNING PERMISSIONS AND 12 | * LIMITATIONS UNDER THE LICENSE. 13 | */@font-face{font-family:"Material Icons";font-style:normal;font-weight:400;src:local("Material Icons"),local("MaterialIcons-Regular"),url("specimen/MaterialIcons-Regular.woff2") format("woff2"),url("specimen/MaterialIcons-Regular.woff") format("woff"),url("specimen/MaterialIcons-Regular.ttf") format("truetype")} -------------------------------------------------------------------------------- /docs/material-theme/assets/fonts/specimen/FontAwesome.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ucbrise/confluo/e9b8621e99f56b1adf5675c4296c006d89e6e582/docs/material-theme/assets/fonts/specimen/FontAwesome.ttf -------------------------------------------------------------------------------- /docs/material-theme/assets/fonts/specimen/FontAwesome.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ucbrise/confluo/e9b8621e99f56b1adf5675c4296c006d89e6e582/docs/material-theme/assets/fonts/specimen/FontAwesome.woff -------------------------------------------------------------------------------- /docs/material-theme/assets/fonts/specimen/FontAwesome.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ucbrise/confluo/e9b8621e99f56b1adf5675c4296c006d89e6e582/docs/material-theme/assets/fonts/specimen/FontAwesome.woff2 -------------------------------------------------------------------------------- /docs/material-theme/assets/fonts/specimen/MaterialIcons-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ucbrise/confluo/e9b8621e99f56b1adf5675c4296c006d89e6e582/docs/material-theme/assets/fonts/specimen/MaterialIcons-Regular.ttf -------------------------------------------------------------------------------- /docs/material-theme/assets/fonts/specimen/MaterialIcons-Regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ucbrise/confluo/e9b8621e99f56b1adf5675c4296c006d89e6e582/docs/material-theme/assets/fonts/specimen/MaterialIcons-Regular.woff -------------------------------------------------------------------------------- /docs/material-theme/assets/fonts/specimen/MaterialIcons-Regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ucbrise/confluo/e9b8621e99f56b1adf5675c4296c006d89e6e582/docs/material-theme/assets/fonts/specimen/MaterialIcons-Regular.woff2 -------------------------------------------------------------------------------- /docs/material-theme/assets/images/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ucbrise/confluo/e9b8621e99f56b1adf5675c4296c006d89e6e582/docs/material-theme/assets/images/favicon.png -------------------------------------------------------------------------------- /docs/material-theme/assets/images/icons/bitbucket.1b09e088.svg: -------------------------------------------------------------------------------- 1 | 3 | 20 | 21 | -------------------------------------------------------------------------------- /docs/material-theme/assets/images/icons/github.f0b8504a.svg: -------------------------------------------------------------------------------- 1 | 3 | 18 | 19 | -------------------------------------------------------------------------------- /docs/material-theme/assets/javascripts/lunr/lunr.jp.js: -------------------------------------------------------------------------------- 1 | !function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(n){if(void 0===n)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===n.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");var i="2"==n.version[0];n.jp=function(){this.pipeline.reset(),this.pipeline.add(n.jp.stopWordFilter,n.jp.stemmer),i?this.tokenizer=n.jp.tokenizer:(n.tokenizer&&(n.tokenizer=n.jp.tokenizer),this.tokenizerFn&&(this.tokenizerFn=n.jp.tokenizer))};var o=new n.TinySegmenter;n.jp.tokenizer=function(e){if(!arguments.length||null==e||null==e)return[];if(Array.isArray(e))return e.map(function(e){return i?new n.Token(e.toLowerCase()):e.toLowerCase()});for(var r=e.toString().toLowerCase().replace(/^\s+/,""),t=r.length-1;0<=t;t--)if(/\S/.test(r.charAt(t))){r=r.substring(0,t+1);break}return o.segment(r).filter(function(e){return!!e}).map(function(e){return i?new n.Token(e):e})},n.jp.stemmer=function(e){return e},n.Pipeline.registerFunction(n.jp.stemmer,"stemmer-jp"),n.jp.wordCharacters="一二三四五六七八九十百千万億兆一-龠々〆ヵヶぁ-んァ-ヴーｱ-ﾝﾞa-zA-Zａ-ｚＡ-Ｚ0-9０-９",n.jp.stopWordFilter=function(e){if(-1===n.jp.stopWordFilter.stopWords.indexOf(i?e.toString():e))return e},n.jp.stopWordFilter=n.generateStopWordFilter("これ それ あれ この その あの ここ そこ あそこ こちら どこ だれ なに なん 何 私 貴方 貴方方 我々 私達 あの人 あのかた 彼女 彼 です あります おります います は が の に を で え から まで より も どの と し それで しかし".split(" ")),n.Pipeline.registerFunction(n.jp.stopWordFilter,"stopWordFilter-jp")}}); -------------------------------------------------------------------------------- /docs/material-theme/assets/javascripts/lunr/lunr.multi.js: -------------------------------------------------------------------------------- 1 | !function(e,i){"function"==typeof define&&define.amd?define(i):"object"==typeof exports?module.exports=i():i()(e.lunr)}(this,function(){return function(o){o.multiLanguage=function(){for(var e=Array.prototype.slice.call(arguments),i=e.join("-"),t="",r=[],n=[],s=0;s 7 |
8 | {{ page.meta.hero }} 9 |
10 | 11 | -------------------------------------------------------------------------------- /docs/material-theme/partials/integrations/analytics.html: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /docs/material-theme/partials/integrations/disqus.html: -------------------------------------------------------------------------------- 1 | {% set disqus = config.extra.disqus %} 2 | {% if page and page.meta and page.meta.disqus is string %} 3 | {% set disqus = page.meta.disqus %} 4 | {% endif %} 5 | {% if not page.is_homepage and disqus %} 6 |

{{ lang.t("meta.comments") }}

7 |
8 | 21 | {% endif %} 22 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language.html: -------------------------------------------------------------------------------- 1 | {% import "partials/language/" + config.theme.language + ".html" as lang %} 2 | {% import "partials/language/en.html" as fallback %} 3 | {% macro t(key) %}{{ { 4 | "direction": config.theme.direction, 5 | "search.language": ( 6 | config.extra.search | default({}) 7 | ).language, 8 | "search.tokenizer": ( 9 | config.extra.search | default({}) 10 | ).tokenizer | default("", true), 11 | }[key] or lang.t(key) or fallback.t(key) }}{% endmacro %} 12 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/ar.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "ar", 3 | "direction": "rtl", 4 | "clipboard.copy": "نسخ إلى الحافظة", 5 | "clipboard.copied": "تم النسخ الى الحافظة", 6 | "edit.link.title": "عدل الصفحة", 7 | "footer.previous": "السابقة", 8 | "footer.next": "التالية", 9 | "meta.comments": "التعليقات", 10 | "meta.source": "المصدر", 11 | "search.language": "", 12 | "search.pipeline.stopwords": false, 13 | "search.pipeline.trimmer": false, 14 | "search.placeholder": "بحث", 15 | "search.result.placeholder": "اكتب لبدء البحث", 16 | "search.result.none": "لا توجد نتائج", 17 | "search.result.one": "نتائج البحث مستند واحد", 18 | "search.result.other": "نتائج البحث # مستندات", 19 | "skip.link.title": "انتقل إلى المحتوى", 20 | "source.link.title": "اذهب إلى المصدر", 21 | "toc.title": "جدول المحتويات" 22 | }[key] }}{% endmacro %} 23 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/ca.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "ca", 3 | "clipboard.copy": "Còpia al porta-retalls", 4 | "clipboard.copied": "Copiat al porta-retalls", 5 | "edit.link.title": "Edita aquesta pàgina", 6 | "footer.previous": "Anterior", 7 | "footer.next": "Següent", 8 | "meta.comments": "Comentaris", 9 | "meta.source": "Codi font", 10 | "search.language": "", 11 | "search.placeholder": "Cerca", 12 | "search.result.placeholder": "Escriu per a començar a cercar", 13 | "search.result.none": "Cap document coincideix", 14 | "search.result.one": "1 document coincident", 15 | "search.result.other": "# documents coincidents", 16 | "skip.link.title": "Salta el contingut", 17 | "source.link.title": "Ves al repositori", 18 | "toc.title": "Taula de continguts" 19 | }[key] }}{% endmacro %} 20 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/cs.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "cs", 3 | "clipboard.copy": "Kopírovat do schránky", 4 | "clipboard.copied": "Zkopírováno do schránky", 5 | "edit.link.title": "Upravit tuto stránku", 6 | "footer.previous": "Předchozí", 7 | "footer.next": "Další", 8 | "meta.comments": "Komentáře", 9 | "meta.source": "Zdroj", 10 | "search.language": "ro", 11 | "search.placeholder": "Hledat", 12 | "search.result.placeholder": "Pište co se má vyhledat", 13 | "search.result.none": "Nenalezeny žádné dokumenty", 14 | "search.result.one": "Nalezený dokument: 1", 15 | "search.result.other": "Nalezené dokumenty: #", 16 | "skip.link.title": "Přeskočit obsah", 17 | "source.link.title": "Přejít do repozitáře", 18 | "toc.title": "Obsah" 19 | }[key] }}{% endmacro %} 20 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/da.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "da", 3 | "clipboard.copy": "Kopiér til udklipsholderen", 4 | "clipboard.copied": "Kopieret til udklipsholderen", 5 | "edit.link.title": "Redigér denne side", 6 | "footer.previous": "Forrige", 7 | "footer.next": "Næste", 8 | "meta.comments": "Kommentarer", 9 | "meta.source": "Kilde", 10 | "search.language": "da", 11 | "search.placeholder": "Søg", 12 | "search.result.placeholder": "Indtask søgeord", 13 | "search.result.none": "Ingen resultater fundet", 14 | "search.result.one": "1 resultat", 15 | "search.result.other": "# resultater", 16 | "skip.link.title": "Gå til indholdet", 17 | "source.link.title": "Åbn arkiv", 18 | "toc.title": "Indholdsfortegnelse" 19 | }[key] }}{% endmacro %} 20 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/de.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "de", 3 | "clipboard.copy": "In Zwischenablage kopieren", 4 | "clipboard.copied": "In Zwischenablage kopiert", 5 | "edit.link.title": "Seite editieren", 6 | "footer.previous": "Zurück", 7 | "footer.next": "Weiter", 8 | "meta.comments": "Kommentare", 9 | "meta.source": "Quellcode", 10 | "search.language": "de", 11 | "search.placeholder": "Suche", 12 | "search.result.placeholder": "Suchbegriff eingeben", 13 | "search.result.none": "Keine Suchergebnisse", 14 | "search.result.one": "1 Suchergebnis", 15 | "search.result.other": "# Suchergebnisse", 16 | "skip.link.title": "Zum Inhalt", 17 | "source.link.title": "Quellcode", 18 | "toc.title": "Inhaltsverzeichnis" 19 | }[key] }}{% endmacro %} 20 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/en.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "en", 3 | "direction": "ltr", 4 | "clipboard.copy": "Copy to clipboard", 5 | "clipboard.copied": "Copied to clipboard", 6 | "edit.link.title": "Edit this page", 7 | "footer.previous": "Previous", 8 | "footer.next": "Next", 9 | "meta.comments": "Comments", 10 | "meta.source": "Source", 11 | "search.language": "en", 12 | "search.pipeline.stopwords": true, 13 | "search.pipeline.trimmer": true, 14 | "search.placeholder": "Search", 15 | "search.result.placeholder": "Type to start searching", 16 | "search.result.none": "No matching documents", 17 | "search.result.one": "1 matching document", 18 | "search.result.other": "# matching documents", 19 | "search.tokenizer": "[\s\-]+", 20 | "skip.link.title": "Skip to content", 21 | "source.link.title": "Go to repository", 22 | "toc.title": "Table of contents" 23 | }[key] }}{% endmacro %} 24 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/es.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "es", 3 | "clipboard.copy": "Copiar al portapapeles", 4 | "clipboard.copied": "Copiado al portapapeles", 5 | "edit.link.title": "Editar esta página", 6 | "footer.previous": "Anterior", 7 | "footer.next": "Siguiente", 8 | "meta.comments": "Comentarios", 9 | "meta.source": "Fuente", 10 | "search.language": "es", 11 | "search.placeholder": "Búsqueda", 12 | "search.result.placeholder": "Teclee para comenzar búsqueda", 13 | "search.result.none": "No se encontraron documentos", 14 | "search.result.one": "1 documento encontrado", 15 | "search.result.other": "# documentos encontrados", 16 | "skip.link.title": "Saltar a contenido", 17 | "source.link.title": "Ir al repositorio", 18 | "toc.title": "Tabla de contenidos" 19 | }[key] }}{% endmacro %} 20 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/fa.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "fa", 3 | "direction": "rtl", 4 | "clipboard.copy": "کپی کردن", 5 | "clipboard.copied": "کپی شد", 6 | "edit.link.title": "این صفحه را ویرایش کنید", 7 | "footer.previous": "قبلی", 8 | "footer.next": "بعدی", 9 | "meta.comments": "نظرات", 10 | "meta.source": "منبع", 11 | "search.language": "", 12 | "search.pipeline.stopwords": false, 13 | "search.pipeline.trimmer": false, 14 | "search.placeholder": "جستجو", 15 | "search.result.placeholder": "برای شروع جستجو تایپ کنید", 16 | "search.result.none": "سندی یافت نشد", 17 | "search.result.one": "1 سند یافت شد", 18 | "search.result.other": "# سند یافت شد", 19 | "skip.link.title": "پرش به محتویات", 20 | "source.link.title": "رفتن به مخزن", 21 | "toc.title": "فهرست موضوعات" 22 | }[key] }}{% endmacro %} 23 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/fi.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "fi", 3 | "clipboard.copy": "Kopioi leikepöydälle", 4 | "clipboard.copied": "Kopioitu leikepöydälle", 5 | "edit.link.title": "Muokkaa tätä sivua", 6 | "footer.previous": "Edellinen", 7 | "footer.next": "Seuraava", 8 | "meta.comments": "Kommentit", 9 | "meta.source": "Lähdekodi", 10 | "search.language": "fi", 11 | "search.placeholder": "Hae", 12 | "search.result.placeholder": "Kirjoita aloittaaksesi haun", 13 | "search.result.none": "Ei täsmääviä dokumentteja", 14 | "search.result.one": "1 täsmäävä dokumentti", 15 | "search.result.other": "# täsmäävää dokumenttia", 16 | "skip.link.title": "Hyppää sisältöön", 17 | "source.link.title": "Mene repositoryyn", 18 | "toc.title": "Sisällysluettelo" 19 | }[key] }}{% endmacro %} 20 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/fr.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "fr", 3 | "clipboard.copy": "Copier dans le presse-papier", 4 | "clipboard.copied": "Copié dans le presse-papier", 5 | "edit.link.title": "Editer cette page", 6 | "footer.previous": "Précédent", 7 | "footer.next": "Suivant", 8 | "meta.comments": "Commentaires", 9 | "meta.source": "Source", 10 | "search.language": "fr", 11 | "search.placeholder": "Rechercher", 12 | "search.result.placeholder": "Taper pour démarrer la recherche", 13 | "search.result.none": "Aucun document trouvé", 14 | "search.result.one": "1 document trouvé", 15 | "search.result.other": "# documents trouvés", 16 | "source.link.title": "Aller au dépôt", 17 | "toc.title": "Table des matières" 18 | }[key] }}{% endmacro %} 19 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/gl.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "gl", 3 | "clipboard.copy": "Copiar no cortapapeis", 4 | "clipboard.copied": "Copiado no cortapapeis", 5 | "edit.link.title": "Editar esta páxina", 6 | "footer.previous": "Anterior", 7 | "footer.next": "Seguinte", 8 | "meta.comments": "Comentarios", 9 | "meta.source": "Fonte", 10 | "search.language": "es", 11 | "search.placeholder": "Busca", 12 | "search.result.placeholder": "Insira un termo", 13 | "search.result.none": "Sen resultados", 14 | "search.result.one": "1 resultado atopado", 15 | "search.result.other": "# resultados atopados", 16 | "skip.link.title": "Ir ao contido", 17 | "source.link.title": "Ir ao repositorio", 18 | "toc.title": "Táboa de contidos" 19 | }[key] }}{% endmacro %} 20 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/he.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "he", 3 | "direction": "rtl", 4 | "clipboard.copy": "העתק ללוח", 5 | "clipboard.copied": "הועתק ללוח", 6 | "edit.link.title": "ערוך דף זה", 7 | "footer.previous": "קודם", 8 | "footer.next": "הַבָּא", 9 | "meta.comments": "הערות", 10 | "meta.source": "מָקוֹר", 11 | "search.language": "", 12 | "search.pipeline.stopwords": false, 13 | "search.pipeline.trimmer": false, 14 | "search.placeholder": "לחפש", 15 | "search.result.placeholder": "הקלד כדי להתחיל לחפש", 16 | "search.result.none": "אין מסמכים תואמים", 17 | "search.result.one": "1 מסמך תואם", 18 | "search.result.other": "# מסמך תואם", 19 | "skip.link.title": "דלג לתוכן", 20 | "source.link.title": "עבור אל מאגר", 21 | "toc.title": "תוכן העניינים" 22 | }[key] }}{% endmacro %} 23 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/hi.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "hi", 3 | "clipboard.copy": "क्लिपबोर्ड पर कॉपी करें", 4 | "clipboard.copied": "क्लिपबोर्ड पर कॉपी कर दिया गया", 5 | "edit.link.title": "इस पृष्ठ को संपादित करें", 6 | "footer.previous": "पिछला", 7 | "footer.next": "आगामी", 8 | "meta.comments": "टिप्पणियाँ", 9 | "meta.source": "स्रोत", 10 | "search.language": "", 11 | "search.pipeline.stopwords": false, 12 | "search.pipeline.trimmer": false, 13 | "search.placeholder": "खोज", 14 | "search.result.placeholder": "खोज शुरू करने के लिए टाइप करें", 15 | "search.result.none": "कोई मिलान डॉक्यूमेंट नहीं", 16 | "search.result.one": "1 मिलान डॉक्यूमेंट", 17 | "search.result.other": "# मिलान डाक्यूमेंट्स", 18 | "skip.link.title": "विषय पर बढ़ें", 19 | "source.link.title": "रिपॉजिटरी पर जाएं", 20 | "toc.title": "विषय - सूची" 21 | }[key] }}{% endmacro %} 22 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/hr.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "hr", 3 | "clipboard.copy": "Kopirajte u međuspremnik", 4 | "clipboard.copied": "Kopirano u međuspremnik", 5 | "edit.link.title": "Uredi stranicu", 6 | "footer.previous": "Prethodno", 7 | "footer.next": "Sljedeće", 8 | "meta.comments": "Komentari", 9 | "meta.source": "Izvor", 10 | "search.language": "", 11 | "search.pipeline.stopwords": false, 12 | "search.pipeline.trimmer": false, 13 | "search.placeholder": "Pretraživanje", 14 | "search.result.placeholder": "Unesite pojam pretraživanja", 15 | "search.result.none": "Ništa nije pronađeno", 16 | "search.result.one": "1 rezultat pretraživanja", 17 | "search.result.other": "# rezultata pretraživanja", 18 | "skip.link.title": "Preskočite na sadržaj", 19 | "source.link.title": "Idite u repozitorij", 20 | "toc.title": "Sadržaj" 21 | }[key] }}{% endmacro %} 22 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/hu.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "hu", 3 | "clipboard.copy": "Másolás vágólapra", 4 | "clipboard.copied": "Vágólapra másolva", 5 | "edit.link.title": "Oldal szerkesztése", 6 | "footer.previous": "Előző", 7 | "footer.next": "Következő", 8 | "meta.comments": "Hozzászólások", 9 | "meta.source": "Forrás", 10 | "search.language": "hu", 11 | "search.placeholder": "Keresés", 12 | "search.result.placeholder": "Kereséshez írj ide valamit", 13 | "search.result.none": "Nincs találat", 14 | "search.result.one": "1 egyező dokumentum", 15 | "search.result.other": "# egyező dokumentum", 16 | "skip.link.title": "Kihagyás", 17 | "source.link.title": "Főoldalra ugrás", 18 | "toc.title": "Tartalomjegyzék" 19 | }[key] }}{% endmacro %} 20 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/id.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "id", 3 | "clipboard.copy": "Salin ke memori", 4 | "clipboard.copied": "Tersalin ke memori", 5 | "edit.link.title": "Ubah halaman ini", 6 | "footer.previous": "Sebelumnya", 7 | "footer.next": "Selanjutnya", 8 | "meta.comments": "Komentar", 9 | "meta.source": "Sumber", 10 | "search.language": "", 11 | "search.pipeline.stopwords": false, 12 | "search.pipeline.trimmer": false, 13 | "search.placeholder": "Cari", 14 | "search.result.placeholder": "Ketik untuk mulai pencarian", 15 | "search.result.none": "Tidak ada dokumen yang sesuai", 16 | "search.result.one": "1 dokumen ditemukan", 17 | "search.result.other": "# dokumen ditemukan", 18 | "skip.link.title": "Lewati ke isi", 19 | "source.link.title": "Menuju repositori", 20 | "toc.title": "Daftar isi" 21 | }[key] }}{% endmacro %} 22 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/it.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "it", 3 | "clipboard.copy": "Copia", 4 | "clipboard.copied": "Copiato", 5 | "edit.link.title": "Modifica", 6 | "footer.previous": "Precedente", 7 | "footer.next": "Prossimo", 8 | "meta.comments": "Commenti", 9 | "meta.source": "Sorgente", 10 | "search.language": "it", 11 | "search.placeholder": "Cerca", 12 | "search.result.placeholder": "Scrivi per iniziare a cercare", 13 | "search.result.none": "Nessun documento trovato", 14 | "search.result.one": "1 documento trovato", 15 | "search.result.other": "# documenti trovati", 16 | "skip.link.title": "Vai al contenuto", 17 | "source.link.title": "Apri repository", 18 | "toc.title": "Indice" 19 | }[key] }}{% endmacro %} 20 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/ja.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "ja", 3 | "clipboard.copy": "クリップボードへコピー", 4 | "clipboard.copied": "コピーしました", 5 | "edit.link.title": "編集", 6 | "footer.previous": "前", 7 | "footer.next": "次", 8 | "meta.comments": "コメント", 9 | "meta.source": "ソース", 10 | "search.language": "jp", 11 | "search.placeholder": "検索", 12 | "search.result.placeholder": "検索キーワードを入力してください", 13 | "search.result.none": "何も見つかりませんでした", 14 | "search.result.one": "1件見つかりました", 15 | "search.result.other": "#件見つかりました", 16 | "search.tokenizer": "[\s\-　、。，．]+", 17 | "source.link.title": "リポジトリへ", 18 | "toc.title": "目次" 19 | }[key] }}{% endmacro %} 20 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/kr.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "kr", 3 | "clipboard.copy": "클립보드로 복사", 4 | "clipboard.copied": "클립보드에 복사됨", 5 | "edit.link.title": "이 페이지를 편집", 6 | "footer.previous": "이전", 7 | "footer.next": "다음", 8 | "meta.comments": "댓글", 9 | "meta.source": "출처", 10 | "search.language": "jp", 11 | "search.placeholder": "검색", 12 | "search.result.placeholder": "검색어를 입력하세요", 13 | "search.result.none": "검색어와 일치하는 문서가 없습니다", 14 | "search.result.one": "1개의 일치하는 문서", 15 | "search.result.other": "#개의 일치하는 문서", 16 | "source.link.title": "저장소로 이동", 17 | "toc.title": "목차" 18 | }[key] }}{% endmacro %} 19 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/nl.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "nl", 3 | "clipboard.copy": "Kopiëren naar klembord", 4 | "clipboard.copied": "Gekopieerd naar klembord", 5 | "edit.link.title": "Wijzig deze pagina", 6 | "footer.previous": "Vorige", 7 | "footer.next": "Volgende", 8 | "meta.comments": "Reacties", 9 | "meta.source": "Bron", 10 | "search.language": "du", 11 | "search.placeholder": "Zoeken", 12 | "search.result.placeholder": "Typ om te beginnen met zoeken", 13 | "search.result.none": "Geen overeenkomende documenten", 14 | "search.result.one": "1 overeenkomende document", 15 | "search.result.other": "# overeenkomende documenten", 16 | "skip.link.title": "Ga naar inhoud", 17 | "source.link.title": "Ga naar repository", 18 | "toc.title": "Inhoudsopgave" 19 | }[key] }}{% endmacro %} 20 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/no.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "no", 3 | "clipboard.copy": "Kopier til utklippstavlen", 4 | "clipboard.copied": "Kopiert til utklippstavlen", 5 | "edit.link.title": "Rediger denne siden", 6 | "footer.previous": "Forrige", 7 | "footer.next": "Neste", 8 | "meta.comments": "Kommentarer", 9 | "meta.source": "Kilde", 10 | "search.language": "no", 11 | "search.placeholder": "Søk", 12 | "search.result.placeholder": "Skriv søkeord", 13 | "search.result.none": "Ingen treff", 14 | "search.result.one": "1 treff", 15 | "search.result.other": "# treff", 16 | "skip.link.title": "Gå til innhold", 17 | "source.link.title": "Gå til kilde", 18 | "toc.title": "Innholdsfortegnelse" 19 | }[key] }}{% endmacro %} 20 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/pl.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "pl", 3 | "clipboard.copy": "Kopiuj do schowka", 4 | "clipboard.copied": "Skopiowane", 5 | "edit.link.title": "Edytuj tę stronę", 6 | "footer.previous": "Poprzednia strona", 7 | "footer.next": "Następna strona", 8 | "meta.comments": "Komentarze", 9 | "meta.source": "Kod źródłowy", 10 | "search.language": "", 11 | "search.pipeline.stopwords": false, 12 | "search.pipeline.trimmer": false, 13 | "search.placeholder": "Szukaj", 14 | "search.result.placeholder": "Zacznij pisać, aby szukać", 15 | "search.result.none": "Brak wyników wyszukiwania", 16 | "search.result.one": "Wyniki wyszukiwania: 1", 17 | "search.result.other": "Wyniki wyszukiwania: #", 18 | "skip.link.title": "Przejdź do treści", 19 | "source.link.title": "Idź do repozytorium", 20 | "toc.title": "Spis treści" 21 | }[key] }}{% endmacro %} 22 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/pt.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "pt", 3 | "clipboard.copy": "Copiar para área de transferência", 4 | "clipboard.copied": "Copiado para área de transferência", 5 | "edit.link.title": "Editar esta página", 6 | "footer.previous": "Anterior", 7 | "footer.next": "Próximo", 8 | "meta.comments": "Comentários", 9 | "meta.source": "Fonte", 10 | "search.language": "pt", 11 | "search.placeholder": "Buscar", 12 | "search.result.placeholder": "Digite para iniciar a busca", 13 | "search.result.none": "Nenhum resultado encontrado", 14 | "search.result.one": "1 resultado encontrado", 15 | "search.result.other": "# resultados encontrados", 16 | "skip.link.title": "Ir para o conteúdo", 17 | "source.link.title": "Ir ao repositório", 18 | "toc.title": "Índice" 19 | }[key] }}{% endmacro %} 20 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/ru.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "ru", 3 | "clipboard.copy": "Копировать в буфер", 4 | "clipboard.copied": "Скопировано в буфер", 5 | "edit.link.title": "Редактировать страницу", 6 | "footer.previous": "Назад", 7 | "footer.next": "Вперед", 8 | "meta.comments": "Комментарии", 9 | "meta.source": "Исходный код", 10 | "search.language": "ru", 11 | "search.placeholder": "Поиск", 12 | "search.result.placeholder": "Начните печатать для поиска", 13 | "search.result.none": "Совпадений не найдено", 14 | "search.result.one": "Найдено 1 совпадение", 15 | "search.result.other": "Найдено # совпадений", 16 | "skip.link.title": "Перейти к содержанию", 17 | "source.link.title": "Перейти к репозиторию", 18 | "toc.title": "Содержание" 19 | }[key] }}{% endmacro %} 20 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/sh.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "sh", 3 | "clipboard.copy": "Kopiraj u klipbord", 4 | "clipboard.copied": "Iskopirano u klipbord", 5 | "edit.link.title": "Uredi stranicu", 6 | "footer.previous": "Prethodno", 7 | "footer.next": "Sledeće", 8 | "meta.comments": "Komentari", 9 | "meta.source": "Izvor", 10 | "search.language": "ro", 11 | "search.placeholder": "Pretraga", 12 | "search.result.placeholder": "Unesite pojam pretrage", 13 | "search.result.none": "Ništa nije pronađeno", 14 | "search.result.one": "1 rezultat pretrage", 15 | "search.result.other": "# rezultata pretrage", 16 | "skip.link.title": "Idi na tekst", 17 | "source.link.title": "Idi u repozitorijum", 18 | "toc.title": "Sadržaj" 19 | }[key] }}{% endmacro %} 20 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/sk.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "sk", 3 | "clipboard.copy": "Kopírovať do schránky", 4 | "clipboard.copied": "Skopírované do schránky", 5 | "edit.link.title": "Upraviť túto stránku", 6 | "footer.previous": "Späť", 7 | "footer.next": "Ďalej", 8 | "meta.comments": "Komentáre", 9 | "meta.source": "Zdroj", 10 | "search.language": "sk", 11 | "search.placeholder": "Hľadať", 12 | "search.result.placeholder": "Pre vyhľadávanie začni písať", 13 | "search.result.none": "Žiadne vyhovujúce dokumenty", 14 | "search.result.one": "Vyhovujúci dokument: 1", 15 | "search.result.other": "Vyhovujúce dokumenty: #", 16 | "skip.link.title": "Preskočiť na obsah", 17 | "source.link.title": "Zobraziť repozitár", 18 | "toc.title": "Obsah" 19 | }[key] }}{% endmacro %} 20 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/sr.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "sr", 3 | "clipboard.copy": "Копирај у клипборд", 4 | "clipboard.copied": "Ископирано у клипборд", 5 | "edit.link.title": "Уреди страницу", 6 | "footer.previous": "Претходно", 7 | "footer.next": "Следеће", 8 | "meta.comments": "Коментари", 9 | "meta.source": "Извор", 10 | "search.language": "", 11 | "search.pipeline.stopwords": false, 12 | "search.pipeline.trimmer": false, 13 | "search.placeholder": "Претрага", 14 | "search.result.placeholder": "Унесите појам претраге", 15 | "search.result.none": "Нису пронађени документи", 16 | "search.result.one": "1 резултат претраге", 17 | "search.result.other": "# резултата претраге", 18 | "skip.link.title": "Иди на текст", 19 | "source.link.title": "Иди у репозиторијум", 20 | "toc.title": "Садржај" 21 | }[key] }}{% endmacro %} 22 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/sv.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "sv", 3 | "clipboard.copy": "Kopiera till urklipp", 4 | "clipboard.copied": "Kopierat till urklipp", 5 | "edit.link.title": "Redigera sidan", 6 | "footer.previous": "Föregående", 7 | "footer.next": "Nästa", 8 | "meta.comments": "Kommentarer", 9 | "meta.source": "Källa", 10 | "search.language": "sv", 11 | "search.placeholder": "Sök", 12 | "search.result.placeholder": "Skriv sökord", 13 | "search.result.none": "Inga sökresultat", 14 | "search.result.one": "1 sökresultat", 15 | "search.result.other": "# sökresultat", 16 | "skip.link.title": "Gå till innehållet", 17 | "source.link.title": "Gå till datakatalog", 18 | "toc.title": "Innehållsförteckning" 19 | }[key] }}{% endmacro %} 20 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/tr.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "tr", 3 | "clipboard.copy": "Kopyala", 4 | "clipboard.copied": "Kopyalandı", 5 | "edit.link.title": "Düzenle", 6 | "footer.previous": "Önceki", 7 | "footer.next": "Sonraki", 8 | "meta.comments": "Yorumlar", 9 | "meta.source": "Kaynak", 10 | "search.language": "tr", 11 | "search.placeholder": "Ara", 12 | "search.result.placeholder": "Aramaya başlamak için yazın", 13 | "search.result.none": "Eşleşen doküman bulunamadı", 14 | "search.result.one": "1 doküman bulundu", 15 | "search.result.other": "# doküman bulundu", 16 | "source.link.title": "Depoya git", 17 | "toc.title": "İçindekiler" 18 | }[key] }}{% endmacro %} 19 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/uk.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "uk", 3 | "clipboard.copy": "Скопіювати в буфер", 4 | "clipboard.copied": "Скопійовано в буфер", 5 | "edit.link.title": "Редагувати сторінку", 6 | "footer.previous": "Назад", 7 | "footer.next": "Вперед", 8 | "meta.comments": "Коментарі", 9 | "meta.source": "Вихідний код", 10 | "search.language": "ru", 11 | "search.placeholder": "Пошук", 12 | "search.result.placeholder": "Розпочніть писати для пошуку", 13 | "search.result.none": "Збігів не знайдено", 14 | "search.result.one": "Знайдено 1 збіг", 15 | "search.result.other": "Знайдено # збігів", 16 | "skip.link.title": "Перейти до змісту", 17 | "source.link.title": "Перейти до репозиторію", 18 | "toc.title": "Зміст" 19 | }[key] }}{% endmacro %} 20 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/vi.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "vi", 3 | "clipboard.copy": "Sao chép vào bộ nhớ", 4 | "clipboard.copied": "Sao chép xong", 5 | "edit.link.title": "Chỉnh sửa", 6 | "footer.previous": "Trước", 7 | "footer.next": "Sau", 8 | "meta.comments": "Bình luận", 9 | "meta.source": "Mã nguồn", 10 | "search.placeholder": "Tìm kiếm", 11 | "search.result.placeholder": "Nhập để bắt đầu tìm kiếm", 12 | "search.result.none": "Không tìm thấy tài liệu liên quan", 13 | "search.result.one": "1 tài liệu liên quan", 14 | "search.result.other": "# tài liệu liên quan", 15 | "skip.link.title": "Vào thẳng nội dung", 16 | "source.link.title": "Đến kho lưu trữ mã nguồn", 17 | "toc.title": "Mục lục" 18 | }[key] }}{% endmacro %} 19 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/zh-Hant.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "zh-Hant", 3 | "clipboard.copy": "拷貝", 4 | "clipboard.copied": "已拷貝", 5 | "edit.link.title": "編輯此頁", 6 | "footer.previous": "上一頁", 7 | "footer.next": "下一頁", 8 | "meta.comments": "評論", 9 | "meta.source": "來源", 10 | "search.language": "jp", 11 | "search.placeholder": "搜尋", 12 | "search.result.placeholder": "鍵入以開始檢索", 13 | "search.result.none": "沒有找到符合條件的結果", 14 | "search.result.one": "找到 1 个符合條件的結果", 15 | "search.result.other": "# 個符合條件的結果", 16 | "search.tokenizer": "[\，\。]+", 17 | "skip.link.title": "跳轉至", 18 | "source.link.title": "前往 Github 倉庫", 19 | "toc.title": "目錄" 20 | }[key] }}{% endmacro %} 21 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/zh-TW.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "zh-Hant", 3 | "clipboard.copy": "複製", 4 | "clipboard.copied": "已複製", 5 | "edit.link.title": "編輯此頁", 6 | "footer.previous": "上一頁", 7 | "footer.next": "下一頁", 8 | "meta.comments": "留言", 9 | "meta.source": "來源", 10 | "search.language": "jp", 11 | "search.placeholder": "搜尋", 12 | "search.result.placeholder": "打字進行搜尋", 13 | "search.result.none": "沒有符合的項目", 14 | "search.result.one": "找到 1 個符合的項目", 15 | "search.result.other": "找到 # 個符合的項目", 16 | "search.tokenizer": "[\，\。]+", 17 | "skip.link.title": "跳轉到", 18 | "source.link.title": "前往倉庫", 19 | "toc.title": "本頁目錄" 20 | }[key] }}{% endmacro %} 21 | -------------------------------------------------------------------------------- /docs/material-theme/partials/language/zh.html: -------------------------------------------------------------------------------- 1 | {% macro t(key) %}{{ { 2 | "language": "zh", 3 | "clipboard.copy": "复制", 4 | "clipboard.copied": "已复制", 5 | "edit.link.title": "编辑此页", 6 | "footer.previous": "后退", 7 | "footer.next": "前进", 8 | "meta.comments": "评论", 9 | "meta.source": "来源", 10 | "search.language": "jp", 11 | "search.placeholder": "搜索", 12 | "search.result.placeholder": "键入以开始搜索", 13 | "search.result.none": "没有找到符合条件的结果", 14 | "search.result.one": "找到 1 个符合条件的结果", 15 | "search.result.other": "# 个符合条件的结果", 16 | "search.tokenizer": "[\，\。]+", 17 | "skip.link.title": "跳转至", 18 | "source.link.title": "前往 Github 仓库", 19 | "toc.title": "目录" 20 | }[key] }}{% endmacro %} 21 | -------------------------------------------------------------------------------- /docs/material-theme/partials/nav.html: -------------------------------------------------------------------------------- 1 | 25 | -------------------------------------------------------------------------------- /docs/material-theme/partials/palette.html: -------------------------------------------------------------------------------- 1 | {% macro primary(key) %}{{ { 2 | "red": "#ef5350", 3 | "pink": "#e91e63", 4 | "purple": "#ab47bc", 5 | "deep-purple": "#7e57c2", 6 | "indigo": "#3f51b5", 7 | "blue": "#2196f3", 8 | "light-blue": "#03a9f4", 9 | "cyan": "#00bcd4", 10 | "teal": "#009688", 11 | "green": "#4caf50", 12 | "light-green": "#7cb342", 13 | "lime": "#c0ca33", 14 | "yellow": "#f9a825", 15 | "amber": "#ffa000", 16 | "orange": "#fb8c00", 17 | "deep-orange": "#ff7043", 18 | "brown": "#795548", 19 | "grey": "#757575", 20 | "blue-grey": "#546e7a" 21 | }[key] }}{% endmacro %} 22 | {% macro accent(key) %}{{ { 23 | "red": "#ff1744", 24 | "pink": "#f50057", 25 | "purple": "#e040fb", 26 | "deep-purple": "#7c4dff", 27 | "indigo": "#536dfe", 28 | "blue": "#448aff", 29 | "light-blue": "#0091ea", 30 | "cyan": "#00b8d4", 31 | "teal": "#00bfa5", 32 | "green": "#00c853", 33 | "light-green": "#64dd17", 34 | "lime": "#aeea00", 35 | "yellow": "#ffd600", 36 | "amber": "#ffab00", 37 | "orange": "#ff9100", 38 | "deep-orange": "#ff6e40" 39 | }[key] }}{% endmacro %} 40 | -------------------------------------------------------------------------------- /docs/material-theme/partials/search.html: -------------------------------------------------------------------------------- 1 | {% import "partials/language.html" as lang with context %} 2 | 24 | -------------------------------------------------------------------------------- /docs/material-theme/partials/social.html: -------------------------------------------------------------------------------- 1 | {% if config.extra.social %} 2 |
3 | 4 | {% for social in config.extra.social %} 5 | 6 | {% endfor %} 7 |
8 | {% endif %} 9 | -------------------------------------------------------------------------------- /docs/material-theme/partials/source.html: -------------------------------------------------------------------------------- 1 | {% import "partials/language.html" as lang with context %} 2 | {% set platform = config.extra.repo_icon or config.repo_url %} 3 | {% if "github" in platform %} 4 | {% set repo_type = "github" %} 5 | {% elif "gitlab" in platform %} 6 | {% set repo_type = "gitlab" %} 7 | {% elif "bitbucket" in platform %} 8 | {% set repo_type = "bitbucket" %} 9 | {% else %} 10 | {% set repo_type = "" %} 11 | {% endif %} 12 | {% block repo %} 13 | 14 | {% if repo_type %} 15 |
16 | 17 | 18 | 19 |
20 | {% endif %} 21 |
22 | {{ config.repo_name }} 23 |
24 | 25 | {% endblock %} 26 | -------------------------------------------------------------------------------- /docs/material-theme/partials/tabs-item.html: -------------------------------------------------------------------------------- 1 | {% if nav_item.is_homepage %} 2 |
  • 3 | {% if not page.ancestors | length and nav | selectattr("url", page.url) %} 4 | 5 | {{ nav_item.title }} 6 | 7 | {% else %} 8 | 9 | {{ nav_item.title }} 10 | 11 | {% endif %} 12 |
    • 13 | {% elif nav_item.children and nav_item.children | length > 0 %} 14 | {% set title = title | default(nav_item.title) %} 15 | {% if (nav_item.children | first).children %} 16 | {% set nav_item = nav_item.children | first %} 17 | {% include "partials/tabs-item.html" %} 18 | {% else %} 19 |
  • 20 | {% if nav_item.active %} 21 | 22 | {{ title }} 23 | 24 | {% else %} 25 | 26 | {{ title }} 27 | 28 | {% endif %} 29 |
    • 30 | {% endif %} 31 | {% endif %} 32 | -------------------------------------------------------------------------------- /docs/material-theme/partials/tabs.html: -------------------------------------------------------------------------------- 1 | {% set class = "md-tabs" %} 2 | {% if page.ancestors | length > 0 %} 3 | {% set class = "md-tabs md-tabs--active" %} 4 | {% endif %} 5 | 14 | -------------------------------------------------------------------------------- /docs/material-theme/partials/toc-item.html: -------------------------------------------------------------------------------- 1 |
  • 2 | 3 | {{ toc_item.title }} 4 | 5 | {% if toc_item.children %} 6 | 13 | {% endif %} 14 |
    • 15 | -------------------------------------------------------------------------------- /docs/material-theme/partials/toc.html: -------------------------------------------------------------------------------- 1 | {% import "partials/language.html" as lang with context %} 2 | 34 | -------------------------------------------------------------------------------- /docs/pyclient/source/index.rst: -------------------------------------------------------------------------------- 1 | .. Confluo documentation master file, created by 2 | sphinx-quickstart on Sun Mar 11 16:38:25 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to Confluo's documentation! 7 | =================================== 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :caption: Contents: 12 | 13 | 14 | 15 | Indices and tables 16 | ================== 17 | 18 | * :ref:`genindex` 19 | * :ref:`modindex` 20 | * :ref:`search` 21 | -------------------------------------------------------------------------------- /docs/pyclient/source/modules.rst: -------------------------------------------------------------------------------- 1 | rpc 2 | === 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | rpc 8 | -------------------------------------------------------------------------------- /docs/src/api.md: -------------------------------------------------------------------------------- 1 | # Confluo API 2 | 3 | * [C++ API](cpp_api.md) 4 | * [Client APIs](client_api.md) 5 | - [C++ Client API](cpp_client_api.md) 6 | - [Python Client API](python_client_api.md) 7 | - [Java Client API](java_client_api.md) -------------------------------------------------------------------------------- /docs/src/applications.md: -------------------------------------------------------------------------------- 1 | # Case Studies 2 | 3 | We have evaluated Confluo for a wide range of applications. Here, we describe the design and implementation for three such applications: 4 | 5 | * [A network monitoring and diagnosis tool](network_monitoring.md) 6 | * [A time-series database](timeseries_db.md) 7 | * [A pub-sub system](pub_sub.md) 8 | 9 | We also evaluate each of these applications against state-of-the art approaches in their respective domains. -------------------------------------------------------------------------------- /docs/src/client_api.md: -------------------------------------------------------------------------------- 1 | # Client API 2 | 3 | * [C++ Client API](cpp_client_api.md) 4 | * [Python Client API](python_client_api.md) 5 | * [Java Client API](java_client_api.md) -------------------------------------------------------------------------------- /docs/src/contact.md: -------------------------------------------------------------------------------- 1 | # People 2 | 3 | 4 |

    [**Anurag Khandelwal**](http://people.eecs.berkeley.edu/~anuragk/)
    5 | Graduate Student at UC Berkeley
    6 | anuragk AT berkeley.edu

    7 |
    8 |
    9 | 10 | 11 |

    [**Rachit Agarwal**](http://www.cs.cornell.edu/~ragarwal/)
    12 | Assistant Professor at Cornell University
    13 | ragarwal AT cs.cornell.edu

    14 |
    15 |
    16 | 17 | 18 |

    [**Ion Stoica**](https://people.eecs.berkeley.edu/~istoica/)
    19 | Professor at UC Berkeley
    20 | istoica AT berkeley.edu

    21 |
    22 |
    23 | 24 | 25 |

    Confluo also benefits from contributions from amazing current and past 26 | undergraduate students at UC Berkeley:

    27 | 28 | * [Ujval Misra](http://ujvl.github.io) 29 | * Neil Giridharan 30 | * [Sritej Attaluri](http://sritej.com) 31 | -------------------------------------------------------------------------------- /docs/src/contributing.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Please contribute by creating 4 | [pull-requests](https://github.com/ucbrise/confluo/pulls) and filing 5 | [issues](https://github.com/ucbrise/confluo/issues) on our GitHub repository. 6 | 7 | When contributing to this repository, please first discuss the change you wish 8 | to make via issue, email, or any other method with the owners of this 9 | repository before making a change. 10 | 11 | Please note we have a 12 | [code of conduct](https://github.com/ucbrise/confluo/blob/single-machine/CODE_OF_CONDUCT.md), 13 | please follow it in all your interactions with the project. 14 | 15 | ## Pull Request Process 16 | 17 | 1. Ensure any install or build dependencies are removed before the end of the layer when doing a 18 | build. 19 | 2. Update the documentation under [docs/](./docs/) with details of changes to the interface, this 20 | includes new environment variables, exposed ports, useful file locations and code examples. 21 | 3. Increase the version numbers in the project's root CMakeLists.txt to the new version that this 22 | Pull Request would represent. The versioning scheme we use is [SemVer](http://semver.org/). 23 | 4. You may merge the Pull Request in once you have the sign-off of two other developers, or if you 24 | do not have permission to do that, you may request the second reviewer to merge it for you. 25 | 26 | -------------------------------------------------------------------------------- /docs/src/cpp_api.md: -------------------------------------------------------------------------------- 1 | # C++ API Documentation 2 | 3 | This page will be replaced by API documentation generated by Doxygen. 4 | -------------------------------------------------------------------------------- /docs/src/cpp_client_api.md: -------------------------------------------------------------------------------- 1 | # C++ Client API Documentation 2 | 3 | This page will be replaced by API documentation generated by Doxygen. 4 | -------------------------------------------------------------------------------- /docs/src/img/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ucbrise/confluo/e9b8621e99f56b1adf5675c4296c006d89e6e582/docs/src/img/logo.png -------------------------------------------------------------------------------- /docs/src/img/pub_sub_perf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ucbrise/confluo/e9b8621e99f56b1adf5675c4296c006d89e6e582/docs/src/img/pub_sub_perf.png -------------------------------------------------------------------------------- /docs/src/img/tsdb_perf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ucbrise/confluo/e9b8621e99f56b1adf5675c4296c006d89e6e582/docs/src/img/tsdb_perf.png -------------------------------------------------------------------------------- /docs/src/index.md: -------------------------------------------------------------------------------- 1 | # Confluo 2 | 3 | ## Overview 4 | 5 | Confluo is a system for real-time monitoring and analysis of data, that supports: 6 | 7 | * high-throughput concurrent writes of millions of data points from multiple data streams; 8 | * online queries at millisecond timescale; and 9 | * ad-hoc queries using minimal CPU resources. 10 | 11 | ## User Guide 12 | 13 | * [Quick Start](quick_start.md) 14 | * [Installation](install.md) 15 | * [Modes of Operation](modes_of_operation.md) 16 | * [Data Storage](loading_data.md) 17 | - [Type System](type_system.md) 18 | * [Querying Data](queries.md) 19 | - [Online Queries](online_queries.md) 20 | - [Offline Queries](offline_queries.md) 21 | * [Archiving Data](data_archival.md) 22 | 23 | ## API Docs 24 | 25 | * [C++ API](cpp_api.md) 26 | * [Client APIs](client_api.md) 27 | - [C++ Client API](cpp_client_api.md) 28 | - [Python Client API](python_client_api.md) 29 | - [Java Client API](java_client_api.md) -------------------------------------------------------------------------------- /docs/src/java_client_api.md: -------------------------------------------------------------------------------- 1 | # Java Client API Documentation 2 | 3 | This page will be replaced by API documentation generated by javadocs. 4 | -------------------------------------------------------------------------------- /docs/src/modes_of_operation.md: -------------------------------------------------------------------------------- 1 | # Modes of Operation 2 | 3 | Confluo can be used in two modes -- embedded and stand-alone. 4 | 5 | ## Embedded Mode 6 | 7 | In the _embedded mode_, Confluo is used as a header-only library in C++, allowing Confluo 8 | to use the same address-space as the application process. This enables ultra low-latency 9 | writes and queries, but only supports applications written in C++. 10 | 11 | ## Stand-alone Mode 12 | 13 | Confluo also supports a _stand-alone mode_, where Confluo runs as a daemon 14 | server process and allows clients to communicate with it using 15 | [Apache Thrift](https://thrift.apache.org) protocol. Operations now incur higher latencies 16 | (due to serialization/deserialization overheads), but can now operate over the network, 17 | and allows Confluo to store data from applications written in different languages. 18 | 19 | ## More on Usage 20 | 21 | Read more on how you can perform different operations with the two modes of operation: 22 | 23 | * [Data Storage and Loading Data](loading_data.md) 24 | * [Querying Data](queries.md) 25 | - [Online Queries](online_queries.md) 26 | - [Offline Queries](offline_queries.md) -------------------------------------------------------------------------------- /docs/src/network_monitoring.md: -------------------------------------------------------------------------------- 1 | # Network Monitoring and Diagnosis 2 | 3 | Network monitoring and diagnosis is an increasingly challenging task for network 4 | operators. Integrating these functionalities within the network stack at the 5 | end-hosts allows efficiently using end-host programmability and resources with 6 | minimal overheads (end-host stack process the incoming packets anyway). However, 7 | achieving this requires tools that support highly concurrent per-packet capture 8 | at line-rate, support for online queries (for monitoring purposes), and offline 9 | queries (for diagnosis purposes). Confluo interface is a natural fit for 10 | building such a tool -- flows, packets headers and header fields at an end-host 11 | map perfectly to Confluo streams, records and attributes. 12 | 13 | For details on the design, implementation and evaluation of the network monitoring 14 | and diagnosis tool, please see our [NSDI paper](https://people.eecs.berkeley.edu/~anuragk/papers/confluo.pdf). -------------------------------------------------------------------------------- /docs/src/python_client_api.md: -------------------------------------------------------------------------------- 1 | # Python Client API Documentation 2 | 3 | This page will be replaced by API documentation generated by Sphinx. 4 | -------------------------------------------------------------------------------- /docs/src/queries.md: -------------------------------------------------------------------------------- 1 | # Querying Data 2 | 3 | Queries in Confluo can either be _online_ or _offline_. Online queries are executed as new data records 4 | are written to an Atomic MultiLog, while offline queries are evaluated on already written records. 5 | In essence, online queries are similar to continuous queries databases. 6 | 7 | In order to support online and offline queries, Confluo makes use of indexes, filters, aggregates 8 | and triggers. The interface for adding these elements to an Atomic MultiLog was described in the 9 | guide on [Data Storage and Loading](loading_data.md). 10 | 11 | To see how Confluo supports online and offline queries, see the individual guides at: 12 | 13 | * [Online Queries](online_queries.md) 14 | * [Offline Queries](offline_queries.md) 15 | -------------------------------------------------------------------------------- /docs/src/research.md: -------------------------------------------------------------------------------- 1 | # Research Papers 2 | 3 | [Confluo: Distributed Monitoring and Diagnosis Stack for High-speed Networks](https://people.eecs.berkeley.edu/~anuragk/papers/confluo.pdf)
    4 | Anurag Khandelwal, Rachit Agarwal, Ion Stoica
    5 | NSDI, Boston, MA, Feb 2019 6 | -------------------------------------------------------------------------------- /ec2/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | curdir="`dirname "$0"`" 4 | curdir="`cd "$curdir"; pwd`" 5 | 6 | sudo yum -y update 7 | sudo yum -y groupinstall "Development Tools" 8 | sudo yum -y install gcc-c++ boost-devel java-1.7.0-openjdk* ant 9 | 10 | wget https://cmake.org/files/v3.9/cmake-3.9.2.tar.gz 11 | tar xvzf cmake-3.9.2.tar.gz 12 | cd cmake-3.9.2 13 | ./bootstrap 14 | make 15 | make install 16 | 17 | cd $curdir/.. 18 | mkdir build 19 | cd build 20 | cmake -DBUILD_TESTS=OFF -DWITH_JAVA_CLIENT=OFF -DWITH_PY_CLIENT=OFF .. 21 | make -j 22 | 23 | cd .. 24 | ./sbin/sync . 25 | -------------------------------------------------------------------------------- /examples/libstreaming/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter") 2 | 3 | set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin) 4 | file(MAKE_DIRECTORY ${EXECUTABLE_OUTPUT_PATH}) 5 | 6 | set(CONFLUO_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/../../libconfluo/confluo) 7 | set(UTILS_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/../../libutils/utils) 8 | set(RPC_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/../../librpc/rpc) 9 | set(STREAM_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/streaming) 10 | 11 | include_directories(${GTEST_INCLUDE_DIR} 12 | ${CONFLUO_INCLUDE} 13 | ${UTILS_INCLUDE} 14 | ${RPC_INCLUDE} 15 | ${STREAM_INCLUDE} 16 | ${Boost_INCLUDE_DIRS}) 17 | 18 | if (BUILD_TESTS) 19 | # Build test 20 | add_executable(stest 21 | streaming/stream_producer.h 22 | streaming/stream_consumer.h 23 | test/streaming_test_utils.h 24 | test/test_main.cc 25 | test/stream_test.h) 26 | target_link_libraries(stest gtest gtest_main rpcclient confluo) 27 | add_dependencies(stest googletest) 28 | 29 | # Register test 30 | enable_testing() 31 | add_test(StreamTest ${EXECUTABLE_OUTPUT_PATH}/stest) 32 | endif () 33 | -------------------------------------------------------------------------------- /examples/libstreaming/streaming/stream_consumer.h: -------------------------------------------------------------------------------- 1 | #ifndef EXAMPLES_STREAM_CONSUMER_H_ 2 | #define EXAMPLES_STREAM_CONSUMER_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "rpc_client.h" 8 | #include "atomic_multilog.h" 9 | 10 | namespace confluo { 11 | 12 | class stream_consumer : public rpc::rpc_client { 13 | public: 14 | stream_consumer(const std::string server_address, int server_port, 15 | uint64_t prefetch_size) 16 | : rpc_client(server_address, server_port), 17 | read_buffer_(std::make_pair(INT64_C(-1), "")) { 18 | prefetch_size_ = prefetch_size; 19 | offset_ = 0; 20 | } 21 | 22 | void consume(std::string& _return) { 23 | if (cur_multilog_id_ == -1) { 24 | throw illegal_state_exception("Must set table first"); 25 | } 26 | int64_t& buf_off = read_buffer_.first; 27 | std::string& buf = read_buffer_.second; 28 | int64_t rbuf_lim = static_cast(buf_off + buf.size()); 29 | if (buf_off == -1 || offset_ < buf_off || offset_ >= rbuf_lim) { 30 | read_buffer_.first = offset_; 31 | client_->read(buf, cur_multilog_id_, buf_off, prefetch_size_); 32 | } 33 | _return = buf.substr(static_cast(offset_ - buf_off), cur_schema_.record_size()); 34 | offset_ += cur_schema_.record_size(); 35 | } 36 | 37 | private: 38 | // Read buffer 39 | std::pair read_buffer_; 40 | uint64_t prefetch_size_; 41 | int64_t offset_; 42 | }; 43 | 44 | } 45 | 46 | #endif /* EXAMPLES_STREAM_CONSUMER_H_ */ 47 | -------------------------------------------------------------------------------- /examples/libstreaming/test/streaming_test_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef EXAMPLES_TEST_STREAMING_TEST_UTILS_H_ 2 | #define EXAMPLES_TEST_STREAMING_TEST_UTILS_H_ 3 | 4 | #include "rpc_client.h" 5 | 6 | class streaming_test_utils { 7 | public: 8 | static void wait_till_server_ready(const std::string& host, int port) { 9 | bool check = true; 10 | while (check) { 11 | try { 12 | confluo::rpc::rpc_client(host, port); 13 | check = false; 14 | } catch (TTransportException& e) { 15 | usleep(100000); 16 | } 17 | } 18 | } 19 | }; 20 | 21 | #endif /* EXAMPLES_TEST_STREAMING_TEST_UTILS_H_ */ 22 | -------------------------------------------------------------------------------- /examples/libstreaming/test/test_main.cc: -------------------------------------------------------------------------------- 1 | #define GTEST_HAS_TR1_TUPLE 0 2 | 3 | #include "error_handling.h" 4 | #include "gtest/gtest.h" 5 | #include "stream_test.h" 6 | 7 | int main(int argc, char** argv) { 8 | utils::error_handling::install_signal_handler(argv[0], SIGSEGV, SIGKILL, 9 | SIGSTOP); 10 | ::testing::InitGoogleTest(&argc, argv); 11 | return RUN_ALL_TESTS(); 12 | } 13 | -------------------------------------------------------------------------------- /examples/libtimeseries/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter") 2 | 3 | set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin) 4 | file(MAKE_DIRECTORY ${EXECUTABLE_OUTPUT_PATH}) 5 | 6 | set(CONFLUO_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/../../libconfluo/confluo) 7 | set(UTILS_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/../../libutils/utils) 8 | set(TIMESERIES_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/timeseries) 9 | 10 | include_directories(${GTEST_INCLUDE_DIR} 11 | ${CONFLUO_INCLUDE} 12 | ${UTILS_INCLUDE} 13 | ${TIMESERIES_INCLUDE} 14 | ${Boost_INCLUDE_DIRS}) 15 | 16 | if (BUILD_TESTS) 17 | # Build test 18 | file(GLOB_RECURSE test_sources test/*.cc) 19 | add_executable(tstest 20 | timeseries/timeseries_db.h 21 | test/test_main.cc 22 | test/timeseries_db_test.h) 23 | target_link_libraries(tstest confluo gtest gtest_main) 24 | add_dependencies(tstest googletest) 25 | 26 | # Register test 27 | enable_testing() 28 | add_test(TimeseriesTest ${EXECUTABLE_OUTPUT_PATH}/tstest) 29 | endif () 30 | -------------------------------------------------------------------------------- /examples/libtimeseries/test/test_main.cc: -------------------------------------------------------------------------------- 1 | #define GTEST_HAS_TR1_TUPLE 0 2 | 3 | #include "error_handling.h" 4 | #include "gtest/gtest.h" 5 | #include "timeseries_db_test.h" 6 | 7 | int main(int argc, char** argv) { 8 | utils::error_handling::install_signal_handler(argv[0], SIGSEGV, SIGKILL, 9 | SIGSTOP); 10 | ::testing::InitGoogleTest(&argc, argv); 11 | return RUN_ALL_TESTS(); 12 | } 13 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | mkdir -p build 5 | cd build 6 | if [[ "$OSTYPE" == "darwin"* ]]; then 7 | cmake .. -DCMAKE_CXX_COMPILER=g++ $@ 8 | else 9 | cmake .. $@ 10 | fi 11 | 12 | START=$(date +%s) 13 | make install 14 | END=$(date +%s) 15 | echo "Total install time (real) = $(( $END - $START )) seconds" 16 | -------------------------------------------------------------------------------- /javaclient/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB_RECURSE CONFLUO_CLIENT_SOURCES src/main/java/*.java) 2 | set(JAVA_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/${LIB_INSTALL_DIR}/java") 3 | set(JAVA_DOC_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/${DOC_INSTALL_DIR}/java") 4 | 5 | set(CONFLUO_CLIENT_ARTIFACTS confluo-client-${CONFLUO_VERSION}.jar confluo-client-${CONFLUO_VERSION}.pom) 6 | set(SERVER_EXEC "${CMAKE_CURRENT_BINARY_DIR}/../librpc/confluod") 7 | 8 | set(BUILD_XML_IN "${CMAKE_CURRENT_SOURCE_DIR}/build.xml.in") 9 | set(BUILD_XML "${CMAKE_CURRENT_BINARY_DIR}/build.xml") 10 | configure_file(${BUILD_XML_IN} ${BUILD_XML} @ONLY) 11 | 12 | set(JAVA_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/main/java) 13 | set(JAVA_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/test/java) 14 | file(GLOB JAVA_SOURCES ${JAVA_SRC_DIR}/*.java) 15 | file(GLOB JAVA_TESTS ${JAVA_TEST_DIR}/*.java) 16 | add_custom_command( 17 | OUTPUT ${CONFLUO_CLIENT_ARTIFACTS} 18 | DEPENDS ${JAVA_SOURCES} ${JAVA_TESTS} 19 | COMMAND ${Ant_EXECUTABLE} ${ANT_FLAGS} -Dbuild.dir="${CMAKE_CURRENT_BINARY_DIR}" -f ${BUILD_XML} jar 20 | MAIN_DEPENDENCY ${BUILD_XML} 21 | DEPENDS ${CONFLUO_CLIENT_SOURCES} 22 | WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} 23 | ) 24 | 25 | add_custom_target(javaclient ALL 26 | COMMENT "Building Confluo Java Client library using Ant" 27 | DEPENDS ${CONFLUO_CLIENT_ARTIFACTS} 28 | ) 29 | 30 | if (BUILD_TESTS) 31 | add_test(NAME JavaClientTest 32 | COMMAND ${Ant_EXECUTABLE} ${ANT_FLAGS} -Dbuild.dir="${CMAKE_CURRENT_BINARY_DIR}" -f ${BUILD_XML} test 33 | WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} 34 | ) 35 | endif() 36 | -------------------------------------------------------------------------------- /javaclient/src/main/java/confluo/rpc/ConfigurationParams.java: -------------------------------------------------------------------------------- 1 | package confluo.rpc; 2 | 3 | public class ConfigurationParams { 4 | /** 5 | * The batch size when reading 6 | */ 7 | public static int READ_BATCH_SIZE = 128; 8 | } 9 | -------------------------------------------------------------------------------- /javaclient/src/main/java/confluo/rpc/DataTypes.java: -------------------------------------------------------------------------------- 1 | package confluo.rpc; 2 | 3 | /** 4 | * Container for all of the primitive data types in Confluo 5 | */ 6 | public class DataTypes { 7 | public static DataType BOOL_TYPE = new DataType(TypeId.BOOL, 1); 8 | public static DataType CHAR_TYPE = new DataType(TypeId.CHAR, 1); 9 | public static DataType UCHAR_TYPE = new DataType(TypeId.UCHAR, 1); 10 | public static DataType SHORT_TYPE = new DataType(TypeId.SHORT, 2); 11 | public static DataType USHORT_TYPE = new DataType(TypeId.USHORT, 2); 12 | public static DataType INT_TYPE = new DataType(TypeId.INT, 4); 13 | public static DataType UINT_TYPE = new DataType(TypeId.UINT, 4); 14 | public static DataType LONG_TYPE = new DataType(TypeId.LONG, 8); 15 | public static DataType ULONG_TYPE = new DataType(TypeId.ULONG, 8); 16 | public static DataType FLOAT_TYPE = new DataType(TypeId.FLOAT, 4); 17 | public static DataType DOUBLE_TYPE = new DataType(TypeId.DOUBLE, 8); 18 | 19 | public static DataType STRING_TYPE(int size) { 20 | return new DataType(TypeId.STRING, size); 21 | } 22 | } 23 | 24 | -------------------------------------------------------------------------------- /javaclient/src/main/java/confluo/rpc/StorageMode.java: -------------------------------------------------------------------------------- 1 | package confluo.rpc; 2 | 3 | /** 4 | * Different storage modes for Confluo 5 | */ 6 | public class StorageMode { 7 | /** 8 | * Data stored in memory 9 | */ 10 | public static rpc_storage_mode IN_MEMORY = rpc_storage_mode.RPC_IN_MEMORY; 11 | /** 12 | * Relaxed (no linearizable guarantees 13 | */ 14 | public static rpc_storage_mode DURABLE_RELAXED = rpc_storage_mode.RPC_DURABLE_RELAXED; 15 | /** 16 | * Data is persisted 17 | */ 18 | public static rpc_storage_mode DURABLE = rpc_storage_mode.RPC_DURABLE; 19 | } 20 | -------------------------------------------------------------------------------- /javaclient/src/main/java/confluo/rpc/TypeConversions.java: -------------------------------------------------------------------------------- 1 | package confluo.rpc; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | /** 7 | * Handler of conversions between RPC and non RPC types 8 | */ 9 | class TypeConversions { 10 | 11 | /** 12 | * Converts a schema to an rpc schema for the client 13 | * 14 | * @param schema The schema to convert 15 | * @return The rpc schema 16 | */ 17 | static List convertToRPCSchema(Schema schema) { 18 | List rpcColumns = new ArrayList<>(); 19 | for (Column column : schema.getColumns()) { 20 | rpc_column rpcCol = new rpc_column(column.getDataType().typeId.getValue(), column.getDataType().size, column.getName()); 21 | rpcColumns.add(rpcCol); 22 | } 23 | return rpcColumns; 24 | } 25 | 26 | /** 27 | * Converts an rpc schema to a schema for confluo 28 | * 29 | * @param rpcSchema The rpc schema to convert 30 | * @return The schema for confluo 31 | */ 32 | static Schema convertToSchema(List rpcSchema) { 33 | SchemaBuilder builder = new SchemaBuilder(); 34 | for (rpc_column column : rpcSchema) { 35 | rpc_data_type type = rpc_data_type.findByValue(column.getTypeId()); 36 | builder.addColumn(new DataType(type, column.getTypeSize()), column.getName()); 37 | } 38 | return new Schema(builder.build()); 39 | } 40 | } 41 | 42 | -------------------------------------------------------------------------------- /javaclient/src/main/java/confluo/rpc/TypeId.java: -------------------------------------------------------------------------------- 1 | package confluo.rpc; 2 | 3 | /** 4 | * Contains the primitive types 5 | */ 6 | public class TypeId { 7 | /** 8 | * NONE: The none type 9 | */ 10 | static rpc_data_type NONE = rpc_data_type.RPC_NONE; 11 | /** 12 | * BOOL: The boolean type 13 | */ 14 | static rpc_data_type BOOL = rpc_data_type.RPC_BOOL; 15 | /** 16 | * CHAR: The character type 17 | */ 18 | static rpc_data_type CHAR = rpc_data_type.RPC_CHAR; 19 | /** 20 | * UCHAR: The unsigned character type 21 | */ 22 | static rpc_data_type UCHAR = rpc_data_type.RPC_UCHAR; 23 | /** 24 | * SHORT: The short type 25 | */ 26 | static rpc_data_type SHORT = rpc_data_type.RPC_SHORT; 27 | /** 28 | * USHORT: The unsigned short type 29 | */ 30 | static rpc_data_type USHORT = rpc_data_type.RPC_USHORT; 31 | /** 32 | * INT: The integer type 33 | */ 34 | static rpc_data_type INT = rpc_data_type.RPC_INT; 35 | /** 36 | * UINT: The unsigned integer type 37 | */ 38 | static rpc_data_type UINT = rpc_data_type.RPC_UINT; 39 | /** 40 | * LONG: The long type 41 | */ 42 | static rpc_data_type LONG = rpc_data_type.RPC_LONG; 43 | /** 44 | * ULONG: The unsigned long type 45 | */ 46 | static rpc_data_type ULONG = rpc_data_type.RPC_ULONG; 47 | /** 48 | * FLOAT: The single precision floating type 49 | */ 50 | static rpc_data_type FLOAT = rpc_data_type.RPC_FLOAT; 51 | /** 52 | * DOUBLE: The double type 53 | */ 54 | static rpc_data_type DOUBLE = rpc_data_type.RPC_DOUBLE; 55 | /** 56 | * STRING: The string type 57 | */ 58 | static rpc_data_type STRING = rpc_data_type.RPC_STRING; 59 | } 60 | -------------------------------------------------------------------------------- /javaclient/src/main/java/confluo/rpc/rpc_iterator_type.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.12.0) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | package confluo.rpc; 8 | 9 | 10 | @javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.12.0)", date = "2019-10-09") 11 | public enum rpc_iterator_type implements org.apache.thrift.TEnum { 12 | RPC_ADHOC(0), 13 | RPC_PREDEF(1), 14 | RPC_COMBINED(2), 15 | RPC_ALERTS(3); 16 | 17 | private final int value; 18 | 19 | private rpc_iterator_type(int value) { 20 | this.value = value; 21 | } 22 | 23 | /** 24 | * Get the integer value of this enum value, as defined in the Thrift IDL. 25 | */ 26 | public int getValue() { 27 | return value; 28 | } 29 | 30 | /** 31 | * Find a the enum type by its integer value, as defined in the Thrift IDL. 32 | * @return null if the value is not found. 33 | */ 34 | @org.apache.thrift.annotation.Nullable 35 | public static rpc_iterator_type findByValue(int value) { 36 | switch (value) { 37 | case 0: 38 | return RPC_ADHOC; 39 | case 1: 40 | return RPC_PREDEF; 41 | case 2: 42 | return RPC_COMBINED; 43 | case 3: 44 | return RPC_ALERTS; 45 | default: 46 | return null; 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /javaclient/src/main/java/confluo/rpc/rpc_storage_mode.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.12.0) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | package confluo.rpc; 8 | 9 | 10 | @javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.12.0)", date = "2019-10-09") 11 | public enum rpc_storage_mode implements org.apache.thrift.TEnum { 12 | RPC_IN_MEMORY(0), 13 | RPC_DURABLE_RELAXED(1), 14 | RPC_DURABLE(2); 15 | 16 | private final int value; 17 | 18 | private rpc_storage_mode(int value) { 19 | this.value = value; 20 | } 21 | 22 | /** 23 | * Get the integer value of this enum value, as defined in the Thrift IDL. 24 | */ 25 | public int getValue() { 26 | return value; 27 | } 28 | 29 | /** 30 | * Find a the enum type by its integer value, as defined in the Thrift IDL. 31 | * @return null if the value is not found. 32 | */ 33 | @org.apache.thrift.annotation.Nullable 34 | public static rpc_storage_mode findByValue(int value) { 35 | switch (value) { 36 | case 0: 37 | return RPC_IN_MEMORY; 38 | case 1: 39 | return RPC_DURABLE_RELAXED; 40 | case 2: 41 | return RPC_DURABLE; 42 | default: 43 | return null; 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /libconfluo/confluo/aggregate/aggregate_manager.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_AGGREGATE_AGGREGATE_MANAGER_H_ 2 | #define CONFLUO_AGGREGATE_AGGREGATE_MANAGER_H_ 3 | 4 | #include "aggregate_ops.h" 5 | 6 | namespace confluo { 7 | 8 | /** 9 | * Manager of the aggregates 10 | */ 11 | class aggregate_manager { 12 | public: 13 | /** 14 | * Registers an aggregate to the manager 15 | * 16 | * @param agg The aggregate to add to the manager 17 | * 18 | * @return The size of the manager 19 | */ 20 | static size_t register_aggregate(const aggregator &agg); 21 | 22 | /** 23 | * Gets an aggregate from the name 24 | * 25 | * @param name The name of the aggregate to get 26 | * 27 | * @return The type of the aggregate 28 | */ 29 | static aggregate_type get_aggregator_id(const std::string &name); 30 | 31 | /** 32 | * Gets the aggregator based on name 33 | * 34 | * @param name The name of the aggregator 35 | * 36 | * @return The aggregator matching the name 37 | */ 38 | static aggregator get_aggregator(const std::string &name); 39 | 40 | /** 41 | * Gets aggregator based on id 42 | * 43 | * @param id The identifier of the aggregator 44 | * 45 | * @return The matching aggregator 46 | */ 47 | static aggregator get_aggregator(size_t id); 48 | 49 | /** 50 | * Gets whether the id is a valid aggregator 51 | * 52 | * @param id The identifier to test 53 | * 54 | * @return True if id is a valid aggregator, false otherwise 55 | */ 56 | static bool is_valid_id(size_t id); 57 | }; 58 | 59 | } 60 | 61 | #endif /* CONFLUO_AGGREGATE_AGGREGATE_MANAGER_H_ */ 62 | -------------------------------------------------------------------------------- /libconfluo/confluo/archival/archival_mode.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_ARCHIVAL_ARCHIVAL_MODE_H_ 2 | #define CONFLUO_ARCHIVAL_ARCHIVAL_MODE_H_ 3 | 4 | namespace confluo { 5 | namespace archival { 6 | 7 | /** 8 | * Describes whether or not an atomic_multilog 9 | * has periodic archival enabled. 10 | */ 11 | enum archival_mode { 12 | OFF = 0, 13 | ON = 1 14 | }; 15 | 16 | } 17 | } 18 | 19 | #endif /* CONFLUO_ARCHIVAL_ARCHIVAL_MODE_H_ */ 20 | -------------------------------------------------------------------------------- /libconfluo/confluo/archival/archival_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_ARCHIVAL_ARCHIVAL_CONSTS_H_ 2 | #define CONFLUO_ARCHIVAL_ARCHIVAL_CONSTS_H_ 3 | 4 | #include "atomic.h" 5 | #include "types/primitive_types.h" 6 | #include "container/reflog.h" 7 | 8 | namespace confluo { 9 | namespace archival { 10 | 11 | /** 12 | * Utilities used by archivers and loaders 13 | */ 14 | class archival_utils { 15 | public: 16 | 17 | /** 18 | * Gets filter archival path for a particular filter. 19 | * @param filter_log_path archival path for filter_log 20 | * @param filter_log_idx index of filter in filter_log 21 | * @return filter archival path 22 | */ 23 | static std::string filter_archival_path(const std::string &filter_log_path, size_t filter_log_idx); 24 | 25 | /** 26 | * Gets index archival path for a particular index. 27 | * @param index_log_path archival path for index_log 28 | * @param index_id index id 29 | * @return 30 | */ 31 | static std::string index_archival_path(const std::string &index_log_path, size_t index_id); 32 | 33 | /** 34 | * Convenience method to swap a bucket pointer of a reflog. 35 | * @param refs reflog 36 | * @param idx starting reflog index of bucket 37 | * @param encoded_bucket new bucket to swap in 38 | */ 39 | static void swap_bucket_ptr(reflog &refs, size_t idx, encoded_reflog_ptr encoded_bucket); 40 | 41 | /** 42 | * Get maximum offset stored in a reflog bucket. 43 | * @param bucket decoded bucket 44 | * @return max offset 45 | */ 46 | static uint64_t max_in_reflog_bucket(uint64_t *bucket); 47 | 48 | }; 49 | 50 | } 51 | } 52 | 53 | #endif /* CONFLUO_ARCHIVAL_ARCHIVAL_CONSTS_H_ */ 54 | -------------------------------------------------------------------------------- /libconfluo/confluo/archival/archiver.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_ARCHIVAL_ARCHIVER_H_ 2 | #define CONFLUO_ARCHIVAL_ARCHIVER_H_ 3 | 4 | class archiver { 5 | 6 | public: 7 | virtual ~archiver() = default; 8 | 9 | /** 10 | * Archive up to data log offset. 11 | * @param offset data log offset 12 | */ 13 | virtual void archive(size_t offset) = 0; 14 | 15 | }; 16 | 17 | #endif /* CONFLUO_ARCHIVAL_ARCHIVER_H_ */ 18 | -------------------------------------------------------------------------------- /libconfluo/confluo/archival/filter_log_archiver.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_ARCHIVAL_FILTER_LOG_ARCHIVER_H_ 2 | #define CONFLUO_ARCHIVAL_FILTER_LOG_ARCHIVER_H_ 3 | 4 | #include "archival_utils.h" 5 | #include "filter_archiver.h" 6 | #include "filter_log.h" 7 | 8 | namespace confluo { 9 | namespace archival { 10 | 11 | class filter_log_archiver { 12 | 13 | public: 14 | filter_log_archiver(); 15 | 16 | /** 17 | * Constructor. 18 | * @param path directory to archive in 19 | * @param filters filter log to archive 20 | */ 21 | filter_log_archiver(const std::string &path, filter_log *filters); 22 | 23 | /** 24 | * Default destructor. 25 | */ 26 | ~filter_log_archiver(); 27 | 28 | /** 29 | * Archive all filters up to a data log offset. Create new 30 | * archivers for new filters since the last archive call. 31 | * @param offset data log offset 32 | */ 33 | void archive(size_t offset); 34 | 35 | private: 36 | /** 37 | * Initialize archivers for filters that haven't 38 | * already had archivers initialized for. 39 | */ 40 | void init_new_archivers(); 41 | 42 | std::string path_; 43 | std::vector filter_archivers_; 44 | filter_log *filters_; 45 | 46 | }; 47 | 48 | } 49 | } 50 | 51 | #endif /* CONFLUO_ARCHIVAL_FILTER_LOG_ARCHIVER_H_ */ 52 | -------------------------------------------------------------------------------- /libconfluo/confluo/archival/index_log_archiver.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_ARCHIVAL_INDEX_LOG_ARCHIVER_H_ 2 | #define CONFLUO_ARCHIVAL_INDEX_LOG_ARCHIVER_H_ 3 | 4 | #include "index_archiver.h" 5 | #include "index_log.h" 6 | #include "schema/schema.h" 7 | 8 | namespace confluo { 9 | namespace archival { 10 | 11 | class index_log_archiver { 12 | 13 | public: 14 | index_log_archiver(); 15 | 16 | /** 17 | * Constructor. 18 | * @param path directory to archive in 19 | * @param indexes index log to archive 20 | * @param schema data schema 21 | */ 22 | index_log_archiver(const std::string &path, index_log *indexes, schema_t *schema); 23 | 24 | /** 25 | * Default destructor. 26 | */ 27 | ~index_log_archiver(); 28 | 29 | /** 30 | * Archive all indexes up to a data log offset. Create new 31 | * archivers for new indexes since the last archive call. 32 | * @param offset data log offset 33 | */ 34 | void archive(size_t offset); 35 | 36 | private: 37 | /** 38 | * Initialize archivers for new indexes. 39 | */ 40 | void init_new_archivers(); 41 | 42 | std::string path_; 43 | std::vector index_archivers_; 44 | index_log *indexes_; 45 | schema_t *schema_; 46 | 47 | }; 48 | 49 | } 50 | } 51 | 52 | #endif /* CONFLUO_ARCHIVAL_INDEX_LOG_ARCHIVER_H_ */ 53 | -------------------------------------------------------------------------------- /libconfluo/confluo/archival/io/incremental_file_offset.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_ARCHIVAL_IO_INCR_FILE_OFFSET_H_ 2 | #define CONFLUO_ARCHIVAL_IO_INCR_FILE_OFFSET_H_ 3 | 4 | #include 5 | 6 | namespace confluo { 7 | namespace archival { 8 | 9 | /** 10 | * Offset into an incremental file, uniquely defined by 11 | * the path to a file and the offset into the file 12 | */ 13 | class incremental_file_offset { 14 | public: 15 | incremental_file_offset(); 16 | 17 | incremental_file_offset(std::string path, size_t offset); 18 | 19 | std::string path() const; 20 | 21 | size_t offset() const; 22 | 23 | private: 24 | std::string path_; 25 | size_t off_; 26 | }; 27 | 28 | } 29 | } 30 | 31 | #endif /* CONFLUO_ARCHIVAL_IO_INCR_FILE_OFFSET_H_ */ 32 | -------------------------------------------------------------------------------- /libconfluo/confluo/archival/io/incremental_file_stream.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_ARCHIVAL_IO_INCR_FILE_STREAM_H_ 2 | #define CONFLUO_ARCHIVAL_IO_INCR_FILE_STREAM_H_ 3 | 4 | #include "file_utils.h" 5 | #include "incremental_file_offset.h" 6 | 7 | namespace confluo { 8 | namespace archival { 9 | 10 | class incremental_file_stream { 11 | 12 | public: 13 | incremental_file_stream(); 14 | 15 | incremental_file_stream(const std::string &path, const std::string &file_prefix); 16 | 17 | std::string cur_path(); 18 | 19 | std::string transaction_log_path(); 20 | 21 | void truncate(incremental_file_offset incr_file_off, size_t transaction_log_off); 22 | 23 | protected: 24 | size_t file_num_; 25 | std::string dir_path_; 26 | std::string file_prefix_; 27 | 28 | }; 29 | 30 | } 31 | } 32 | 33 | #endif /* CONFLUO_ARCHIVAL_IO_INCR_FILE_STREAM_H_ */ 34 | -------------------------------------------------------------------------------- /libconfluo/confluo/compression/confluo_encoder.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_COMPRESSION_CONFLUO_ENCODER_H_ 2 | #define CONFLUO_COMPRESSION_CONFLUO_ENCODER_H_ 3 | 4 | #include "delta_encoder.h" 5 | #include "lz4_encoder.h" 6 | #include "exceptions.h" 7 | #include "storage/ptr_aux_block.h" 8 | #include "storage/ptr_metadata.h" 9 | #include "container/unique_byte_array.h" 10 | 11 | namespace confluo { 12 | namespace compression { 13 | 14 | using namespace storage; 15 | 16 | class confluo_encoder { 17 | public: 18 | /** 19 | * Encode pointer. 20 | * @param ptr unencoded data pointer, allocated by the storage allocator. 21 | * @param size size of unencoded data in bytes 22 | * @return pointer to raw encoded data 23 | */ 24 | static unique_byte_array encode(void *ptr, size_t size, uint8_t encoding); 25 | 26 | private: 27 | /** 28 | * No-op deleter. 29 | * 30 | * @param ptr unencoded data pointer, allocated by the storage allocator. 31 | */ 32 | static void no_op_delete(uint8_t *ptr); 33 | 34 | }; 35 | 36 | } 37 | } 38 | 39 | #endif /* CONFLUO_COMPRESSION_CONFLUO_ENCODER_H_ */ 40 | -------------------------------------------------------------------------------- /libconfluo/confluo/compression/delta_encoder.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_COMPRESSION_DELTA_ENCODER_H_ 2 | #define CONFLUO_COMPRESSION_DELTA_ENCODER_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "container/bitmap/delta_encoded_array.h" 14 | #include "container/unique_byte_array.h" 15 | 16 | namespace confluo { 17 | namespace compression { 18 | 19 | /** 20 | * A stateless encoder. Takes as input a buffer of bytes and applies delta 21 | * encoding on that data. Supports full encoding and packaged for full 22 | * or partial decoding 23 | */ 24 | class delta_encoder { 25 | public: 26 | /** 27 | * Encodes the input buffer data using delta encoding 28 | * 29 | * @param source_buffer The buffer data to encode 30 | * @param source_length The length of the input buffer array 31 | */ 32 | template 33 | static unique_byte_array encode(T *source_buffer, size_t source_length) { 34 | std::sort(source_buffer, source_buffer + source_length); 35 | elias_gamma_encoded_array enc_array(source_buffer, source_length); 36 | size_t buffer_size = enc_array.storage_size() + sizeof(size_t); 37 | uint8_t *output_buffer = new uint8_t[buffer_size]; 38 | std::memcpy(output_buffer, &source_length, sizeof(size_t)); 39 | enc_array.to_byte_array(output_buffer + sizeof(size_t)); 40 | return unique_byte_array(output_buffer, buffer_size); 41 | } 42 | 43 | }; 44 | 45 | } 46 | } 47 | 48 | #endif /* CONFLUO_COMPRESSION_DELTA_ENCODER_H_ */ 49 | -------------------------------------------------------------------------------- /libconfluo/confluo/conf/configuration_parser.h: -------------------------------------------------------------------------------- 1 | #ifndef LIBCONFLUO_CONFLUO_CONF_CONFIGURATION_PARSER_H_ 2 | #define LIBCONFLUO_CONFLUO_CONF_CONFIGURATION_PARSER_H_ 3 | 4 | #include "exceptions.h" 5 | #include "storage/ptr_aux_block.h" 6 | 7 | namespace confluo { 8 | 9 | class encoding_params { 10 | public: 11 | static inline std::string UNENCODED() { 12 | return "unencoded"; 13 | } 14 | 15 | static inline std::string LZ4() { 16 | return "lz4"; 17 | } 18 | static inline std::string ELIAS_GAMMA() { 19 | return "elias_gamma"; 20 | } 21 | }; 22 | 23 | class configuration_parser { 24 | public: 25 | static uint8_t to_encoding_type(const std::string ¶m) { 26 | if (param == encoding_params::UNENCODED()) { 27 | return storage::encoding_type::D_UNENCODED; 28 | } else if (param == encoding_params::LZ4()) { 29 | return storage::encoding_type::D_LZ4; 30 | } else if (param == encoding_params::ELIAS_GAMMA()) { 31 | return storage::encoding_type::D_ELIAS_GAMMA; 32 | } else { 33 | THROW(illegal_state_exception, "Invalid encoding type!"); 34 | } 35 | } 36 | 37 | }; 38 | 39 | } 40 | 41 | #endif /* LIBCONFLUO_CONFLUO_CONF_CONFIGURATION_PARSER_H_ */ 42 | -------------------------------------------------------------------------------- /libconfluo/confluo/container/cursor/alert_cursor.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_CONTAINER_CURSOR_ALERT_CURSOR_H_ 2 | #define CONFLUO_CONTAINER_CURSOR_ALERT_CURSOR_H_ 3 | 4 | #include "batched_cursor.h" 5 | #include "alert.h" 6 | #include "alert_index.h" 7 | 8 | namespace confluo { 9 | 10 | typedef batched_cursor alert_cursor; 11 | 12 | /** 13 | * A cursor for triggers and alerts 14 | */ 15 | class trigger_alert_cursor : public alert_cursor { 16 | public: 17 | /** The list of alerts */ 18 | typedef monitor::alert_index::alert_list alert_list; 19 | /** The iterator for alerts */ 20 | typedef monitor::alert_index::alert_list::const_iterator alert_iterator; 21 | 22 | /** 23 | * Initializes a trigger alert cursor 24 | * 25 | * @param alerts The list of alerts 26 | * @param trigger_name The name of the trigger 27 | * @param batch_size The number of records in the batch 28 | */ 29 | trigger_alert_cursor(const alert_list &alerts, const std::string &trigger_name, size_t batch_size = 64); 30 | 31 | /** 32 | * Loads the next record batch 33 | * 34 | * @return The size of the batch 35 | */ 36 | virtual size_t load_next_batch() override; 37 | 38 | private: 39 | alert_iterator cur_; 40 | alert_iterator end_; 41 | std::string trigger_name_; 42 | }; 43 | 44 | } 45 | 46 | #endif /* CONFLUO_CONTAINER_CURSOR_ALERT_CURSOR_H_ */ 47 | -------------------------------------------------------------------------------- /libconfluo/confluo/container/data_log.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_CONTAINER_DATA_LOG_H_ 2 | #define CONFLUO_CONTAINER_DATA_LOG_H_ 3 | 4 | #include "monolog/monolog.h" 5 | #include "archival/monolog_linear_archiver.h" 6 | 7 | namespace confluo { 8 | 9 | /** 10 | * Constants for the data log 11 | */ 12 | class data_log_constants { 13 | public: 14 | /** Maximum number of blocks */ 15 | static const size_t MAX_BUCKETS = 65536; 16 | /** The size of each block */ 17 | static const size_t BUCKET_SIZE = 67108864; 18 | /** The size of the buffer */ 19 | static const size_t BUFFER_SIZE = 1048576; 20 | }; 21 | 22 | typedef monolog::monolog_linear data_log; 26 | 27 | typedef archival::monolog_linear_archiver data_log_archiver; 31 | 32 | typedef storage::read_only_encoded_ptr read_only_data_log_ptr; 33 | typedef storage::encoded_ptr encoded_data_log_ptr; 34 | typedef storage::decoded_ptr data_ptr; 35 | 36 | } 37 | 38 | #endif /* CONFLUO_CONTAINER_DATA_LOG_H_ */ 39 | -------------------------------------------------------------------------------- /libconfluo/confluo/container/monolog/monolog.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_CONTAINER_MONOLOG_MONOLOG_H_ 2 | #define CONFLUO_CONTAINER_MONOLOG_MONOLOG_H_ 3 | 4 | /** 5 | * Adds includes for all monolog types. 6 | */ 7 | 8 | #include "monolog_exp2.h" 9 | #include "monolog_exp2_linear.h" 10 | #include "monolog_iterator.h" 11 | #include "monolog_linear.h" 12 | 13 | #endif /* CONFLUO_CONTAINER_MONOLOG_MONOLOG_H_ */ 14 | -------------------------------------------------------------------------------- /libconfluo/confluo/container/reflog.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_CONTAINER_REFLOG_H_ 2 | #define CONFLUO_CONTAINER_REFLOG_H_ 3 | 4 | #include "storage/swappable_encoded_ptr.h" 5 | #include "storage/encoded_ptr.h" 6 | #include "monolog/monolog.h" 7 | 8 | using namespace ::confluo::monolog; 9 | 10 | namespace confluo { 11 | 12 | class reflog_constants { 13 | public: 14 | static const size_t NCONTAINERS = 18; 15 | static const size_t BUCKET_SIZE = 1024; 16 | }; 17 | 18 | /** 19 | * typedef for RefLog type -- a MonoLog of type uint64_t, 20 | * 18 bucket containers and bucket size of 1024. 21 | */ 22 | typedef monolog_exp2_linear reflog; 23 | typedef storage::read_only_encoded_ptr read_only_reflog_ptr; 24 | typedef storage::encoded_ptr encoded_reflog_ptr; 25 | typedef storage::decoded_ptr decoded_reflog_ptr; 26 | 27 | } 28 | 29 | #endif /* CONFLUO_CONTAINER_REFLOG_H_ */ 30 | -------------------------------------------------------------------------------- /libconfluo/confluo/container/sketch/frequency_functions.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_CONTAINER_SKETCH_FREQUENCY_FUNCTIONS_H 2 | #define CONFLUO_CONTAINER_SKETCH_FREQUENCY_FUNCTIONS_H 3 | 4 | namespace confluo { 5 | namespace sketch { 6 | 7 | /** 8 | * These are monotonically increasing frequency-domain functions. 9 | * in f and O(f^2), where f is the frequency of the data point 10 | * @tparam counter_t type of input 11 | */ 12 | template 13 | class frequency_functions { 14 | 15 | public: 16 | template using fn = std::function; 17 | 18 | static arg_t l2_norm(arg_t freq) { 19 | return freq * freq; 20 | } 21 | 22 | static double entropy(arg_t freq) { 23 | return freq * std::log(freq); 24 | } 25 | 26 | static arg_t cardinality(arg_t freq) { 27 | return 1; 28 | } 29 | 30 | }; 31 | 32 | } 33 | } 34 | 35 | #endif // CONFLUO_CONTAINER_SKETCH_FREQUENCY_FUNCTIONS_H 36 | -------------------------------------------------------------------------------- /libconfluo/confluo/container/unique_byte_array.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_STORAGE_UNIQUE_BYTE_ARRAY_H_ 2 | #define CONFLUO_STORAGE_UNIQUE_BYTE_ARRAY_H_ 3 | 4 | #include 5 | 6 | namespace confluo { 7 | 8 | class unique_byte_array { 9 | 10 | public: 11 | typedef void (*deleter_fn)(uint8_t *data_ptr); 12 | typedef std::unique_ptr unique_uint8_t_ptr; 13 | 14 | unique_byte_array(uint8_t *data, size_t size, deleter_fn deleter = array_delete) 15 | : ptr_(data, deleter), 16 | size_(size) { 17 | } 18 | 19 | unique_byte_array(size_t size, deleter_fn deleter = array_delete) 20 | : unique_byte_array(new uint8_t[size], size, deleter) { 21 | } 22 | 23 | uint8_t *get() { 24 | return ptr_.get(); 25 | } 26 | 27 | size_t size() { 28 | return size_; 29 | } 30 | 31 | private: 32 | static void array_delete(uint8_t *ptr) { delete[] ptr; } 33 | 34 | unique_uint8_t_ptr ptr_; 35 | size_t size_; 36 | 37 | }; 38 | 39 | } 40 | 41 | #endif /* CONFLUO_CONTAINER_UNIQUE_BYTE_ARRAY_H_ */ 42 | -------------------------------------------------------------------------------- /libconfluo/confluo/exceptions.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_EXCEPTIONS_H_ 2 | #define CONFLUO_EXCEPTIONS_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "error_handling.h" 8 | 9 | namespace confluo { 10 | 11 | /** 12 | * Defines an exception for a given name 13 | */ 14 | #define DEFINE_EXCEPTION(name)\ 15 | class name : public std::exception {\ 16 | public:\ 17 | explicit name(std::string msg)\ 18 | : msg_(std::move(msg)) {\ 19 | }\ 20 | name()\ 21 | : msg_("") {\ 22 | }\ 23 | const char* what() const noexcept {\ 24 | return msg_.c_str();\ 25 | }\ 26 | name& operator=(const name& other) {\ 27 | msg_ = other.msg_;\ 28 | return *this;\ 29 | }\ 30 | private:\ 31 | std::string msg_;\ 32 | }; 33 | 34 | /** Defines exception for archival */ 35 | DEFINE_EXCEPTION(archival_exception) 36 | /** Defines exception when parsing fails */ 37 | DEFINE_EXCEPTION(parse_exception) 38 | /** Defines exception for an invalid access */ 39 | DEFINE_EXCEPTION(invalid_access_exception) 40 | /** Defines exception for an invalid cast */ 41 | DEFINE_EXCEPTION(invalid_cast_exception) 42 | /** Defines exception for an invalid operation */ 43 | DEFINE_EXCEPTION(invalid_operation_exception) 44 | /** Defines an exception for an illegal state */ 45 | DEFINE_EXCEPTION(illegal_state_exception) 46 | /** Defines exception for a memory error */ 47 | DEFINE_EXCEPTION(memory_exception) 48 | /** Defines exception for an unsupported operation */ 49 | DEFINE_EXCEPTION(unsupported_exception) 50 | /** Defines exception for a management error */ 51 | DEFINE_EXCEPTION(management_exception) 52 | 53 | #define THROW(ex, msg)\ 54 | throw ex(msg) 55 | 56 | } 57 | 58 | #endif /* CONFLUO_EXCEPTIONS_H_ */ 59 | -------------------------------------------------------------------------------- /libconfluo/confluo/filter_log.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_FILTER_LOG_H_ 2 | #define CONFLUO_FILTER_LOG_H_ 3 | 4 | #include "container/monolog/monolog_exp2.h" 5 | #include "filter.h" 6 | 7 | namespace confluo { 8 | 9 | /** 10 | * A filter log is a type of monolog for filters 11 | */ 12 | typedef monolog::monolog_exp2 filter_log; 13 | 14 | } 15 | 16 | #endif /* CONFLUO_FILTER_LOG_H_ */ 17 | -------------------------------------------------------------------------------- /libconfluo/confluo/index_log.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_INDEX_LOG_H_ 2 | #define CONFLUO_INDEX_LOG_H_ 3 | 4 | #include "container/monolog/monolog_exp2.h" 5 | #include "container/radix_tree.h" 6 | 7 | namespace confluo { 8 | 9 | /** An index_log is a type of monolog for supporting indexes */ 10 | typedef monolog::monolog_exp2 index_log; 11 | 12 | } 13 | 14 | #endif /* CONFLUO_INDEX_LOG_H_ */ 15 | -------------------------------------------------------------------------------- /libconfluo/confluo/schema/column_snapshot.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_SCHEMA_COLUMN_SNAPSHOT_H_ 2 | #define CONFLUO_SCHEMA_COLUMN_SNAPSHOT_H_ 3 | 4 | #include 5 | 6 | #include "types/data_type.h" 7 | 8 | namespace confluo { 9 | 10 | /** 11 | * Data for snapshot of column 12 | */ 13 | struct column_snapshot { 14 | /** The data type of the column */ 15 | data_type type; 16 | /** The offset of the column */ 17 | size_t offset; 18 | /** Whether the column is indexed */ 19 | bool indexed; 20 | /** The identifier for the index */ 21 | uint32_t index_id; 22 | /** The bucket size for the index */ 23 | double index_bucket_size; 24 | }; 25 | 26 | } 27 | 28 | #endif /* CONFLUO_SCHEMA_COLUMN_SNAPSHOT_H_ */ 29 | -------------------------------------------------------------------------------- /libconfluo/confluo/storage/allocator.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_STORAGE_ALLOCATOR_H_ 2 | #define CONFLUO_STORAGE_ALLOCATOR_H_ 3 | 4 | #include "storage_allocator.h" 5 | 6 | namespace confluo { 7 | 8 | /** Static storage allocator */ 9 | class allocator { 10 | public: 11 | static storage::storage_allocator &instance() { 12 | static storage::storage_allocator alloc; 13 | return alloc; 14 | } 15 | }; 16 | 17 | } 18 | 19 | #endif /* CONFLUO_STORAGE_ALLOCATOR_H_ */ 20 | -------------------------------------------------------------------------------- /libconfluo/confluo/storage/memory_stat.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_STORAGE_MEMPOOL_STAT_H_ 2 | #define CONFLUO_STORAGE_MEMPOOL_STAT_H_ 3 | 4 | #include "atomic.h" 5 | #include 6 | #include 7 | 8 | namespace confluo { 9 | namespace storage { 10 | 11 | /** 12 | * The memory stat class. Contains functionality for modifying the amount 13 | * of memory used. 14 | */ 15 | class memory_stat { 16 | 17 | public: 18 | /** 19 | * Initializes memory statistics. 20 | */ 21 | memory_stat(); 22 | 23 | /** 24 | * Increments the memory used by the specified size 25 | * 26 | * @param size The size to increment by in bytes 27 | */ 28 | void increment(size_t size); 29 | 30 | /** 31 | * Decrements the memory used by the specified size 32 | * 33 | * @param size The size to increment by in bytes 34 | */ 35 | void decrement(size_t size); 36 | 37 | /** 38 | * Loads the amount of memory used 39 | * 40 | * @return The amount of memory used 41 | */ 42 | size_t get(); 43 | 44 | private: 45 | atomic::type memory_used_; 46 | 47 | }; 48 | 49 | } 50 | } 51 | 52 | #endif /* CONFLUO_STORAGE_MEMPOOL_STAT_H_ */ 53 | -------------------------------------------------------------------------------- /libconfluo/confluo/storage/ptr_aux_block.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_STORAGE_PTR_AUX_BLOCK_H_ 2 | #define CONFLUO_STORAGE_PTR_AUX_BLOCK_H_ 3 | 4 | #include 5 | #include "ptr_metadata.h" 6 | 7 | namespace confluo { 8 | namespace storage { 9 | 10 | /** 11 | * Archival state of data pointed to. 12 | */ 13 | typedef struct state_type { 14 | static const uint8_t D_IN_MEMORY = 0; 15 | static const uint8_t D_ARCHIVED = 1; 16 | } state_type; 17 | 18 | /** 19 | * Encoding type of data pointed to. 20 | */ 21 | typedef struct encoding_type { 22 | static const uint8_t D_UNENCODED = 0; 23 | static const uint8_t D_LZ4 = 1; 24 | static const uint8_t D_ELIAS_GAMMA = 2; 25 | } encoding_type; 26 | 27 | /** 28 | * Pointer auxiliary block containing 29 | * contextual metadata about the data 30 | * pointed to. This data structure takes 31 | * 1B of space on its own but when set to 32 | * a ptr_metadata object will only use 4 bits. 33 | */ 34 | typedef struct ptr_aux_block { 35 | uint8_t encoding_ : 3; 36 | uint8_t state_ : 1; 37 | 38 | ptr_aux_block(); 39 | 40 | ptr_aux_block(uint8_t state, uint8_t encoding); 41 | 42 | static ptr_aux_block get(ptr_metadata *metadata); 43 | 44 | } aux_block; 45 | 46 | } 47 | } 48 | 49 | #endif /* CONFLUO_STORAGE_PTR_AUX_BLOCK_H_ */ 50 | -------------------------------------------------------------------------------- /libconfluo/confluo/storage/ptr_metadata.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_STORAGE_PTR_METADATA_H_ 2 | #define CONFLUO_STORAGE_PTR_METADATA_H_ 3 | 4 | #include 5 | namespace confluo { 6 | namespace storage { 7 | 8 | // TODO add enums to convert between raw byte types 9 | 10 | /** 11 | * Type of allocation. 12 | */ 13 | struct alloc_type { 14 | /** Default allocation */ 15 | static const uint8_t D_DEFAULT = 0; 16 | /** Memory mapped allocation */ 17 | static const uint8_t D_MMAP = 1; 18 | }; 19 | 20 | /** 21 | * Pointer metadata set for memory allocated by the 22 | * allocator. Not all fields may necessarily be set. 23 | */ 24 | struct ptr_metadata { 25 | // Do NOT re-order. 26 | uint32_t data_size_ : 32; // size of data 27 | uint16_t offset_: 16; // data offset from allocated pointer location 28 | uint16_t thread_id_ : 11; // allocating thread id 29 | uint8_t alloc_type_ : 1; // allocation type 30 | uint8_t aux_ : 4; // data-related state information 31 | 32 | ptr_metadata() = default; 33 | 34 | ~ptr_metadata() = default; 35 | 36 | /** 37 | * Get metadata associated with a pointer 38 | * @param ptr The pointer to get metadaata of 39 | * @return The metadata associated with the pointer 40 | */ 41 | static ptr_metadata *get(void *ptr); 42 | }; 43 | 44 | } 45 | } 46 | 47 | #endif /* CONFLUO_STORAGE_PTR_METADATA_H_ */ 48 | -------------------------------------------------------------------------------- /libconfluo/confluo/storage/reference_counts.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_STORAGE_REFERENCE_COUNTS_H_ 2 | #define CONFLUO_STORAGE_REFERENCE_COUNTS_H_ 3 | 4 | #include 5 | 6 | namespace confluo { 7 | namespace storage { 8 | 9 | /** 10 | * Packs two 16-bit ref counts in a single 32-bit integer. 11 | */ 12 | class reference_counts { 13 | public: 14 | reference_counts(); 15 | 16 | reference_counts(reference_counts &other); 17 | 18 | reference_counts operator=(reference_counts &other); 19 | 20 | void increment_first(); 21 | 22 | void increment_second(); 23 | 24 | void increment_both(); 25 | 26 | void decrement_first(); 27 | 28 | void decrement_second(); 29 | 30 | void decrement_both(); 31 | 32 | bool decrement_first_and_compare(); 33 | 34 | bool decrement_second_and_compare(); 35 | 36 | uint32_t get_first(); 37 | 38 | uint32_t get_second(); 39 | 40 | private: 41 | atomic::type ref_counts_; 42 | 43 | static const uint32_t FIRST_DELTA = 1U; 44 | static const uint32_t SECOND_DELTA = 1U << 16; 45 | static const uint32_t BOTH_DELTA = FIRST_DELTA + SECOND_DELTA; 46 | static const uint32_t FIRST_MASK = (1U << 16) - 1; 47 | static const uint32_t SECOND_SHIFT = 16; 48 | 49 | }; 50 | 51 | } 52 | } 53 | 54 | #endif /* CONFLUO_STORAGE_REFERENCE_COUNTS_H_ */ 55 | -------------------------------------------------------------------------------- /libconfluo/confluo/storage/storage_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_STORAGE_STORAGE_UTILS_H_ 2 | #define CONFLUO_STORAGE_STORAGE_UTILS_H_ 3 | 4 | #include 5 | #include "ptr_metadata.h" 6 | 7 | namespace confluo { 8 | namespace storage { 9 | 10 | template 11 | struct lifecycle_util { 12 | /** 13 | * Construction using placement new. 14 | * @param ptr pointer to array allocated by an allocator 15 | */ 16 | static void construct(void *ptr) { 17 | size_t len = ptr_metadata::get(ptr)->data_size_ / sizeof(T); 18 | T *casted_ptr = reinterpret_cast(ptr); 19 | for (size_t i = 0; i < len; i++) { 20 | new(casted_ptr + i) T(); 21 | } 22 | } 23 | 24 | /** 25 | * Utility to call destructor explicitly for pointer. 26 | * This is useful for allocations done using placement new 27 | * since delete[] would not call the destructor. 28 | * @param ptr pointer to array allocated by an allocator 29 | */ 30 | static void destroy(void *ptr) { 31 | size_t len = ptr_metadata::get(ptr)->data_size_ / sizeof(T); 32 | T *casted_ptr = reinterpret_cast(ptr); 33 | for (size_t i = 0; i < len; i++) { 34 | casted_ptr[i].~T(); 35 | } 36 | } 37 | }; 38 | 39 | /** 40 | * Specialization does nothing for fundamental types. 41 | */ 42 | template 43 | struct lifecycle_util::value>> { 44 | static void destroy(void *ptr) {} 45 | }; 46 | 47 | } 48 | } 49 | 50 | #endif /* CONFLUO_STORAGE_STORAGE_UTILS_H_ */ 51 | -------------------------------------------------------------------------------- /libconfluo/confluo/threads/periodic_task.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_THREADS_PERIODIC_TASK_H_ 2 | #define CONFLUO_THREADS_PERIODIC_TASK_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "atomic.h" 11 | #include "logger.h" 12 | 13 | /** 14 | * The periodic task class. Contains functionality to start and stop 15 | * the specified task. 16 | */ 17 | class periodic_task { 18 | public: 19 | /** 20 | * Constructor for periodic name that initializes the task 21 | * @param name The name of the task 22 | */ 23 | periodic_task(const std::string &name); 24 | 25 | /** 26 | * Default destructor that stops the task 27 | */ 28 | ~periodic_task(); 29 | 30 | /** 31 | * Stops the periodic task 32 | * @return True if the task was successfully stopped, false if the 33 | * task was already stopped 34 | */ 35 | bool stop(); 36 | 37 | /** 38 | * Starts the periodic task to run 39 | * @param task The function that represents the work to do 40 | * @param interval_ms The time in between executions of the task 41 | * @return True if the task was executed successfully, false otherwise 42 | */ 43 | bool start(std::function task, uint64_t interval_ms = 1); 44 | 45 | private: 46 | std::string name_; 47 | atomic::type enabled_; 48 | std::thread executor_; 49 | }; 50 | 51 | #endif /* CONFLUO_THREADS_PERIODIC_TASK_H_ */ 52 | -------------------------------------------------------------------------------- /libconfluo/confluo/trigger_log.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_TRIGGER_LOG_H_ 2 | #define CONFLUO_TRIGGER_LOG_H_ 3 | 4 | #include "container/monolog/monolog_exp2.h" 5 | #include "trigger.h" 6 | 7 | namespace confluo { 8 | 9 | /** A log for triggers */ 10 | typedef monolog::monolog_exp2 trigger_log; 11 | 12 | } 13 | 14 | #endif /* CONFLUO_TRIGGER_LOG_H_ */ 15 | -------------------------------------------------------------------------------- /libconfluo/confluo/univ_sketch_log.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_UNIV_SKETCH_LOG_H 2 | #define CONFLUO_UNIV_SKETCH_LOG_H 3 | 4 | #include "container/monolog/monolog_exp2.h" 5 | #include "container/sketch/universal_sketch.h" 6 | 7 | namespace confluo { 8 | 9 | /** A log for universal sketches */ 10 | typedef monolog::monolog_exp2 univ_sketch_log; 11 | 12 | } 13 | 14 | #endif //CONFLUO_UNIV_SKETCH_LOG_H 15 | -------------------------------------------------------------------------------- /libconfluo/src/aggregate/aggregate_manager.cc: -------------------------------------------------------------------------------- 1 | #include "aggregate/aggregate_manager.h" 2 | 3 | namespace confluo { 4 | 5 | size_t aggregate_manager::register_aggregate(const aggregator &agg) { 6 | size_t id = aggregators::instance().size(); 7 | aggregators::instance().push_back(agg); 8 | return id; 9 | } 10 | 11 | aggregate_type aggregate_manager::get_aggregator_id(const std::string &name) { 12 | return find_aggregator_id(name); 13 | } 14 | 15 | aggregator aggregate_manager::get_aggregator(const std::string &name) { 16 | return aggregators::instance()[get_aggregator_id(name)]; 17 | } 18 | 19 | aggregator aggregate_manager::get_aggregator(size_t id) { 20 | if (id < aggregators::instance().size()) { 21 | return aggregators::instance()[id]; 22 | } 23 | return aggregators::instance()[0]; 24 | } 25 | 26 | bool aggregate_manager::is_valid_id(size_t id) { 27 | return id >= 1 && id < aggregators::instance().size(); 28 | } 29 | 30 | } -------------------------------------------------------------------------------- /libconfluo/src/aggregate/aggregate_ops.cc: -------------------------------------------------------------------------------- 1 | #include "aggregate/aggregate_ops.h" 2 | 3 | namespace confluo { 4 | 5 | numeric sum_agg(const numeric &a, const numeric &b) { 6 | return a + b; 7 | } 8 | 9 | numeric min_agg(const numeric &a, const numeric &b) { 10 | return a < b ? a : b; 11 | } 12 | 13 | numeric max_agg(const numeric &a, const numeric &b) { 14 | return a < b ? b : a; 15 | } 16 | 17 | numeric count_agg(const numeric &a, const numeric &b) { 18 | return a + aggregators::count_one(); 19 | } 20 | 21 | numeric invalid_agg(const numeric &a, const numeric &b) { 22 | throw invalid_operation_exception("Invalid aggregation performed."); 23 | } 24 | 25 | aggregate_type find_aggregator_id(const std::string &name) { 26 | std::string u_name = utils::string_utils::to_upper(name); 27 | for (unsigned int i = 0; i < aggregators::instance().size(); i++) { 28 | std::string a_name = utils::string_utils::to_upper(aggregators::instance()[i].name); 29 | if (u_name.compare(a_name) == 0) { 30 | return i; 31 | } 32 | } 33 | return 0; 34 | } 35 | 36 | } -------------------------------------------------------------------------------- /libconfluo/src/alert.cc: -------------------------------------------------------------------------------- 1 | #include "alert.h" 2 | 3 | namespace confluo { 4 | namespace monitor { 5 | 6 | alert::alert() 7 | : version(0), 8 | time_bucket(0) { 9 | } 10 | 11 | alert::alert(uint64_t _time_bucket, 12 | const std::string &trigger_name, 13 | const std::string &trigger_expr, 14 | const numeric &_value, 15 | uint64_t _version) 16 | : trigger_name(trigger_name), 17 | trigger_expr(trigger_expr), 18 | value(_value), 19 | version(_version), 20 | time_bucket(_time_bucket) { 21 | } 22 | 23 | std::string alert::to_string() const { 24 | return "{ timestamp: " + std::to_string(time_bucket) + ", trigger-name: \"" + trigger_name 25 | + "\", trigger-expression: \"" + trigger_expr + "\", trigger-value: \"" + value.to_string() + "\", version: " 26 | + std::to_string(version) + " }"; 27 | } 28 | 29 | bool operator<(const alert &left, const alert &right) { 30 | if (left.time_bucket == right.time_bucket 31 | && left.trigger_name == right.trigger_name) { 32 | return left.value < right.value; 33 | } else if (left.time_bucket == right.time_bucket) { 34 | return left.trigger_name < right.trigger_name; 35 | } 36 | return left.time_bucket < right.time_bucket; 37 | } 38 | 39 | } 40 | } -------------------------------------------------------------------------------- /libconfluo/src/alert_index.cc: -------------------------------------------------------------------------------- 1 | #include "alert_index.h" 2 | 3 | namespace confluo { 4 | 5 | monitor::alert_index::alert_index() 6 | : idx_(8, 256) { 7 | } 8 | 9 | void monitor::alert_index::add_alert(uint64_t time_bucket, 10 | const std::string &trigger_name, 11 | const std::string &trigger_expr, 12 | const numeric &value, 13 | uint64_t version) { 14 | auto log = idx_.get_or_create(make_key(time_bucket)); 15 | if (find_alert(log, trigger_name, value) == -1) 16 | log->push_back( 17 | alert(time_bucket, trigger_name, trigger_expr, value, version)); 18 | } 19 | 20 | monitor::alert_index::alert_list monitor::alert_index::get_alerts(uint64_t t1, uint64_t t2) const { 21 | return idx_.range_lookup(make_key(t1), make_key(t2)); 22 | } 23 | 24 | byte_string monitor::alert_index::make_key(uint64_t time_bucket) const { 25 | return byte_string(time_bucket); 26 | } 27 | 28 | int64_t monitor::alert_index::find_alert(monitor::alert_index::alert_log *log, 29 | const std::string &trigger_name, 30 | const numeric &value) const { 31 | size_t n_alerts = log->size(); 32 | for (size_t i = 0; i < n_alerts; i++) { 33 | const alert &a = log->at(i); 34 | if (a.trigger_name == trigger_name && a.value == value) { 35 | return static_cast(i); 36 | } 37 | } 38 | return -1; 39 | } 40 | 41 | } -------------------------------------------------------------------------------- /libconfluo/src/archival/archival_metadata.cc: -------------------------------------------------------------------------------- 1 | #include "archival/archival_metadata.h" 2 | 3 | namespace confluo { 4 | namespace archival { 5 | 6 | filter_aggregates_archival_metadata::filter_aggregates_archival_metadata(byte_string ts_block, 7 | size_t version, 8 | size_t num_aggs) 9 | : ts_block_(ts_block), version_(version), num_aggs_(num_aggs) { 10 | } 11 | 12 | filter_aggregates_archival_metadata filter_aggregates_archival_metadata::read(incremental_file_reader &reader) { 13 | byte_string ts_block = byte_string(reader.read(sizeof(uint64_t))); 14 | size_t version = reader.read(); 15 | size_t num_aggs = reader.read(); 16 | return filter_aggregates_archival_metadata(ts_block, version, num_aggs); 17 | } 18 | 19 | void filter_aggregates_archival_metadata::append(filter_aggregates_archival_metadata metadata, 20 | incremental_file_writer &writer) { 21 | writer.append(metadata.ts_block_.data(), sizeof(uint64_t)); 22 | writer.append(metadata.version_); 23 | writer.append(metadata.num_aggs_); 24 | } 25 | 26 | byte_string filter_aggregates_archival_metadata::ts_block() { 27 | return ts_block_; 28 | } 29 | 30 | size_t filter_aggregates_archival_metadata::version() { 31 | return version_; 32 | } 33 | 34 | size_t filter_aggregates_archival_metadata::num_aggregates() { 35 | return num_aggs_; 36 | } 37 | 38 | } 39 | } -------------------------------------------------------------------------------- /libconfluo/src/archival/archival_utils.cc: -------------------------------------------------------------------------------- 1 | #include "archival/archival_utils.h" 2 | 3 | namespace confluo { 4 | namespace archival { 5 | 6 | std::string archival_utils::filter_archival_path(const std::string &filter_log_path, size_t filter_log_idx) { 7 | return filter_log_path + "/filter_" + std::to_string(filter_log_idx) + "/"; 8 | } 9 | 10 | std::string archival_utils::index_archival_path(const std::string &index_log_path, size_t index_id) { 11 | return index_log_path + "/index_" + std::to_string(index_id) + "/"; 12 | } 13 | 14 | void archival_utils::swap_bucket_ptr(reflog &refs, size_t idx, encoded_reflog_ptr encoded_bucket) { 15 | size_t bucket_idx, container_idx; 16 | refs.raw_data_location(idx, container_idx, bucket_idx); 17 | atomic::load(&refs.data()[container_idx])[bucket_idx].swap_ptr(encoded_bucket); 18 | } 19 | 20 | uint64_t archival_utils::max_in_reflog_bucket(uint64_t *bucket) { 21 | uint64_t max = 0; 22 | for (size_t i = 0; i < reflog_constants::BUCKET_SIZE && bucket[i] != limits::ulong_max; i++) 23 | max = std::max(max, bucket[i]); 24 | return max; 25 | } 26 | 27 | } 28 | } -------------------------------------------------------------------------------- /libconfluo/src/archival/filter_log_archiver.cc: -------------------------------------------------------------------------------- 1 | #include "archival/filter_log_archiver.h" 2 | 3 | namespace confluo { 4 | namespace archival { 5 | 6 | filter_log_archiver::filter_log_archiver() 7 | : filter_log_archiver("", nullptr) { 8 | } 9 | 10 | filter_log_archiver::filter_log_archiver(const std::string &path, filter_log *filters) 11 | : path_(path), 12 | filter_archivers_(), 13 | filters_(filters) { 14 | } 15 | 16 | filter_log_archiver::~filter_log_archiver() { 17 | for (auto *archiver : filter_archivers_) 18 | delete archiver; 19 | } 20 | 21 | void filter_log_archiver::archive(size_t offset) { 22 | init_new_archivers(); 23 | for (size_t i = 0; i < filters_->size(); i++) { 24 | if (filters_->at(i)->is_valid()) 25 | filter_archivers_.at(i)->archive(offset); 26 | } 27 | } 28 | 29 | void filter_log_archiver::init_new_archivers() { 30 | for (size_t i = filter_archivers_.size(); i < filters_->size(); i++) { 31 | std::string filter_path = archival_utils::filter_archival_path(path_, i); 32 | file_utils::create_dir(filter_path); 33 | filter_archivers_.push_back(new filter_archiver(filter_path, filters_->at(i))); 34 | } 35 | } 36 | 37 | } 38 | } -------------------------------------------------------------------------------- /libconfluo/src/archival/index_log_archiver.cc: -------------------------------------------------------------------------------- 1 | #include "archival/index_log_archiver.h" 2 | 3 | namespace confluo { 4 | namespace archival { 5 | 6 | index_log_archiver::index_log_archiver() 7 | : index_log_archiver("", nullptr, nullptr) { 8 | } 9 | 10 | index_log_archiver::index_log_archiver(const std::string &path, index_log *indexes, schema_t *schema) 11 | : path_(path), 12 | index_archivers_(), 13 | indexes_(indexes), 14 | schema_(schema) { 15 | } 16 | 17 | index_log_archiver::~index_log_archiver() { 18 | for (auto *archiver : index_archivers_) 19 | delete archiver; 20 | } 21 | 22 | void index_log_archiver::archive(size_t offset) { 23 | init_new_archivers(); 24 | for (size_t i = 0; i < schema_->size(); i++) { 25 | auto &col = (*schema_)[i]; 26 | if (col.is_indexed()) { 27 | index_archivers_.at(col.index_id())->archive(offset); 28 | } 29 | } 30 | } 31 | 32 | void index_log_archiver::init_new_archivers() { 33 | for (size_t i = 0; i < schema_->size(); i++) { 34 | auto &col = (*schema_)[i]; 35 | if (col.is_indexed()) { 36 | auto id = col.index_id(); 37 | if (index_archivers_.size() <= id) { 38 | index_archivers_.resize(id + 1); 39 | } 40 | if (index_archivers_[id] == nullptr) { 41 | std::string index_path = path_ + "/index_" + std::to_string(id) + "/"; 42 | file_utils::create_dir(index_path); 43 | index_archivers_[id] = new index_archiver(index_path, indexes_->at(id), col); 44 | } 45 | } 46 | } 47 | } 48 | 49 | } 50 | } -------------------------------------------------------------------------------- /libconfluo/src/archival/io/incremental_file_offset.cc: -------------------------------------------------------------------------------- 1 | #include "archival/io/incremental_file_offset.h" 2 | 3 | namespace confluo { 4 | namespace archival { 5 | 6 | incremental_file_offset::incremental_file_offset() 7 | : path_(""), 8 | off_(0) { 9 | } 10 | 11 | incremental_file_offset::incremental_file_offset(std::string path, size_t offset) 12 | : path_(path), 13 | off_(offset) { 14 | } 15 | 16 | std::string incremental_file_offset::path() const { 17 | return path_; 18 | } 19 | 20 | size_t incremental_file_offset::offset() const { 21 | return off_; 22 | } 23 | 24 | } 25 | } -------------------------------------------------------------------------------- /libconfluo/src/archival/io/incremental_file_stream.cc: -------------------------------------------------------------------------------- 1 | #include "archival/io/incremental_file_stream.h" 2 | 3 | namespace confluo { 4 | namespace archival { 5 | 6 | incremental_file_stream::incremental_file_stream() 7 | : incremental_file_stream("", "") { 8 | } 9 | 10 | incremental_file_stream::incremental_file_stream(const std::string &path, const std::string &file_prefix) 11 | : file_num_(0), 12 | dir_path_(path), 13 | file_prefix_(file_prefix) { 14 | } 15 | 16 | std::string incremental_file_stream::cur_path() { 17 | return dir_path_ + "/" + file_prefix_ + "_" + std::to_string(file_num_) + ".dat"; 18 | } 19 | 20 | std::string incremental_file_stream::transaction_log_path() { 21 | return dir_path_ + "/" + file_prefix_ + "_transaction_log.dat"; 22 | } 23 | 24 | void incremental_file_stream::truncate(incremental_file_offset incr_file_off, size_t transaction_log_off) { 25 | // TODO delete succeeding files as well 26 | utils::file_utils::truncate_file(incr_file_off.path(), incr_file_off.offset()); 27 | utils::file_utils::truncate_file(transaction_log_path(), transaction_log_off); 28 | } 29 | 30 | } 31 | } -------------------------------------------------------------------------------- /libconfluo/src/compression/confluo_encoder.cc: -------------------------------------------------------------------------------- 1 | #include "compression/confluo_encoder.h" 2 | 3 | namespace confluo { 4 | namespace compression { 5 | 6 | unique_byte_array confluo_encoder::encode(void *ptr, size_t size, uint8_t encoding) { 7 | switch (encoding) { 8 | case encoding_type::D_UNENCODED: { 9 | return unique_byte_array(reinterpret_cast(ptr), size, no_op_delete); 10 | } 11 | case encoding_type::D_ELIAS_GAMMA: { 12 | uint64_t *casted = reinterpret_cast(ptr); 13 | size_t array_len = size / sizeof(uint64_t); 14 | return delta_encoder::encode(casted, array_len); 15 | } 16 | case encoding_type::D_LZ4: { 17 | uint8_t *casted = reinterpret_cast(ptr); 18 | return lz4_encoder<>::encode(casted, size); 19 | } 20 | default : { 21 | THROW(illegal_state_exception, "Invalid encoding type!"); 22 | } 23 | } 24 | } 25 | 26 | void confluo_encoder::no_op_delete(uint8_t *) {} 27 | 28 | } 29 | } -------------------------------------------------------------------------------- /libconfluo/src/container/cursor/alert_cursor.cc: -------------------------------------------------------------------------------- 1 | #include "container/cursor/alert_cursor.h" 2 | 3 | namespace confluo { 4 | 5 | trigger_alert_cursor::trigger_alert_cursor(const trigger_alert_cursor::alert_list &alerts, 6 | const std::string &trigger_name, 7 | size_t batch_size) 8 | : alert_cursor(batch_size), 9 | cur_(alerts.begin()), 10 | end_(alerts.end()), 11 | trigger_name_(trigger_name) { 12 | init(); 13 | } 14 | 15 | size_t trigger_alert_cursor::load_next_batch() { 16 | size_t i = 0; 17 | for (; i < current_batch_.size() && cur_ != end_; ++i, ++cur_) { 18 | current_batch_[i] = *cur_; 19 | if (trigger_name_ != "" && trigger_name_ != current_batch_[i].trigger_name) { 20 | --i; 21 | } 22 | } 23 | return i; 24 | } 25 | 26 | } -------------------------------------------------------------------------------- /libconfluo/src/container/cursor/offset_cursors.cc: -------------------------------------------------------------------------------- 1 | #include "container/cursor/offset_cursors.h" 2 | 3 | namespace confluo { 4 | 5 | data_log_cursor::data_log_cursor(uint64_t version, uint64_t record_size, size_t batch_size) 6 | : offset_cursor(batch_size), 7 | current_offset_(0), 8 | version_(version), 9 | record_size_(record_size) { 10 | init(); 11 | } 12 | 13 | size_t data_log_cursor::load_next_batch() { 14 | size_t i = 0; 15 | for (; i < current_batch_.size() && current_offset_ < version_; i++, current_offset_ += record_size_) { 16 | current_batch_[i] = current_offset_; 17 | } 18 | return i; 19 | } 20 | 21 | } -------------------------------------------------------------------------------- /libconfluo/src/container/data_log.cc: -------------------------------------------------------------------------------- 1 | #include "container/data_log.h" 2 | 3 | namespace confluo { 4 | 5 | const size_t data_log_constants::MAX_BUCKETS; 6 | const size_t data_log_constants::BUCKET_SIZE; 7 | const size_t data_log_constants::BUFFER_SIZE; 8 | 9 | } -------------------------------------------------------------------------------- /libconfluo/src/container/reflog.cc: -------------------------------------------------------------------------------- 1 | #include "container/reflog.h" 2 | 3 | namespace confluo { 4 | 5 | const size_t reflog_constants::NCONTAINERS; 6 | const size_t reflog_constants::BUCKET_SIZE; 7 | 8 | } -------------------------------------------------------------------------------- /libconfluo/src/container/sketch/hash_manager.cc: -------------------------------------------------------------------------------- 1 | #include "container/sketch/hash_manager.h" 2 | 3 | namespace confluo { 4 | namespace sketch { 5 | 6 | const size_t pairwise_indep_hash::PRIME = 39916801UL; 7 | 8 | pairwise_indep_hash::pairwise_indep_hash() 9 | : pairwise_indep_hash(0, 0) { 10 | } 11 | 12 | pairwise_indep_hash::pairwise_indep_hash(size_t a, size_t b) 13 | : a_(a), 14 | b_(b) { 15 | } 16 | 17 | pairwise_indep_hash pairwise_indep_hash::generate_random() { 18 | return { utils::rand_utils::rand_uint64(PRIME), utils::rand_utils::rand_uint64(PRIME) }; 19 | } 20 | 21 | hash_manager::hash_manager(size_t num_hashes) 22 | : hashes_() { 23 | this->guarantee_initialized(num_hashes); 24 | } 25 | 26 | void hash_manager::guarantee_initialized(size_t num_hashes) { 27 | size_t cur_size = hashes_.size(); 28 | size_t num_new_hashes = num_hashes > cur_size ? num_hashes - cur_size : 0; 29 | for (size_t i = 0; i < num_new_hashes; i++) { 30 | hashes_.push_back(pairwise_indep_hash::generate_random()); 31 | } 32 | } 33 | 34 | size_t hash_manager::size() const { 35 | return hashes_.size(); 36 | } 37 | 38 | size_t hash_manager::storage_size() const { 39 | return sizeof(pairwise_indep_hash) * hashes_.capacity(); 40 | } 41 | 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /libconfluo/src/parser/aggregate_parser.cc: -------------------------------------------------------------------------------- 1 | #include "parser/aggregate_parser.h" 2 | 3 | namespace confluo { 4 | namespace parser { 5 | 6 | parsed_aggregate parse_aggregate(const std::string &t) { 7 | using boost::spirit::ascii::space; 8 | typedef std::string::const_iterator iterator_type; 9 | typedef aggregate_parser grammar; 10 | grammar g; 11 | std::string::const_iterator iter = t.begin(); 12 | std::string::const_iterator end = t.end(); 13 | parsed_aggregate pt; 14 | bool r = phrase_parse(iter, end, g, space, pt); 15 | if (iter != end || !r) { 16 | std::string rest(iter, end); 17 | throw confluo::parse_exception(std::string("Parse failed at ") + rest); 18 | } 19 | return pt; 20 | } 21 | 22 | } 23 | } -------------------------------------------------------------------------------- /libconfluo/src/parser/schema_parser.cc: -------------------------------------------------------------------------------- 1 | #include "parser/schema_parser.h" 2 | 3 | namespace confluo { 4 | namespace parser { 5 | 6 | std::vector parse_schema(const std::string &s) { 7 | using boost::spirit::ascii::space; 8 | typedef std::string::const_iterator iterator_type; 9 | typedef schema_parser grammar; 10 | grammar g; 11 | std::string::const_iterator iter = s.begin(); 12 | std::string::const_iterator end = s.end(); 13 | kv_list kvs; 14 | bool r = phrase_parse(iter, end, g, space, kvs); 15 | if (iter != end || !r) { 16 | std::string rest(iter, end); 17 | throw parse_exception(std::string("Parse failed at ") + rest); 18 | } 19 | schema_builder b; 20 | for (const auto &kv : kvs) { 21 | b.add_column(data_type::from_string(kv.second), kv.first); 22 | } 23 | return b.get_columns(); 24 | } 25 | 26 | } 27 | } -------------------------------------------------------------------------------- /libconfluo/src/parser/trigger_parser.cc: -------------------------------------------------------------------------------- 1 | #include "parser/trigger_parser.h" 2 | 3 | namespace confluo { 4 | namespace parser { 5 | 6 | parsed_trigger parse_trigger(const std::string &t) { 7 | using boost::spirit::ascii::space; 8 | typedef std::string::const_iterator iterator_type; 9 | typedef trigger_parser grammar; 10 | grammar g; 11 | std::string::const_iterator iter = t.begin(); 12 | std::string::const_iterator end = t.end(); 13 | parsed_trigger pt; 14 | bool r = phrase_parse(iter, end, g, space, pt); 15 | if (iter != end || !r) { 16 | std::string rest(iter, end); 17 | throw confluo::parse_exception(std::string("Parse failed at ") + rest); 18 | } 19 | return pt; 20 | } 21 | 22 | } 23 | } -------------------------------------------------------------------------------- /libconfluo/src/read_tail.cc: -------------------------------------------------------------------------------- 1 | #include "read_tail.h" 2 | #include "io_utils.h" 3 | 4 | namespace confluo { 5 | 6 | read_tail::read_tail() { 7 | read_tail_ = nullptr; 8 | mode_ = storage::IN_MEMORY; 9 | } 10 | 11 | read_tail::read_tail(const std::string &data_path, const storage::storage_mode &mode, bool load) { 12 | init(data_path, mode, load); 13 | } 14 | 15 | void read_tail::init(const std::string &data_path, const storage::storage_mode &mode, bool load) { 16 | mode_ = mode; 17 | auto path = data_path + "/read_tail"; 18 | uint64_t value = 0; 19 | if (load) { 20 | std::ifstream rt_ifstream(path); 21 | value = io_utils::read(rt_ifstream); 22 | } 23 | auto storage_func = storage::storage_mode_functions::STORAGE_FNS()[mode_]; 24 | read_tail_ = (atomic::type *) storage_func.allocate(path, sizeof(uint64_t)); 25 | atomic::store(read_tail_, value); 26 | } 27 | 28 | uint64_t read_tail::get() const { 29 | return atomic::load(read_tail_); 30 | } 31 | 32 | void read_tail::advance(uint64_t old_tail, uint32_t bytes) { 33 | uint64_t expected = old_tail; 34 | while (!atomic::weak::cas(read_tail_, &expected, old_tail + bytes)) { 35 | expected = old_tail; 36 | std::this_thread::yield(); 37 | } 38 | storage::storage_mode_functions::STORAGE_FNS()[mode_].flush(read_tail_, sizeof(uint64_t)); 39 | } 40 | 41 | } -------------------------------------------------------------------------------- /libconfluo/src/schema/field.cc: -------------------------------------------------------------------------------- 1 | #include "schema/field.h" 2 | 3 | namespace confluo { 4 | 5 | field_t::field_t(uint16_t idx, 6 | const data_type &type, 7 | void *data, 8 | bool indexed, 9 | uint16_t index_id, 10 | double index_bucket_size) 11 | : idx_(idx), 12 | value_(type, data), 13 | indexed_(indexed), 14 | index_bucket_size_(index_bucket_size), 15 | index_id_(index_id) { 16 | } 17 | 18 | uint16_t field_t::idx() const { 19 | return idx_; 20 | } 21 | 22 | const data_type &field_t::type() const { 23 | return value_.type(); 24 | } 25 | 26 | const immutable_value &field_t::value() const { 27 | return value_; 28 | } 29 | 30 | bool field_t::is_indexed() const { 31 | return indexed_; 32 | } 33 | 34 | uint16_t field_t::index_id() const { 35 | return index_id_; 36 | } 37 | 38 | byte_string field_t::get_key() const { 39 | return value_.to_key(index_bucket_size_); 40 | } 41 | 42 | std::string field_t::to_string() const { 43 | return value_.to_string(); 44 | } 45 | 46 | } -------------------------------------------------------------------------------- /libconfluo/src/schema/index_state.cc: -------------------------------------------------------------------------------- 1 | #include "schema/index_state.h" 2 | 3 | namespace confluo { 4 | 5 | const uint8_t index_state_t::UNINDEXED; 6 | const uint8_t index_state_t::INDEXING; 7 | const uint8_t index_state_t::INDEXED; 8 | 9 | index_state_t::index_state_t() 10 | : state_(UNINDEXED), 11 | id_(UINT16_MAX), 12 | bucket_size_(1) {} 13 | 14 | index_state_t::index_state_t(const index_state_t &other) 15 | : state_(atomic::load(&other.state_)), 16 | id_(other.id_), 17 | bucket_size_(other.bucket_size_) {} 18 | 19 | uint16_t index_state_t::id() const { 20 | return id_; 21 | } 22 | 23 | double index_state_t::bucket_size() const { 24 | return bucket_size_; 25 | } 26 | 27 | index_state_t &index_state_t::operator=(const index_state_t &other) { 28 | atomic::init(&state_, atomic::load(&other.state_)); 29 | id_ = other.id_; 30 | return *this; 31 | } 32 | 33 | bool index_state_t::is_indexed() const { 34 | return atomic::load(&state_) == INDEXED; 35 | } 36 | 37 | bool index_state_t::set_indexing() { 38 | uint8_t expected = UNINDEXED; 39 | return atomic::strong::cas(&state_, &expected, INDEXING); 40 | } 41 | 42 | void index_state_t::set_indexed(uint16_t index_id, double bucket_size) { 43 | id_ = index_id; 44 | bucket_size_ = bucket_size; 45 | atomic::store(&state_, INDEXED); 46 | } 47 | 48 | void index_state_t::set_unindexed() { 49 | atomic::store(&state_, UNINDEXED); 50 | } 51 | 52 | bool index_state_t::disable_indexing() { 53 | uint8_t expected = INDEXED; 54 | return atomic::strong::cas(&state_, &expected, UNINDEXED); 55 | } 56 | 57 | } -------------------------------------------------------------------------------- /libconfluo/src/schema/record_batch.cc: -------------------------------------------------------------------------------- 1 | #include "schema/record_batch.h" 2 | 3 | namespace confluo { 4 | 5 | const int64_t record_batch_builder::TIME_BLOCK; 6 | 7 | int64_t record_batch::start_time_block() const { 8 | return blocks.front().time_block; 9 | } 10 | 11 | int64_t record_batch::end_time_block() const { 12 | return blocks.back().time_block; 13 | } 14 | 15 | record_batch_builder::record_batch_builder(const schema_t &schema) 16 | : schema_(schema) { 17 | } 18 | 19 | void record_batch_builder::add_record(const void *data) { 20 | size_t record_size = schema_.record_size(); 21 | int64_t ts = *reinterpret_cast(data); 22 | int64_t time_block = ts / TIME_BLOCK; 23 | batch_sizes_[time_block] += record_size; 24 | batch_[time_block].write(reinterpret_cast(data), record_size); 25 | } 26 | 27 | void record_batch_builder::add_record(const std::vector &rec) { 28 | void *data = schema_.record_vector_to_data(rec); 29 | add_record(data); 30 | delete[] reinterpret_cast(data); 31 | } 32 | 33 | record_batch record_batch_builder::get_batch() { 34 | record_batch batch; 35 | batch.blocks.resize(batch_.size()); 36 | batch.nrecords = 0; 37 | size_t i = 0; 38 | for (auto &entry : batch_) { 39 | batch.blocks[i].time_block = entry.first; 40 | batch.blocks[i].data = entry.second.str(); 41 | batch.blocks[i].nrecords = batch.blocks[i].data.size() 42 | / schema_.record_size(); 43 | batch.nrecords += batch.blocks[i].nrecords; 44 | i++; 45 | } 46 | return batch; 47 | } 48 | 49 | } -------------------------------------------------------------------------------- /libconfluo/src/schema/schema_snapshot.cc: -------------------------------------------------------------------------------- 1 | #include "schema/schema_snapshot.h" 2 | 3 | namespace confluo { 4 | 5 | void schema_snapshot::add_column(const column_snapshot &snap) { 6 | snapshot_.push_back(snap); 7 | } 8 | 9 | void schema_snapshot::add_column(column_snapshot &&snap) { 10 | snapshot_.push_back(std::move(snap)); 11 | } 12 | 13 | immutable_value schema_snapshot::get(void *data, uint32_t i) const { 14 | return immutable_value(snapshot_[i].type, reinterpret_cast(data) + snapshot_[i].offset); 15 | } 16 | 17 | byte_string schema_snapshot::time_key(int64_t time_block) const { 18 | return primitive_types::LONG_TYPE().key_transform()(immutable_raw_data(reinterpret_cast(&time_block), 19 | primitive_types::LONG_TYPE().size), 1.0); 20 | } 21 | 22 | byte_string schema_snapshot::get_key(void *ptr, uint32_t i) const { 23 | return snapshot_[i].type.key_transform()( 24 | immutable_raw_data(reinterpret_cast(ptr) + snapshot_[i].offset, snapshot_[i].type.size), 25 | snapshot_[i].index_bucket_size); 26 | } 27 | 28 | int64_t schema_snapshot::get_timestamp(void *ptr) const { 29 | return *reinterpret_cast(ptr); 30 | } 31 | 32 | bool schema_snapshot::is_indexed(size_t i) const { 33 | return snapshot_[i].indexed; 34 | } 35 | 36 | uint32_t schema_snapshot::index_id(size_t i) const { 37 | return snapshot_[i].index_id; 38 | } 39 | 40 | double schema_snapshot::index_bucket_size(size_t i) const { 41 | return snapshot_[i].index_bucket_size; 42 | } 43 | 44 | size_t schema_snapshot::num_columns() const { 45 | return snapshot_.size(); 46 | } 47 | 48 | } -------------------------------------------------------------------------------- /libconfluo/src/storage/memory_stat.cc: -------------------------------------------------------------------------------- 1 | #include "storage/memory_stat.h" 2 | 3 | namespace confluo { 4 | namespace storage { 5 | 6 | memory_stat::memory_stat() 7 | : memory_used_(0) { 8 | } 9 | 10 | void memory_stat::increment(size_t size) { 11 | atomic::faa(&memory_used_, size); 12 | } 13 | 14 | void memory_stat::decrement(size_t size) { 15 | atomic::fas(&memory_used_, size); 16 | } 17 | 18 | size_t memory_stat::get() { 19 | return atomic::load(&memory_used_); 20 | } 21 | 22 | } 23 | } -------------------------------------------------------------------------------- /libconfluo/src/storage/ptr_aux_block.cc: -------------------------------------------------------------------------------- 1 | #include "storage/ptr_aux_block.h" 2 | 3 | namespace confluo { 4 | namespace storage { 5 | 6 | const uint8_t state_type::D_IN_MEMORY; 7 | const uint8_t state_type::D_ARCHIVED; 8 | const uint8_t encoding_type::D_UNENCODED; 9 | 10 | ptr_aux_block::ptr_aux_block() 11 | : ptr_aux_block(state_type::D_IN_MEMORY, encoding_type::D_UNENCODED) { 12 | } 13 | 14 | ptr_aux_block::ptr_aux_block(uint8_t state, uint8_t encoding) 15 | : encoding_(encoding), 16 | state_(state) { 17 | } 18 | 19 | ptr_aux_block ptr_aux_block::get(ptr_metadata *metadata) { 20 | uint8_t aux = metadata->aux_; 21 | return *reinterpret_cast(&aux); 22 | } 23 | 24 | } 25 | } -------------------------------------------------------------------------------- /libconfluo/src/storage/ptr_metadata.cc: -------------------------------------------------------------------------------- 1 | #include "storage/ptr_metadata.h" 2 | 3 | namespace confluo { 4 | namespace storage { 5 | 6 | const uint8_t alloc_type::D_DEFAULT; 7 | const uint8_t alloc_type::D_MMAP; 8 | 9 | ptr_metadata *ptr_metadata::get(void *ptr) { 10 | return static_cast(ptr) - 1; 11 | } 12 | 13 | } 14 | } -------------------------------------------------------------------------------- /libconfluo/src/storage/storage.cc: -------------------------------------------------------------------------------- 1 | #include "storage/storage.h" 2 | 3 | namespace confluo { 4 | namespace storage { 5 | 6 | storage_functions &storage_mode_functions::IN_MEMORY_FNS() { 7 | static storage_functions fns 8 | {storage_mode::IN_MEMORY, in_memory::allocate, in_memory::allocate_bucket, in_memory::free_mem, 9 | in_memory::flush}; 10 | return fns; 11 | } 12 | 13 | storage_functions &storage_mode_functions::DURABLE_RELAXED_FNS() { 14 | static storage_functions fns 15 | {storage_mode::DURABLE_RELAXED, durable_relaxed::allocate, durable_relaxed::allocate_bucket, 16 | durable_relaxed::free, 17 | durable_relaxed::flush}; 18 | return fns; 19 | } 20 | 21 | storage_functions &storage_mode_functions::DURABLE_FNS() { 22 | static storage_functions 23 | fns{storage_mode::DURABLE, durable::allocate, durable::allocate_bucket, durable::free, durable::flush}; 24 | return fns; 25 | } 26 | 27 | storage_functions *storage_mode_functions::STORAGE_FNS() { 28 | static storage_functions fns[3] = 29 | {storage_mode_functions::IN_MEMORY_FNS(), storage_mode_functions::DURABLE_RELAXED_FNS(), 30 | storage_mode_functions::DURABLE_FNS()}; 31 | return fns; 32 | } 33 | 34 | } 35 | } -------------------------------------------------------------------------------- /libconfluo/src/trigger.cc: -------------------------------------------------------------------------------- 1 | #include "trigger.h" 2 | 3 | namespace confluo { 4 | 5 | monitor::trigger::trigger(const std::string &name, 6 | const std::string &aggregate_name, 7 | reational_op_id op, 8 | const numeric &threshold, 9 | const uint64_t periodicity_ms) 10 | : name_(name), 11 | aggregate_name_(aggregate_name), 12 | op_(op), 13 | threshold_(threshold), 14 | periodicity_ms_(periodicity_ms), 15 | is_valid_(true) { 16 | } 17 | 18 | std::string monitor::trigger::name() const { 19 | return name_; 20 | } 21 | 22 | std::string monitor::trigger::expr() const { 23 | return aggregate_name_ + " " + relop_utils::op_to_str(op_) + " " + threshold_.to_string(); 24 | } 25 | 26 | std::string monitor::trigger::aggregate_name() const { 27 | return aggregate_name_; 28 | } 29 | 30 | reational_op_id monitor::trigger::op() const { 31 | return op_; 32 | } 33 | 34 | const numeric monitor::trigger::threshold() const { 35 | return threshold_; 36 | } 37 | 38 | uint64_t monitor::trigger::periodicity_ms() const { 39 | return periodicity_ms_; 40 | } 41 | 42 | bool monitor::trigger::invalidate() { 43 | bool expected = true; 44 | return atomic::strong::cas(&is_valid_, &expected, false); 45 | } 46 | 47 | bool monitor::trigger::is_valid() const { 48 | return atomic::load(&is_valid_); 49 | } 50 | 51 | std::string monitor::trigger::to_string() const { 52 | return "{ name: \"" + name() + "\", expression: \"" + expr() + "\"}"; 53 | } 54 | 55 | } -------------------------------------------------------------------------------- /libconfluo/src/types/type_manager.cc: -------------------------------------------------------------------------------- 1 | #include "types/type_manager.h" 2 | #include "types/primitive_types.h" 3 | 4 | namespace confluo { 5 | 6 | size_t type_manager::register_type(const type_properties &type_def) { 7 | size_t id = data_type_properties::instance().size(); 8 | data_type_properties::instance().push_back(type_def); 9 | return id; 10 | } 11 | 12 | data_type type_manager::get_type(const std::string &type_name, size_t size) { 13 | return data_type(type_name, size); 14 | } 15 | 16 | data_type type_manager::get_type(size_t id, size_t size) { 17 | if (id < data_type_properties::instance().size()) { 18 | return data_type(static_cast(id), size); 19 | } 20 | return data_type(); 21 | } 22 | 23 | } -------------------------------------------------------------------------------- /libconfluo/test/container/bitmap/bitmap_array_test.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_TEST_BITMAP_ARRAY_TEST_H_ 2 | #define CONFLUO_TEST_BITMAP_ARRAY_TEST_H_ 3 | 4 | #include "container/bitmap/bitmap_array.h" 5 | #include "bit_utils.h" 6 | #include "gtest/gtest.h" 7 | 8 | using namespace confluo; 9 | 10 | class BitmapArrayTest : public testing::Test { 11 | public: 12 | const uint64_t kArraySize = (1024ULL * 1024ULL); // 1 KBytes 13 | const uint8_t kBitWidth = 20; // 20 bits 14 | }; 15 | 16 | TEST_F(BitmapArrayTest, UnsizedBitmapArrayTest) { 17 | unsized_bitmap_array array(kArraySize, kBitWidth); 18 | for (uint64_t i = 0; i < kArraySize; i++) { 19 | array[i] = i; 20 | } 21 | 22 | for (uint64_t i = 0; i < kArraySize; i++) { 23 | ASSERT_EQ(i, array[i]); 24 | } 25 | } 26 | 27 | TEST_F(BitmapArrayTest, UnsignedBitmapArrayTest) { 28 | unsigned_bitmap_array array(kArraySize, kBitWidth); 29 | for (uint64_t i = 0; i < kArraySize; i++) { 30 | array[i] = i; 31 | } 32 | 33 | for (uint64_t i = 0; i < kArraySize; i++) { 34 | ASSERT_EQ(i, array[i]); 35 | } 36 | } 37 | 38 | TEST_F(BitmapArrayTest, SignedBitmapArrayTest) { 39 | signed_bitmap_array array(kArraySize, kBitWidth); 40 | for (int64_t i = 0; i < static_cast(kArraySize); i++) { 41 | if (i % 2 == 0) { 42 | array[i] = i; 43 | } else { 44 | array[i] = -i; 45 | } 46 | } 47 | 48 | for (int64_t i = 0; i < static_cast(kArraySize); i++) { 49 | if (i % 2 == 0) { 50 | ASSERT_EQ(i, array[i]); 51 | } else { 52 | ASSERT_EQ(-i, array[i]); 53 | } 54 | } 55 | } 56 | 57 | #endif /* CONFLUO_TEST_BITMAP_ARRAY_TEST_H_ */ 58 | -------------------------------------------------------------------------------- /libconfluo/test/container/bitmap/bitmap_test.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_TEST_BITMAP_TEST_H_ 2 | #define CONFLUO_TEST_BITMAP_TEST_H_ 3 | 4 | #include "container/bitmap/bitmap.h" 5 | #include "bit_utils.h" 6 | #include "gtest/gtest.h" 7 | 8 | using namespace confluo; 9 | using namespace utils; 10 | 11 | class BitmapTest : public testing::Test { 12 | public: 13 | const uint64_t kBitmapSize = (1024ULL * 1024ULL); // 1 KBits 14 | 15 | protected: 16 | virtual void SetUp() override { 17 | bitmap_ = new bitmap(kBitmapSize); 18 | } 19 | 20 | virtual void TearDown() override { 21 | delete bitmap_; 22 | } 23 | 24 | bitmap *bitmap_; 25 | }; 26 | 27 | TEST_F(BitmapTest, GetSetBitTest) { 28 | for (uint64_t i = 0; i < kBitmapSize; i++) { 29 | if (i % 2 == 0) { 30 | bitmap_->set_bit(i); 31 | } 32 | } 33 | 34 | for (uint64_t i = 0; i < kBitmapSize; i++) { 35 | bool val = bitmap_->get_bit(i); 36 | ASSERT_EQ(i % 2 == 0, val); 37 | } 38 | } 39 | 40 | TEST_F(BitmapTest, GetSetValPosTest) { 41 | uint64_t pos = 0; 42 | for (uint64_t i = 0; i < 10000; i++) { 43 | bitmap_->set_val_pos(pos, i, bit_utils::bit_width(i)); 44 | pos += bit_utils::bit_width(i); 45 | } 46 | 47 | pos = 0; 48 | for (uint64_t i = 0; i < 10000; i++) { 49 | uint64_t val = bitmap_->get_val_pos(pos, bit_utils::bit_width(i)); 50 | ASSERT_EQ(i, val); 51 | pos += bit_utils::bit_width(i); 52 | } 53 | } 54 | 55 | #endif /* CONFLUO_TEST_BITMAP_TEST_H_ */ 56 | -------------------------------------------------------------------------------- /libconfluo/test/container/flatten_test.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_TEST_FLATTEN_TEST_H_ 2 | #define CONFLUO_TEST_FLATTEN_TEST_H_ 3 | 4 | #include "container/flatten.h" 5 | 6 | #include "gtest/gtest.h" 7 | 8 | #include "container/radix_tree.h" 9 | 10 | using namespace ::confluo; 11 | 12 | class FlattenTest : public testing::Test { 13 | }; 14 | 15 | TEST_F(FlattenTest, VectorTest) { 16 | typedef std::vector> vector_t; 17 | 18 | vector_t vec = {{1, 2, 3}, {}, {4, 5}, {6}, {}, {}, {7, 8, 9}}; 19 | flattened_container fvec(vec); 20 | 21 | int n = 1; 22 | for (auto it = fvec.begin(); it != fvec.end(); ++it) { 23 | ASSERT_EQ(n, *it); 24 | n++; 25 | } 26 | } 27 | 28 | TEST_F(FlattenTest, RadixTreeTest) { 29 | radix_index tree(sizeof(int32_t), 256); 30 | for (int32_t i = 0; i < 256; i++) 31 | tree.insert(byte_string(i * 8), i); 32 | 33 | auto res1 = tree.range_lookup_reflogs(byte_string(0), byte_string(16)); 34 | radix_index::rt_result res1f(res1); 35 | ASSERT_EQ(static_cast(3), res1f.count()); 36 | uint64_t i = 0; 37 | for (uint64_t val : res1f) { 38 | ASSERT_EQ(i, val); 39 | i++; 40 | } 41 | 42 | auto res2 = tree.range_lookup_reflogs(byte_string(1), byte_string(15)); 43 | radix_index::rt_result res2f(res2); 44 | ASSERT_EQ(static_cast(1), res2f.count()); 45 | i = 1; 46 | for (uint64_t val : res2f) { 47 | ASSERT_EQ(i, val); 48 | i++; 49 | } 50 | } 51 | 52 | #endif /* CONFLUO_TEST_FLATTEN_TEST_H_ */ 53 | -------------------------------------------------------------------------------- /libconfluo/test/parser/aggregate_parser_test.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_TEST_AGGREGATE_PARSER_TEST_H_ 2 | #define CONFLUO_TEST_AGGREGATE_PARSER_TEST_H_ 3 | 4 | #include "parser/aggregate_parser.h" 5 | 6 | #include "gtest/gtest.h" 7 | 8 | using namespace ::confluo::parser; 9 | 10 | class AggregateParserTest : public testing::Test { 11 | }; 12 | 13 | TEST_F(AggregateParserTest, ParseTest) { 14 | auto t1 = parse_aggregate("SUM(c)"); 15 | ASSERT_EQ("SUM", t1.agg); 16 | ASSERT_EQ("c", t1.field_name); 17 | 18 | auto t2 = parse_aggregate("MIN(d)"); 19 | ASSERT_EQ("MIN", t2.agg); 20 | ASSERT_EQ("d", t2.field_name); 21 | 22 | auto t3 = parse_aggregate("MAX(e)"); 23 | ASSERT_EQ("MAX", t3.agg); 24 | ASSERT_EQ("e", t3.field_name); 25 | 26 | auto t4 = parse_aggregate("CNT(d)"); 27 | ASSERT_EQ("CNT", t4.agg); 28 | ASSERT_EQ("d", t4.field_name); 29 | } 30 | 31 | #endif /* CONFLUO_TEST_AGGREGATE_PARSER_TEST_H_ */ 32 | -------------------------------------------------------------------------------- /libconfluo/test/parser/trigger_parser_test.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_TEST_TRIGGER_PARSER_TEST_H_ 2 | #define CONFLUO_TEST_TRIGGER_PARSER_TEST_H_ 3 | 4 | #include "parser/trigger_parser.h" 5 | 6 | #include "gtest/gtest.h" 7 | 8 | using namespace ::confluo::parser; 9 | 10 | class TriggerParserTest : public testing::Test { 11 | }; 12 | 13 | TEST_F(TriggerParserTest, ParseTest) { 14 | auto t1 = parse_trigger("SUM_c < 10"); 15 | ASSERT_EQ("SUM_c", t1.aggregate_name); 16 | ASSERT_EQ("<", t1.relop); 17 | ASSERT_EQ("10", t1.threshold); 18 | 19 | auto t2 = parse_trigger("MIN_d >= 200"); 20 | ASSERT_EQ("MIN_d", t2.aggregate_name); 21 | ASSERT_EQ(">=", t2.relop); 22 | ASSERT_EQ("200", t2.threshold); 23 | 24 | auto t3 = parse_trigger("MAX_e == 4000"); 25 | ASSERT_EQ("MAX_e", t3.aggregate_name); 26 | ASSERT_EQ("==", t3.relop); 27 | ASSERT_EQ("4000", t3.threshold); 28 | 29 | auto t4 = parse_trigger("CNT_d <= 10"); 30 | ASSERT_EQ("CNT_d", t4.aggregate_name); 31 | ASSERT_EQ("<=", t4.relop); 32 | ASSERT_EQ("10", t4.threshold); 33 | } 34 | 35 | #endif /* CONFLUO_TEST_TRIGGER_PARSER_TEST_H_ */ 36 | -------------------------------------------------------------------------------- /libconfluo/test/schema/column_test.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_TEST_COLUMN_TEST_H_ 2 | #define CONFLUO_TEST_COLUMN_TEST_H_ 3 | 4 | #include "schema/column.h" 5 | #include "gtest/gtest.h" 6 | 7 | using namespace ::confluo; 8 | 9 | class ColumnTest : public testing::Test { 10 | }; 11 | 12 | TEST_F(ColumnTest, GetterTest) { 13 | column_t col(0, 14 | 0, 15 | primitive_types::INT_TYPE(), 16 | "test", 17 | mutable_value(primitive_types::INT_TYPE(), primitive_types::INT_TYPE().min()), 18 | mutable_value(primitive_types::INT_TYPE(), primitive_types::INT_TYPE().max())); 19 | 20 | ASSERT_EQ(0, col.idx()); 21 | ASSERT_EQ(0, col.offset()); 22 | ASSERT_EQ("TEST", col.name()); 23 | ASSERT_TRUE(primitive_types::INT_TYPE() == col.type()); 24 | ASSERT_FALSE(col.is_indexed()); 25 | } 26 | 27 | TEST_F(ColumnTest, IndexStateTest) { 28 | column_t col(0, 29 | 0, 30 | primitive_types::INT_TYPE(), 31 | "test", 32 | mutable_value(primitive_types::INT_TYPE(), primitive_types::INT_TYPE().min()), 33 | mutable_value(primitive_types::INT_TYPE(), primitive_types::INT_TYPE().max())); 34 | ASSERT_FALSE(col.is_indexed()); 35 | 36 | bool success = col.set_indexing(); 37 | ASSERT_TRUE(success); 38 | 39 | col.set_indexed(3, 0.1); 40 | ASSERT_TRUE(col.is_indexed()); 41 | ASSERT_EQ(3, col.index_id()); 42 | ASSERT_EQ(static_cast(0.1), col.index_bucket_size()); 43 | 44 | col.set_unindexed(); 45 | ASSERT_FALSE(col.is_indexed()); 46 | } 47 | 48 | #endif /* CONFLUO_TEST_COLUMN_TEST_H_ */ 49 | -------------------------------------------------------------------------------- /libconfluo/test/schema/index_state_test.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_TEST_INDEX_STATE_TEST_H_ 2 | #define CONFLUO_TEST_INDEX_STATE_TEST_H_ 3 | 4 | #include "schema/index_state.h" 5 | 6 | #include "gtest/gtest.h" 7 | 8 | using namespace ::confluo; 9 | 10 | class IndexStateTest : public testing::Test { 11 | }; 12 | 13 | TEST_F(IndexStateTest, StateTransitionTest) { 14 | index_state_t state; 15 | ASSERT_FALSE(state.is_indexed()); 16 | 17 | bool success = state.set_indexing(); 18 | ASSERT_TRUE(success); 19 | 20 | state.set_indexed(3, 0.1); 21 | ASSERT_TRUE(state.is_indexed()); 22 | ASSERT_EQ(3, state.id()); 23 | ASSERT_EQ(static_cast(0.1), state.bucket_size()); 24 | 25 | state.set_unindexed(); 26 | ASSERT_FALSE(state.is_indexed()); 27 | } 28 | 29 | #endif /* CONFLUO_TEST_INDEX_STATE_TEST_H_ */ 30 | -------------------------------------------------------------------------------- /libconfluo/test/storage/memory_stat_test.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_TEST_MEMORY_STAT_TEST_H_ 2 | #define CONFLUO_TEST_MEMORY_STAT_TEST_H_ 3 | 4 | #include "storage/memory_stat.h" 5 | #include "gtest/gtest.h" 6 | 7 | using namespace ::confluo; 8 | 9 | class MemoryStatTest : public testing::Test { 10 | }; 11 | 12 | TEST_F(MemoryStatTest, IncrementGetTest) { 13 | 14 | storage::memory_stat stat; 15 | ASSERT_EQ(stat.get(), 0); 16 | 17 | stat.increment(8); 18 | stat.increment(16); 19 | 20 | ASSERT_EQ(stat.get(), 24); 21 | 22 | } 23 | 24 | TEST_F(MemoryStatTest, DecrementGetTest) { 25 | 26 | storage::memory_stat stat; 27 | 28 | stat.increment(1024); 29 | stat.decrement(1000); 30 | 31 | ASSERT_EQ(stat.get(), 24); 32 | 33 | } 34 | 35 | #endif /* CONFLUO_TEST_MEMORY_STAT_TEST_H_ */ 36 | -------------------------------------------------------------------------------- /libconfluo/test/test_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_TEST_TEST_UTILS_H_ 2 | #define CONFLUO_TEST_TEST_UTILS_H_ 3 | 4 | #include "gtest/gtest.h" 5 | 6 | namespace confluo { 7 | namespace test { 8 | class test_utils { 9 | public: 10 | template 11 | static inline bool test_fail(F f) { 12 | bool fail = false; 13 | try { 14 | f(); 15 | } catch (std::exception &ex) { 16 | fail = true; 17 | } 18 | return fail; 19 | } 20 | }; 21 | 22 | } 23 | } 24 | 25 | #endif /* CONFLUO_TEST_TEST_UTILS_H_ */ 26 | -------------------------------------------------------------------------------- /libconfluo/test/threads/periodic_task_test.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_TEST_PERIODIC_TASK_TEST_H_ 2 | #define CONFLUO_TEST_PERIODIC_TASK_TEST_H_ 3 | 4 | #include "threads/periodic_task.h" 5 | 6 | #include "gtest/gtest.h" 7 | 8 | class PeriodicTaskTest : public testing::Test { 9 | 10 | }; 11 | 12 | TEST_F(PeriodicTaskTest, DummyTaskTest) { 13 | periodic_task task("dummy"); 14 | std::vector vec; 15 | int num = 0; 16 | task.start([&vec, &num] { 17 | vec.push_back(num); 18 | num++; 19 | }, 1); 20 | sleep(1); 21 | task.stop(); 22 | for (int i = 0; i < static_cast(vec.size()); i++) { 23 | ASSERT_EQ(vec[i], i); 24 | } 25 | } 26 | 27 | #endif /* CONFLUO_TEST_PERIODIC_TASK_TEST_H_ */ 28 | -------------------------------------------------------------------------------- /libconfluo/test/threads/task_test.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_TEST_TASK_TEST_H_ 2 | #define CONFLUO_TEST_TASK_TEST_H_ 3 | 4 | #include "gtest/gtest.h" 5 | 6 | #include "threads/task_pool.h" 7 | 8 | using namespace confluo; 9 | 10 | class TaskTest : public testing::Test { 11 | }; 12 | 13 | TEST_F(TaskTest, AddTaskTest) { 14 | task_pool p; 15 | 16 | auto fut = p.submit([]() -> int { 17 | return 1; 18 | }); 19 | 20 | int val = fut.get(); 21 | 22 | ASSERT_EQ(1, val); 23 | } 24 | 25 | #endif /* CONFLUO_TEST_TASK_TEST_H_ */ 26 | -------------------------------------------------------------------------------- /libconfluo/test/threads/thread_manager_test.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_TEST_THREAD_MANAGER_TEST_H_ 2 | #define CONFLUO_TEST_THREAD_MANAGER_TEST_H_ 3 | 4 | #include "threads/thread_manager.h" 5 | #include "gtest/gtest.h" 6 | 7 | using namespace ::confluo; 8 | 9 | class ThreadManagerTest : public testing::Test { 10 | 11 | }; 12 | 13 | TEST_F(ThreadManagerTest, RegisterDeregisterTest) { 14 | int id = thread_manager::register_thread(); 15 | ASSERT_TRUE(-1 != id); 16 | ASSERT_EQ(id, thread_manager::get_id()); 17 | ASSERT_EQ(id, thread_manager::deregister_thread()); 18 | } 19 | 20 | TEST_F(ThreadManagerTest, MultiThreadedRegisterDeregisterTest) { 21 | std::vector threads; 22 | for (int i = 0; i < thread_manager::get_max_concurrency(); i++) { 23 | threads.push_back(std::thread([] { 24 | int id = thread_manager::register_thread(); 25 | ASSERT_TRUE(-1 != id); 26 | ASSERT_TRUE(id < defaults::HARDWARE_CONCURRENCY()); 27 | ASSERT_EQ(id, thread_manager::get_id()); 28 | ASSERT_EQ(id, thread_manager::deregister_thread()); 29 | })); 30 | } 31 | 32 | for (auto &t : threads) { 33 | if (t.joinable()) 34 | t.join(); 35 | } 36 | 37 | } 38 | 39 | #endif /* CONFLUO_TEST_THREAD_MANAGER_TEST_H_ */ 40 | -------------------------------------------------------------------------------- /libconfluo/test/types/data_types_test.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_TEST_DATA_TYPES_TEST_H_ 2 | #define CONFLUO_TEST_DATA_TYPES_TEST_H_ 3 | 4 | #include "gtest/gtest.h" 5 | #include "types/data_type.h" 6 | 7 | using namespace ::confluo; 8 | 9 | class DataTypesTest : public testing::Test { 10 | }; 11 | 12 | TEST_F(DataTypesTest, GetterTest) { 13 | data_type t1 = primitive_types::BOOL_TYPE(), t2 = primitive_types::CHAR_TYPE(), t3 = primitive_types::SHORT_TYPE(), 14 | t4 = primitive_types::INT_TYPE(), t5 = primitive_types::LONG_TYPE(), t6 = primitive_types::FLOAT_TYPE(), 15 | t7 = primitive_types::DOUBLE_TYPE(), t8 = primitive_types::NONE_TYPE(); 16 | 17 | ASSERT_TRUE(primitive_types::BOOL_TYPE() == t1); 18 | ASSERT_TRUE(primitive_types::CHAR_TYPE() == t2); 19 | ASSERT_TRUE(primitive_types::SHORT_TYPE() == t3); 20 | ASSERT_TRUE(primitive_types::INT_TYPE() == t4); 21 | ASSERT_TRUE(primitive_types::LONG_TYPE() == t5); 22 | ASSERT_TRUE(primitive_types::FLOAT_TYPE() == t6); 23 | ASSERT_TRUE(primitive_types::DOUBLE_TYPE() == t7); 24 | ASSERT_TRUE(primitive_types::NONE_TYPE() == t8); 25 | 26 | ASSERT_TRUE(primitive_types::BOOL_TYPE() != primitive_types::CHAR_TYPE()); 27 | 28 | ASSERT_TRUE(t1.name() == "bool"); 29 | ASSERT_TRUE(t2.name() == "char"); 30 | ASSERT_TRUE(t3.name() == "short"); 31 | ASSERT_TRUE(t4.name() == "int"); 32 | ASSERT_TRUE(t5.name() == "long"); 33 | ASSERT_TRUE(t6.name() == "float"); 34 | ASSERT_TRUE(t7.name() == "double"); 35 | ASSERT_TRUE(t8.name() == "none"); 36 | } 37 | 38 | #endif /* CONFLUO_TEST_DATA_TYPES_TEST_H_ */ 39 | -------------------------------------------------------------------------------- /librpc/rpc/rpc_alert_stream.h: -------------------------------------------------------------------------------- 1 | #ifndef RPC_RPC_ALERT_STREAM_H_ 2 | #define RPC_RPC_ALERT_STREAM_H_ 3 | 4 | #include 5 | 6 | #include "rpc_service.h" 7 | 8 | namespace confluo { 9 | namespace rpc { 10 | 11 | /** 12 | * Wrapper for the alert stream 13 | */ 14 | class rpc_alert_stream { 15 | public: 16 | /** The rpc client type */ 17 | typedef rpc_serviceClient rpc_client; 18 | 19 | /** 20 | * Constructs an alert stream for the rpc client 21 | * 22 | * @param table_id The identifier for the table 23 | * @param client The rpc client 24 | * @param handle The data for the stream 25 | */ 26 | rpc_alert_stream(int64_t table_id, std::shared_ptr client, rpc_iterator_handle &&handle); 27 | 28 | /** 29 | * Gets the alert 30 | * 31 | * @return String containing the alert 32 | */ 33 | const std::string &get() const; 34 | 35 | /** 36 | * Advances the alert stream 37 | * 38 | * @return This updated rpc alert stream 39 | */ 40 | rpc_alert_stream &operator++(); 41 | 42 | /** 43 | * Checks whether there is any more elements in the stream 44 | * 45 | * @return True if the stream or handle has any more elements, false 46 | * otherwise 47 | */ 48 | bool has_more() const; 49 | 50 | /** 51 | * Checks whether the alert stream is empty 52 | * 53 | * @return True if the alert stream is empty, false otherwise 54 | */ 55 | bool empty() const; 56 | 57 | private: 58 | int64_t table_id_; 59 | rpc_iterator_handle handle_; 60 | std::stringstream stream_; 61 | std::string alert_; 62 | std::shared_ptr client_; 63 | }; 64 | 65 | } 66 | } 67 | 68 | #endif /* RPC_RPC_ALERT_STREAM_H_ */ 69 | -------------------------------------------------------------------------------- /librpc/rpc/rpc_configuration_params.h: -------------------------------------------------------------------------------- 1 | #ifndef RPC_RPC_CONFIGURATION_PARAMS_H_ 2 | #define RPC_RPC_CONFIGURATION_PARAMS_H_ 3 | 4 | #include 5 | #include 6 | #include "rpc_defaults.h" 7 | 8 | namespace confluo { 9 | namespace rpc { 10 | 11 | /** 12 | * Configuration parameters for the rpc client 13 | */ 14 | class rpc_configuration_params { 15 | public: 16 | /** Iterator for the batches */ 17 | static size_t ITERATOR_BATCH_SIZE() { 18 | return conf::instance().get("iterator_batch_size", rpc_defaults::DEFAULT_ITERATOR_BATCH_SIZE()); 19 | } 20 | }; 21 | 22 | } 23 | } 24 | 25 | #endif /* RPC_RPC_CONFIGURATION_PARAMS_H_ */ 26 | -------------------------------------------------------------------------------- /librpc/rpc/rpc_constants.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.12.0) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | #ifndef rpc_CONSTANTS_H 8 | #define rpc_CONSTANTS_H 9 | 10 | #include "rpc_types.h" 11 | 12 | namespace confluo { namespace rpc { 13 | 14 | class rpcConstants { 15 | public: 16 | rpcConstants(); 17 | 18 | }; 19 | 20 | extern const rpcConstants g_rpc_constants; 21 | 22 | }} // namespace 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /librpc/rpc/rpc_defaults.h: -------------------------------------------------------------------------------- 1 | #ifndef RPC_RPC_DEFAULTS_H_ 2 | #define RPC_RPC_DEFAULTS_H_ 3 | 4 | #include 5 | 6 | namespace confluo { 7 | namespace rpc { 8 | 9 | /** 10 | * Holder of default values for parameters 11 | */ 12 | class rpc_defaults { 13 | public: 14 | // Iterator 15 | /** Default batch size for the iterator */ 16 | static inline size_t DEFAULT_ITERATOR_BATCH_SIZE() { 17 | return 20; 18 | } 19 | }; 20 | 21 | } 22 | } 23 | 24 | #endif /* RPC_RPC_DEFAULTS_H_ */ 25 | -------------------------------------------------------------------------------- /librpc/rpc/rpc_handler_registry.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFLUO_RPC_HANDLER_REGISTRY_H 2 | #define CONFLUO_RPC_HANDLER_REGISTRY_H 3 | 4 | #include 5 | 6 | namespace confluo { 7 | namespace rpc { 8 | 9 | /** 10 | * RPC handler registry 11 | */ 12 | class rpc_handler_registry { 13 | public: 14 | /** 15 | * Add a handler to the registry. 16 | * 17 | * @return A unique identifier for the handler 18 | */ 19 | static uint64_t add(); 20 | 21 | static void remove(uint64_t id); 22 | 23 | private: 24 | /** Current handler id **/ 25 | static std::atomic handler_id_; 26 | }; 27 | 28 | } 29 | } 30 | 31 | #endif //CONFLUO_RPC_HANDLER_REGISTRY_H 32 | -------------------------------------------------------------------------------- /librpc/src/rpc_alert_stream.cc: -------------------------------------------------------------------------------- 1 | #include "rpc_alert_stream.h" 2 | 3 | namespace confluo { 4 | namespace rpc { 5 | 6 | rpc_alert_stream::rpc_alert_stream(int64_t table_id, 7 | std::shared_ptr client, 8 | rpc_iterator_handle &&handle) 9 | : table_id_(table_id), 10 | handle_(std::move(handle)), 11 | stream_(handle_.data) { 12 | if (has_more()) { 13 | std::getline(stream_, alert_); 14 | } 15 | } 16 | 17 | const std::string &rpc_alert_stream::get() const { 18 | return alert_; 19 | } 20 | 21 | rpc_alert_stream &rpc_alert_stream::operator++() { 22 | if (has_more()) { 23 | if (!std::getline(stream_, alert_) && handle_.has_more) { 24 | client_->get_more(handle_, table_id_, handle_.desc); 25 | stream_.str(handle_.data); 26 | } 27 | } 28 | return *this; 29 | } 30 | 31 | bool rpc_alert_stream::has_more() const { 32 | return !stream_.eof() || handle_.has_more; 33 | } 34 | 35 | bool rpc_alert_stream::empty() const { 36 | return !has_more(); 37 | } 38 | 39 | } 40 | } -------------------------------------------------------------------------------- /librpc/src/rpc_constants.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.12.0) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | #include "rpc_constants.h" 8 | 9 | namespace confluo { namespace rpc { 10 | 11 | const rpcConstants g_rpc_constants; 12 | 13 | rpcConstants::rpcConstants() { 14 | } 15 | 16 | }} // namespace 17 | 18 | -------------------------------------------------------------------------------- /librpc/src/rpc_handler_registry.cc: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | 4 | #include "rpc_handler_registry.h" 5 | 6 | namespace confluo { 7 | namespace rpc { 8 | 9 | std::atomic rpc_handler_registry::handler_id_{0}; 10 | 11 | uint64_t rpc_handler_registry::add() { 12 | return handler_id_.fetch_add(1ULL); 13 | } 14 | 15 | void rpc_handler_registry::remove(uint64_t) { 16 | // Do nothing 17 | } 18 | 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /librpc/src/rpc_record_stream.cc: -------------------------------------------------------------------------------- 1 | #include "rpc_record_stream.h" 2 | 3 | namespace confluo { 4 | namespace rpc { 5 | 6 | rpc_record_stream::rpc_record_stream(int64_t multilog_id, 7 | const schema_t &schema, 8 | std::shared_ptr client, 9 | rpc_iterator_handle &&handle) 10 | : multilog_id_(multilog_id), 11 | schema_(schema), 12 | handle_(std::move(handle)), 13 | cur_off_(0), 14 | client_(std::move(client)) { 15 | } 16 | 17 | record_t rpc_record_stream::get() { 18 | return schema_.apply_unsafe(0, &handle_.data[cur_off_]); 19 | } 20 | 21 | rpc_record_stream &rpc_record_stream::operator++() { 22 | if (has_more()) { 23 | cur_off_ += schema_.record_size(); 24 | if (cur_off_ == handle_.data.size() && handle_.has_more) { 25 | client_->get_more(handle_, multilog_id_, handle_.desc); 26 | cur_off_ = 0; 27 | } 28 | } 29 | return *this; 30 | } 31 | 32 | bool rpc_record_stream::has_more() const { 33 | return handle_.has_more || cur_off_ != handle_.data.size(); 34 | } 35 | 36 | bool rpc_record_stream::empty() const { 37 | return !has_more(); 38 | } 39 | 40 | } 41 | } -------------------------------------------------------------------------------- /librpc/test/client_connection_test.h: -------------------------------------------------------------------------------- 1 | #ifndef RPC_TEST_SERVER_CLIENT_TEST_H_ 2 | #define RPC_TEST_SERVER_CLIENT_TEST_H_ 3 | 4 | #include 5 | 6 | #include "gtest/gtest.h" 7 | 8 | #include "conf/configuration_params.h" 9 | #include "rpc_client.h" 10 | #include "rpc_server.h" 11 | #include "rpc_test_utils.h" 12 | 13 | using namespace ::confluo::rpc; 14 | using namespace ::confluo; 15 | 16 | class ClientConnectionTest : public testing::Test { 17 | public: 18 | const std::string SERVER_ADDRESS = "127.0.0.1"; 19 | const int SERVER_PORT = 9090; 20 | }; 21 | 22 | TEST_F(ClientConnectionTest, ConcurrentConnectionsTest) { 23 | auto store = new confluo_store("/tmp"); 24 | auto server = rpc_server::create(store, SERVER_ADDRESS, SERVER_PORT); 25 | std::thread serve_thread([&server] { 26 | server->serve(); 27 | }); 28 | 29 | rpc_test_utils::wait_till_server_ready(SERVER_ADDRESS, SERVER_PORT); 30 | std::vector clients(4); 31 | for (auto &client : clients) { 32 | client.connect(SERVER_ADDRESS, SERVER_PORT); 33 | } 34 | 35 | for (auto &client : clients) { 36 | client.disconnect(); 37 | } 38 | 39 | server->stop(); 40 | 41 | if (serve_thread.joinable()) { 42 | serve_thread.join(); 43 | } 44 | } 45 | 46 | #endif /* RPC_TEST_SERVER_CLIENT_TEST_H_ */ 47 | -------------------------------------------------------------------------------- /librpc/test/rpc_test_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef RPC_TEST_RPC_TEST_UTILS_H_ 2 | #define RPC_TEST_RPC_TEST_UTILS_H_ 3 | 4 | #include "rpc_client.h" 5 | 6 | class rpc_test_utils { 7 | public: 8 | static void wait_till_server_ready(const std::string &host, int port) { 9 | bool check = true; 10 | while (check) { 11 | try { 12 | confluo::rpc::rpc_client(host, port); 13 | check = false; 14 | } catch (TTransportException &e) { 15 | usleep(100000); 16 | } 17 | } 18 | } 19 | }; 20 | 21 | #endif /* RPC_TEST_RPC_TEST_UTILS_H_ */ 22 | -------------------------------------------------------------------------------- /librpc/test/test_main.cc: -------------------------------------------------------------------------------- 1 | #define GTEST_HAS_TR1_TUPLE 0 2 | 3 | #include "gtest/gtest.h" 4 | #include "client_connection_test.h" 5 | #include "client_read_ops_test.h" 6 | #include "client_write_ops_test.h" 7 | #include "error_handling.h" 8 | 9 | int main(int argc, char **argv) { 10 | utils::error_handling::install_signal_handler(argv[0], SIGSEGV, SIGKILL, SIGSTOP); 11 | ::testing::InitGoogleTest(&argc, argv); 12 | return RUN_ALL_TESTS(); 13 | } 14 | -------------------------------------------------------------------------------- /libutils/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories(utils) 2 | 3 | add_library(confluoutils STATIC 4 | utils/byte_utils.h 5 | utils/function_traits.h 6 | utils/bit_utils.h 7 | utils/string_utils.h 8 | utils/cmd_parse.h 9 | utils/file_utils.h 10 | utils/rand_utils.h 11 | utils/assertions.h 12 | utils/mmap_utils.h 13 | utils/optional.h 14 | utils/thread_utils.h 15 | utils/config_utils.h 16 | utils/time_utils.h 17 | utils/math_utils.h 18 | utils/io_utils.h 19 | utils/debug_utils.h 20 | utils/logger.h 21 | utils/error_handling.h 22 | utils/atomic.h 23 | src/utils/assertions.cc 24 | src/utils/cmd_parse.cc 25 | src/utils/config_utils.cc 26 | src/utils/error_handling.cc 27 | src/utils/file_utils.cc 28 | src/utils/io_utils.cc 29 | src/utils/logger.cc 30 | src/utils/math_utils.cc 31 | src/utils/mmap_utils.cc 32 | src/utils/rand_utils.cc 33 | src/utils/string_utils.cc 34 | src/utils/time_utils.cc) 35 | 36 | # install 37 | install(DIRECTORY utils/ 38 | DESTINATION include 39 | FILES_MATCHING PATTERN "*") 40 | -------------------------------------------------------------------------------- /libutils/src/utils/assertions.cc: -------------------------------------------------------------------------------- 1 | #include "assertions.h" 2 | 3 | namespace utils { 4 | namespace assert { 5 | 6 | assertion::assertion(const char *expr, const char *file, int line) 7 | : expr_(expr), 8 | file_(file), 9 | line_(line) { 10 | } 11 | 12 | assertion::~assertion() noexcept(false) { 13 | if (!msg_.str().empty()) 14 | msg_ << ": "; 15 | 16 | std::string expr_str(expr_); 17 | if (expr_str == "false" || expr_str == "FALSE" || expr_str == "0") 18 | msg_ << "Unreachable code assertion"; 19 | else 20 | msg_ << "Assertion '" << expr_str << "'"; 21 | 22 | msg_ << " failed, file '" << file_ << "' at line " << line_; 23 | throw assertion_failure_exception(msg_.str()); 24 | } 25 | 26 | } 27 | } -------------------------------------------------------------------------------- /libutils/src/utils/io_utils.cc: -------------------------------------------------------------------------------- 1 | #include "io_utils.h" 2 | 3 | namespace utils { 4 | 5 | template<> 6 | void io_utils::write(std::ostream &out, const std::string &value) { 7 | size_t size = value.length(); 8 | out.write(reinterpret_cast(&size), sizeof(size_t)); 9 | out.write(value.c_str(), value.length()); 10 | } 11 | 12 | template<> 13 | std::string io_utils::read(std::istream &in) { 14 | size_t size; 15 | in.read(reinterpret_cast(&size), sizeof(size_t)); 16 | std::string value; 17 | value.resize(size); 18 | in.read(&value[0], size); 19 | return value; 20 | } 21 | 22 | std::string io_utils::read(std::istream &in, size_t length) { 23 | std::string value; 24 | value.resize(length); 25 | in.read(&value[0], length); 26 | return value; 27 | } 28 | 29 | void io_utils::flush(std::ostream &out) { 30 | out.flush(); 31 | } 32 | 33 | } -------------------------------------------------------------------------------- /libutils/src/utils/logger.cc: -------------------------------------------------------------------------------- 1 | #include "logger.h" 2 | 3 | namespace utils { 4 | namespace log { 5 | 6 | log_level logger::LOG_LEVEL = log_level::INFO; 7 | 8 | logger::logger() { 9 | msg_level_ = log_level::INFO; 10 | } 11 | 12 | logger::~logger() { 13 | os_ << std::endl; 14 | if (msg_level_ >= LOG_LEVEL) { 15 | fprintf(stderr, "%s", os_.str().c_str()); 16 | fflush(stderr); 17 | } 18 | } 19 | 20 | std::ostringstream &logger::get(const log_level level) { 21 | msg_level_ = level; 22 | os_ << time_utils::current_date_time(); 23 | os_ << " " << to_string(level) << ": "; 24 | return os_; 25 | } 26 | 27 | std::string logger::to_string(const log_level level) { 28 | switch (level) { 29 | case log_level::TRACE:return "TRACE"; 30 | case log_level::DEBUG:return "DEBUG"; 31 | case log_level::INFO:return "INFO"; 32 | case log_level::WARN:return "WARN"; 33 | case log_level::ERROR:return "ERROR"; 34 | case log_level::FATAL:return "FATAL"; 35 | default:return ""; 36 | } 37 | } 38 | 39 | } 40 | } -------------------------------------------------------------------------------- /libutils/src/utils/math_utils.cc: -------------------------------------------------------------------------------- 1 | #include "math_utils.h" 2 | 3 | namespace utils { 4 | 5 | uint64_t math_utils::pow(uint64_t base, uint64_t exp) { 6 | uint64_t result = 1; 7 | while (exp) { 8 | if (exp & 1) 9 | result *= base; 10 | exp >>= 1; 11 | base *= base; 12 | } 13 | return result; 14 | } 15 | 16 | uint32_t math_utils::murmur3_32(const uint8_t *key, size_t len, uint32_t seed) { 17 | uint32_t h = seed; 18 | if (len > 3) { 19 | const uint32_t *key_x4 = (const uint32_t *) key; 20 | size_t i = len >> 2; 21 | do { 22 | uint32_t k = *key_x4++; 23 | k *= 0xcc9e2d51; 24 | k = (k << 15) | (k >> 17); 25 | k *= 0x1b873593; 26 | h ^= k; 27 | h = (h << 13) | (h >> 19); 28 | h = (h * 5) + 0xe6546b64; 29 | } while (--i); 30 | key = (const uint8_t *) key_x4; 31 | } 32 | if (len & 3) { 33 | size_t i = len & 3; 34 | uint32_t k = 0; 35 | key = &key[i - 1]; 36 | do { 37 | k <<= 8; 38 | k |= *key--; 39 | } while (--i); 40 | k *= 0xcc9e2d51; 41 | k = (k << 15) | (k >> 17); 42 | k *= 0x1b873593; 43 | h ^= k; 44 | } 45 | h ^= len; 46 | h ^= h >> 16; 47 | h *= 0x85ebca6b; 48 | h ^= h >> 13; 49 | h *= 0xc2b2ae35; 50 | h ^= h >> 16; 51 | return h; 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /libutils/src/utils/mmap_utils.cc: -------------------------------------------------------------------------------- 1 | #include "mmap_utils.h" 2 | 3 | namespace utils { 4 | 5 | const int mmap_utils::PERMISSIONS; 6 | const int mmap_utils::PROT_RW; 7 | const int mmap_utils::FLAGS; 8 | 9 | void *mmap_utils::map(int fd, void *addr_hint, size_t offset, size_t size, int prot, int flags) { 10 | void *data = mmap(addr_hint, size, prot, flags, fd, static_cast(offset)); 11 | assert_throw( 12 | data != MAP_FAILED, 13 | "mmap(" << addr_hint << ", " << size << ", " << prot << ", " << flags << ", " << fd << ", " << offset << "): " 14 | << strerror(errno)); 15 | return data; 16 | } 17 | 18 | void mmap_utils::unmap(void *addr, size_t size) { 19 | int ret = munmap(addr, size); 20 | assert_throw(ret != -1, "munmap(" << addr << ", " << size << "): " << strerror(errno)); 21 | } 22 | 23 | void mmap_utils::flush(void *addr, size_t size) { 24 | if (size == 0) 25 | return; 26 | static size_t page_size = static_cast(sysconf(_SC_PAGESIZE)); 27 | size_t off = (size_t) addr % page_size; 28 | int ret = msync(((char *) addr) - off, size + off, MS_SYNC); 29 | assert_throw(ret != -1, "msync(" << addr << ", " << size << "): " << strerror(errno)); 30 | } 31 | 32 | } -------------------------------------------------------------------------------- /libutils/src/utils/rand_utils.cc: -------------------------------------------------------------------------------- 1 | #include "rand_utils.h" 2 | 3 | namespace utils { 4 | 5 | int64_t rand_utils::rand_int64(const int64_t &max) { 6 | return rand_int64(0, max); 7 | } 8 | 9 | int64_t rand_utils::rand_int64(const int64_t &min, const int64_t &max) { 10 | static thread_local std::mt19937 generator; 11 | std::uniform_int_distribution distribution(min, max); 12 | return distribution(generator); 13 | } 14 | 15 | uint64_t rand_utils::rand_uint64(const uint64_t &max) { 16 | return rand_uint64(0, max); 17 | } 18 | 19 | uint64_t rand_utils::rand_uint64(const uint64_t &min, const uint64_t &max) { 20 | static thread_local std::mt19937 generator; 21 | std::uniform_int_distribution distribution(min, max); 22 | return distribution(generator); 23 | } 24 | 25 | int32_t rand_utils::rand_int32(const int32_t &max) { 26 | return rand_int32(0, max); 27 | } 28 | 29 | int32_t rand_utils::rand_int32(const int32_t &min, const int32_t &max) { 30 | static thread_local std::mt19937 generator; 31 | std::uniform_int_distribution distribution(min, max); 32 | return distribution(generator); 33 | } 34 | 35 | uint64_t rand_utils::rand_uint32(const uint32_t &max) { 36 | return rand_uint32(0, max); 37 | } 38 | 39 | uint64_t rand_utils::rand_uint32(const uint32_t &min, const uint32_t &max) { 40 | static thread_local std::mt19937 generator; 41 | std::uniform_int_distribution distribution(min, max); 42 | return distribution(generator); 43 | } 44 | 45 | } -------------------------------------------------------------------------------- /libutils/src/utils/string_utils.cc: -------------------------------------------------------------------------------- 1 | #include "string_utils.h" 2 | 3 | namespace utils { 4 | 5 | template<> 6 | bool string_utils::lexical_cast(const std::string &s) { 7 | std::stringstream ss(to_lower(s)); 8 | 9 | bool result; 10 | if ((ss >> std::boolalpha >> result).fail() || !(ss >> std::ws).eof()) { 11 | throw std::bad_cast(); 12 | } 13 | 14 | return result; 15 | } 16 | 17 | std::vector string_utils::split(const std::string &s, char delim, size_t count) { 18 | std::stringstream ss(s); 19 | std::string item; 20 | std::vector elems; 21 | size_t i = 0; 22 | while (std::getline(ss, item, delim) && i < count) { 23 | elems.push_back(std::move(item)); 24 | i++; 25 | } 26 | while (std::getline(ss, item, delim)) 27 | elems.back() += item; 28 | return elems; 29 | } 30 | 31 | std::vector string_utils::split(const std::string &s, char delim) { 32 | return split(s, delim, UINT64_MAX); 33 | } 34 | 35 | std::string string_utils::mk_string(const std::vector &v, const std::string &delim) { 36 | std::string str = ""; 37 | size_t i = 0; 38 | for (; i < v.size() - 1; i++) { 39 | str += v[i] + delim; 40 | } 41 | return str + v[i]; 42 | } 43 | 44 | std::string string_utils::to_upper(const std::string &str) { 45 | return transform(str, ::toupper); 46 | } 47 | 48 | std::string string_utils::to_lower(const std::string &str) { 49 | return transform(str, ::tolower); 50 | } 51 | 52 | } -------------------------------------------------------------------------------- /libutils/src/utils/time_utils.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "time_utils.h" 3 | 4 | using namespace ::std::chrono; 5 | 6 | namespace utils { 7 | 8 | std::string time_utils::current_date_time() { 9 | std::time_t rawtime; 10 | std::tm *timeinfo; 11 | char buffer[100]; 12 | 13 | std::time(&rawtime); 14 | timeinfo = std::localtime(&rawtime); 15 | std::strftime(buffer, sizeof(buffer), "%Y-%m-%d %X", timeinfo); 16 | return std::string(buffer); 17 | } 18 | uint64_t time_utils::cur_ns() { 19 | time_point now = system_clock::now(); 20 | return static_cast(duration_cast(now.time_since_epoch()).count()); 21 | } 22 | uint64_t time_utils::cur_us() { 23 | time_point now = system_clock::now(); 24 | return static_cast(duration_cast(now.time_since_epoch()).count()); 25 | } 26 | uint64_t time_utils::cur_ms() { 27 | time_point now = system_clock::now(); 28 | return static_cast(duration_cast(now.time_since_epoch()).count()); 29 | } 30 | uint64_t time_utils::cur_s() { 31 | time_point now = system_clock::now(); 32 | return static_cast(duration_cast(now.time_since_epoch()).count()); 33 | } 34 | } -------------------------------------------------------------------------------- /libutils/utils/debug_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_DEBUG_UTILS_H_ 2 | #define UTILS_DEBUG_UTILS_H_ 3 | 4 | #include 5 | #include 6 | #ifndef _MSC_VER 7 | # include 8 | #endif 9 | #include 10 | #include 11 | #include 12 | 13 | namespace utils { 14 | 15 | class debug_utils { 16 | public: 17 | template 18 | static std::string type_name() { 19 | typedef typename std::remove_reference::type TR; 20 | std::unique_ptr own( 21 | abi::__cxa_demangle(typeid(TR).name(), nullptr, nullptr, nullptr), 22 | std::free); 23 | std::string r = own != nullptr ? own.get() : typeid(TR).name(); 24 | if (std::is_const::value) 25 | r += " const"; 26 | if (std::is_volatile::value) 27 | r += " volatile"; 28 | if (std::is_lvalue_reference::value) 29 | r += "&"; 30 | else if (std::is_rvalue_reference::value) 31 | r += "&&"; 32 | return r; 33 | } 34 | }; 35 | 36 | } 37 | 38 | #endif /* UTILS_DEBUG_UTILS_H_ */ 39 | -------------------------------------------------------------------------------- /libutils/utils/error_handling.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_ERROR_HANDLING_H_ 2 | #define UTILS_ERROR_HANDLING_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | 14 | namespace utils { 15 | 16 | class error_handling { 17 | public: 18 | static const uint32_t MAX_FRAMES = 64; 19 | 20 | static inline void install_signal_handler(const char *exec) {} 21 | 22 | template 23 | static inline void install_signal_handler(const char *exec, int sig, 24 | args ... more) { 25 | signal(sig, error_handling::sighandler_stacktrace); 26 | error_handling::install_signal_handler(exec, std::forward(more)...); 27 | } 28 | 29 | static std::string stacktrace(); 30 | 31 | private: 32 | static void sighandler_stacktrace(int sig); 33 | }; 34 | 35 | } 36 | 37 | #endif /* UTILS_ERROR_HANDLING_H_ */ 38 | -------------------------------------------------------------------------------- /libutils/utils/file_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_FILE_UTILS_H_ 2 | #define UTILS_FILE_UTILS_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "assertions.h" 17 | 18 | namespace utils { 19 | 20 | class file_utils { 21 | public: 22 | static const int PERMISSIONS = (S_IRWXU | S_IRWXG | S_IRWXO); 23 | 24 | static size_t file_size(const std::string &path); 25 | 26 | static void create_dir(const std::string &path); 27 | 28 | static void clear_dir(const std::string &path); 29 | 30 | static void delete_dir(const std::string &path); 31 | 32 | static bool exists_file(const std::string &path); 33 | 34 | static int delete_file(const std::string &path); 35 | 36 | static int open_file(const std::string &path, int flags); 37 | 38 | static void truncate_file(const std::string &path, size_t size); 39 | 40 | static void truncate_file(int fd, size_t size); 41 | 42 | static void close_file(int fd); 43 | 44 | static std::string full_path(const std::string &path); 45 | 46 | private: 47 | static int unlink_cb(const char *fpath, const struct stat *sb, int typeflag, struct FTW *ftwbuf); 48 | 49 | }; 50 | 51 | } 52 | 53 | #endif /* UTILS_FILE_UTILS_H_ */ 54 | -------------------------------------------------------------------------------- /libutils/utils/io_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_IO_UTILS_H_ 2 | #define UTILS_IO_UTILS_H_ 3 | 4 | #include 5 | #include 6 | 7 | namespace utils { 8 | 9 | class io_utils { 10 | public: 11 | template 12 | static void write(std::ostream &out, const T &value) { 13 | out.write(reinterpret_cast(&value), sizeof(T)); 14 | } 15 | 16 | template 17 | static void write(std::ostream &out, const T *values, size_t length) { 18 | out.write(reinterpret_cast(values), length * sizeof(T)); 19 | } 20 | 21 | template 22 | static T read(std::istream &in) { 23 | T val; 24 | in.read(reinterpret_cast(&val), sizeof(T)); 25 | return val; 26 | } 27 | 28 | static std::string read(std::istream &in, size_t length); 29 | 30 | static void flush(std::ostream &out); 31 | }; 32 | 33 | template<> 34 | void io_utils::write(std::ostream &out, const std::string &value); 35 | 36 | template<> 37 | std::string io_utils::read(std::istream &in); 38 | 39 | } 40 | 41 | #endif /* UTILS_IO_UTILS_H_ */ 42 | -------------------------------------------------------------------------------- /libutils/utils/logger.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_LOGGER_H_ 2 | #define UTILS_LOGGER_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "time_utils.h" 9 | 10 | #define SET_LOG_LEVEL(level) utils::log::logger::LOG_LEVEL = utils::log::log_level::level; 11 | #define LOG_TRACE utils::log::logger().get(utils::log::log_level::TRACE) 12 | #define LOG_DEBUG utils::log::logger().get(utils::log::log_level::DEBUG) 13 | #define LOG_INFO utils::log::logger().get(utils::log::log_level::INFO) 14 | #define LOG_WARN utils::log::logger().get(utils::log::log_level::WARN) 15 | #define LOG_ERROR utils::log::logger().get(utils::log::log_level::ERROR) 16 | #define LOG_FATAL utils::log::logger().get(utils::log::log_level::FATAL) 17 | 18 | namespace utils { 19 | namespace log { 20 | 21 | enum log_level { 22 | ALL = 0, 23 | TRACE = 1, 24 | DEBUG = 2, 25 | INFO = 3, 26 | WARN = 4, 27 | ERROR = 5, 28 | FATAL = 6, 29 | OFF = 7 30 | }; 31 | 32 | class logger { 33 | public: 34 | static log_level LOG_LEVEL; 35 | 36 | logger(); 37 | 38 | virtual ~logger(); 39 | 40 | std::ostringstream &get(const log_level level); 41 | 42 | private: 43 | std::string to_string(const log_level level); 44 | 45 | std::ostringstream os_; 46 | log_level msg_level_; 47 | }; 48 | 49 | } 50 | } 51 | 52 | #endif /* UTILS_LOGGER_H_ */ 53 | -------------------------------------------------------------------------------- /libutils/utils/math_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_MATH_UTILS_H_ 2 | #define UTILS_MATH_UTILS_H_ 3 | 4 | #include 5 | #include 6 | 7 | namespace utils { 8 | 9 | class math_utils { 10 | public: 11 | static uint64_t pow(uint64_t base, uint64_t exp); 12 | 13 | static uint32_t murmur3_32(const uint8_t *key, size_t len, uint32_t seed); 14 | }; 15 | 16 | } 17 | 18 | #endif /* UTILS_MATH_UTILS_H_ */ 19 | -------------------------------------------------------------------------------- /libutils/utils/mmap_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_MMAP_UTILS_H_ 2 | #define UTILS_MMAP_UTILS_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "file_utils.h" 18 | #include "assertions.h" 19 | 20 | #ifndef MAP_POPULATE 21 | #define MAP_POPULATE 0 22 | #endif 23 | 24 | namespace utils { 25 | 26 | class mmap_utils { 27 | public: 28 | static const int PERMISSIONS = (S_IRWXU | S_IRWXG | S_IRWXO); 29 | static const int PROT_RW = PROT_READ | PROT_WRITE; 30 | static const int FLAGS = MAP_SHARED; 31 | 32 | static void *map(int fd, void *addr_hint, size_t offset, size_t size, int prot = PROT_RW, int flags = FLAGS); 33 | 34 | static void unmap(void *addr, size_t size); 35 | 36 | static void flush(void *addr, size_t size); 37 | }; 38 | 39 | } 40 | 41 | #endif /* UTILS_MMAP_UTILS_H_ */ 42 | -------------------------------------------------------------------------------- /libutils/utils/optional.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_OPTIONAL_H_ 2 | #define UTILS_OPTIONAL_H_ 3 | 4 | namespace utils { 5 | 6 | template 7 | class optional { 8 | public: 9 | optional() 10 | : has_value_(false) { 11 | } 12 | 13 | optional(const T &value) 14 | : value_(value), 15 | has_value_(true) { 16 | } 17 | 18 | bool has_value() const { 19 | return has_value_; 20 | } 21 | 22 | T value() const { 23 | if (has_value_) 24 | return value_; 25 | throw -1; 26 | } 27 | 28 | optional &operator=(const optional &other) { 29 | has_value_ = other.has_value_; 30 | value_ = other.value_; 31 | return *this; 32 | } 33 | 34 | optional &operator=(const T &value) { 35 | has_value_ = true; 36 | value_ = value; 37 | return *this; 38 | } 39 | 40 | private: 41 | bool has_value_; 42 | T value_; 43 | }; 44 | 45 | } 46 | 47 | #endif /* UTILS_OPTIONAL_H_ */ 48 | -------------------------------------------------------------------------------- /libutils/utils/rand_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_RAND_UTILS_H_ 2 | #define UTILS_RAND_UTILS_H_ 3 | 4 | #include 5 | #include 6 | 7 | namespace utils { 8 | 9 | class rand_utils { 10 | public: 11 | 12 | static int64_t rand_int64(const int64_t &max); 13 | 14 | static int64_t rand_int64(const int64_t &min, const int64_t &max); 15 | 16 | static uint64_t rand_uint64(const uint64_t &max); 17 | 18 | static uint64_t rand_uint64(const uint64_t &min, const uint64_t &max); 19 | 20 | static int32_t rand_int32(const int32_t &max); 21 | 22 | static int32_t rand_int32(const int32_t &min, const int32_t &max); 23 | 24 | static uint64_t rand_uint32(const uint32_t &max); 25 | 26 | static uint64_t rand_uint32(const uint32_t &min, const uint32_t &max); 27 | }; 28 | 29 | } 30 | 31 | #endif /* UTILS_RAND_UTILS_H_ */ 32 | -------------------------------------------------------------------------------- /libutils/utils/string_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_STRING_UTILS_H_ 2 | #define UTILS_STRING_UTILS_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | namespace utils { 12 | 13 | class string_utils { 14 | public: 15 | static std::vector split(const std::string &s, char delim, size_t count); 16 | 17 | static std::vector split(const std::string &s, char delim); 18 | 19 | static std::string mk_string(const std::vector &v, const std::string &delim); 20 | 21 | template 22 | static std::string transform(const std::string &str, functor f) { 23 | std::string out; 24 | out.resize(str.length()); 25 | std::transform(str.begin(), str.end(), out.begin(), f); 26 | return out; 27 | } 28 | 29 | static std::string to_upper(const std::string &str); 30 | 31 | static std::string to_lower(const std::string &str); 32 | 33 | template 34 | static T lexical_cast(const std::string &s) { 35 | std::stringstream ss(s); 36 | 37 | T result; 38 | if ((ss >> result).fail() || !(ss >> std::ws).eof()) { 39 | throw std::bad_cast(); 40 | } 41 | 42 | return result; 43 | } 44 | }; 45 | 46 | template<> 47 | bool string_utils::lexical_cast(const std::string &s); 48 | 49 | } 50 | 51 | #endif /* UTILS_STRING_UTILS_H_ */ 52 | -------------------------------------------------------------------------------- /libutils/utils/thread_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_THREAD_UTILS_H_ 2 | #define UTILS_THREAD_UTILS_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | namespace utils { 10 | 11 | class thread_utils { 12 | public: 13 | static inline int set_self_core_affinity(int core_id) { 14 | return set_core_affinity(pthread_self(), core_id); 15 | } 16 | 17 | static inline int set_core_affinity(std::thread &t, int core_id) { 18 | return set_core_affinity(t.native_handle(), core_id); 19 | } 20 | 21 | static inline int set_core_affinity(pthread_t thread_handle, int core_id) { 22 | int ret = 0; 23 | #ifdef _GNU_SOURCE 24 | int num_cores = sysconf(_SC_NPROCESSORS_ONLN); 25 | if (core_id < 0 || core_id >= num_cores) { 26 | return EINVAL; 27 | } 28 | 29 | cpu_set_t cpuset; 30 | CPU_ZERO(&cpuset); 31 | CPU_SET(core_id, &cpuset); 32 | 33 | ret = pthread_setaffinity_np(thread_handle, sizeof(cpu_set_t), &cpuset); 34 | #endif 35 | return ret; 36 | } 37 | }; 38 | 39 | } 40 | 41 | #endif /* UTILS_THREAD_UTILS_H_ */ 42 | -------------------------------------------------------------------------------- /libutils/utils/time_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_TIME_UTILS_H_ 2 | #define UTILS_TIME_UTILS_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | namespace utils { 9 | 10 | class time_utils { 11 | public: 12 | static std::string current_date_time(); 13 | 14 | static uint64_t cur_ns(); 15 | 16 | static uint64_t cur_us(); 17 | 18 | static uint64_t cur_ms(); 19 | 20 | static uint64_t cur_s(); 21 | }; 22 | 23 | } 24 | 25 | #endif /* LIBUTILS_UTILS_TIME_UTILS_H_ */ 26 | -------------------------------------------------------------------------------- /pyclient/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(SETUP_PY_IN "${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in") 2 | set(SETUP_PY "${CMAKE_CURRENT_BINARY_DIR}/setup.py") 3 | configure_file(${SETUP_PY_IN} ${SETUP_PY}) 4 | 5 | file(GLOB_RECURSE files RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.py *.cfg *.xml) 6 | create_symlinks(${files}) 7 | 8 | add_custom_target(pyclient ALL 9 | COMMAND ${PYTHON_EXECUTABLE} setup.py build --build-base ${CMAKE_CURRENT_BINARY_DIR} 10 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 11 | COMMENT "Building Python client" 12 | ) 13 | 14 | if (BUILD_TESTS) 15 | set(SERVER_EXEC "${CMAKE_CURRENT_BINARY_DIR}/../librpc/confluod") 16 | add_test(NAME PythonClientTest 17 | COMMAND ${CMAKE_COMMAND} -E env CONFLUO_SERVER_EXEC=${SERVER_EXEC} ${PYTHON_EXECUTABLE} setup.py test 18 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) 19 | endif () 20 | 21 | install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} setup.py install WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})") 22 | -------------------------------------------------------------------------------- /pyclient/confluo/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ucbrise/confluo/e9b8621e99f56b1adf5675c4296c006d89e6e582/pyclient/confluo/__init__.py -------------------------------------------------------------------------------- /pyclient/confluo/rpc/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ['ttypes', 'constants', 'rpc_service'] 2 | -------------------------------------------------------------------------------- /pyclient/confluo/rpc/configuration_params.py: -------------------------------------------------------------------------------- 1 | """ The batch size when reading. 2 | """ 3 | READ_BATCH_SIZE = 128 4 | -------------------------------------------------------------------------------- /pyclient/confluo/rpc/constants.py: -------------------------------------------------------------------------------- 1 | # 2 | # Autogenerated by Thrift Compiler (0.12.0) 3 | # 4 | # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | # 6 | # options string: py 7 | # 8 | 9 | from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException 10 | from thrift.protocol.TProtocol import TProtocolException 11 | from thrift.TRecursive import fix_spec 12 | 13 | import sys 14 | from .ttypes import * 15 | -------------------------------------------------------------------------------- /pyclient/confluo/rpc/storage.py: -------------------------------------------------------------------------------- 1 | from ttypes import * 2 | 3 | 4 | # Alias for rpc_storage_mode 5 | class StorageMode: 6 | """ Confluo storage modes. 7 | 8 | Attributes: 9 | IN_MEMORY: Data stored in memory. 10 | DURABLE_RELAXED: Relaxed (no linearizable guarantees). 11 | DURABLE: Data is persisted. 12 | """ 13 | 14 | def __init__(self): 15 | pass 16 | 17 | IN_MEMORY = rpc_storage_mode.RPC_IN_MEMORY 18 | DURABLE_RELAXED = rpc_storage_mode.RPC_DURABLE_RELAXED 19 | DURABLE = rpc_storage_mode.RPC_DURABLE 20 | -------------------------------------------------------------------------------- /pyclient/confluo/rpc/type_conversions.py: -------------------------------------------------------------------------------- 1 | from data_types import DataType 2 | from schema import SchemaBuilder, Schema 3 | from ttypes import rpc_column 4 | 5 | 6 | def convert_to_rpc_schema(schema): 7 | """ Converts a schema to an rpc schema for the client. 8 | 9 | Args: 10 | schema: The schema to convert. 11 | Returns: 12 | The rpc schema. 13 | """ 14 | rpc_columns = [] 15 | for column in schema.columns_: 16 | rpc_col = rpc_column(column.data_type_.type_id_, column.data_type_.size_, column.name_) 17 | rpc_columns.append(rpc_col) 18 | return rpc_columns 19 | 20 | 21 | def convert_to_schema(rpc_schema): 22 | """ Converts an rpc schema to a schema for confluo. 23 | 24 | Args: 25 | rpc_schema: The rpc_schema to convert. 26 | Returns: 27 | The schema for confluo. 28 | """ 29 | builder = SchemaBuilder() 30 | for column in rpc_schema: 31 | builder.add_column(DataType(column.type_id, column.type_size), column.name) 32 | return Schema(builder.build()) 33 | -------------------------------------------------------------------------------- /pyclient/preamble.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import atexit 4 | import os 5 | import readline 6 | import rlcompleter 7 | import sys 8 | from thrift import Thrift 9 | 10 | from confluo.rpc import client 11 | 12 | confluo = None 13 | 14 | 15 | def confluo_connect(host='localhost', port=9090): 16 | global confluo 17 | confluo = client.RpcClient(host, port) 18 | 19 | 20 | try: 21 | confluo_connect() 22 | print "Confluo Client is now available as confluo." 23 | except Thrift.TException, tx: 24 | print tx.message 25 | print 'Check your server status and retry connecting with confluo_connect(host, port)' 26 | 27 | # Add auto-completion and a stored history file of commands to your Python 28 | # interactive interpreter. Requires Python 2.0+, readline. Autocomplete is 29 | # bound to the Esc key by default. 30 | # 31 | # Store the file in ~/.pystartup, and set an environment variable to point 32 | # to it: "export PYTHONSTARTUP=~/.pystartup" in bash. 33 | 34 | 35 | historyPath = os.path.expanduser("~/.pyhistory") 36 | 37 | 38 | def save_history(path=historyPath): 39 | import readline 40 | readline.write_history_file(path) 41 | 42 | 43 | if os.path.exists(historyPath): 44 | readline.read_history_file(historyPath) 45 | 46 | atexit.register(save_history) 47 | readline.parse_and_bind('tab: complete') 48 | 49 | del os, atexit, readline, rlcompleter, save_history, historyPath, sys 50 | -------------------------------------------------------------------------------- /pyclient/setup.cfg: -------------------------------------------------------------------------------- 1 | [install] 2 | optimize = 1 3 | 4 | [aliases] 5 | test=pytest 6 | 7 | [tool:pytest] 8 | addopts = --color=yes --verbose 9 | python_files = test/*.py -------------------------------------------------------------------------------- /pyclient/setup.py.in: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup 4 | from setuptools import Command 5 | import sys 6 | import os 7 | 8 | class ConfluoShell(Command): 9 | """ Run Confluo shell 10 | """ 11 | 12 | description = "Run interactive Confluo shell" 13 | 14 | user_options = [('hostname', 'h', 'Server hostname'), 15 | ('port', 'p', 'Server port')] 16 | 17 | def initialize_options(self): 18 | self.hostname = "localhost" 19 | self.port = "9090" 20 | 21 | def finalize_options(self): 22 | if self.hostname is None: 23 | raise Exception("Parameter --port is missing") 24 | if self.port is None: 25 | raise Exception("Parameter --port is missing") 26 | 27 | def run(self): 28 | python_ex = sys.executable 29 | os.execv(python_ex, [ python_ex, "-i", "preamble.py"]) 30 | 31 | setup(name='confluo', 32 | version='${CONFLUO_VERSION}', 33 | description='Python Client for Confluo', 34 | author='Anurag Khandelwal', 35 | author_email='anuragk@berkley.edu', 36 | url='https://www.github.com/ucbrise/confluo', 37 | package_dir={'conflo': 'confluo'}, 38 | packages=['confluo.rpc'], 39 | setup_requires=['pytest-runner>=2.0,<4.0', 'thrift>=0.10.0'], 40 | tests_require=['pytest>2.0,<4.0', 'thrift>=${THRIFT_VERSION}'], 41 | install_requires=['thrift>=0.10.0'], 42 | cmdclass={'shell' : ConfluoShell} 43 | ) 44 | -------------------------------------------------------------------------------- /sbin/confluo-config.sh: -------------------------------------------------------------------------------- 1 | # resolve links - $0 may be a softlink 2 | this="${BASH_SOURCE:-$0}" 3 | common_bin="$(cd -P -- "$(dirname -- "$this")" && pwd -P)" 4 | script="$(basename -- "$this")" 5 | this="$common_bin/$script" 6 | 7 | # convert relative path to absolute path 8 | config_bin="`dirname "$this"`" 9 | script="`basename "$this"`" 10 | config_bin="`cd "$config_bin"; pwd`" 11 | this="$config_bin/$script" 12 | 13 | export CONFLUO_PREFIX="`dirname "$this"`"/.. 14 | export CONFLUO_HOME="${CONFLUO_PREFIX}" 15 | export CONFLUO_CONF_DIR="${CONFLUO_CONF_DIR:-"$CONFLUO_HOME/conf"}" 16 | -------------------------------------------------------------------------------- /sbin/load-confluo-env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script loads confluo-env.sh if it exists, and ensures it is only loaded once. 4 | # confluo-env.sh is loaded from CONFLUO_CONF_DIR if set, or within the current directory's 5 | # conf/ subdirectory. 6 | 7 | if [ -z "$CONFLUO_ENV_LOADED" ]; then 8 | export CONFLUO_ENV_LOADED=1 9 | 10 | # Returns the parent of the directory this script lives in. 11 | parent_dir="$(cd "`dirname "$0"`"/..; pwd)" 12 | 13 | user_conf_dir="${CONFLUO_CONF_DIR:-"$parent_dir"/conf}" 14 | 15 | if [ -f "${user_conf_dir}/confluo-env.sh" ]; then 16 | # Promote all variable declarations to environment (exported) variables 17 | set -a 18 | . "${user_conf_dir}/confluo-env.sh" 19 | set +a 20 | fi 21 | fi 22 | -------------------------------------------------------------------------------- /sbin/start-confluo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | sbin="`dirname "$0"`" 4 | sbin="`cd "$sbin"; pwd`" 5 | 6 | . "$sbin/confluo-config.sh" 7 | . "$CONFLUO_PREFIX/sbin/load-confluo-env.sh" 8 | 9 | LOG_PATH="$sbin/../log" 10 | mkdir -p $LOG_PATH 11 | 12 | if [ "$BIND_ADDRESS" = "" ]; then 13 | BIND_ADDRESS="0.0.0.0" 14 | fi 15 | 16 | if [ "$BIND_PORT" = "" ]; then 17 | BIND_PORT="9090" 18 | fi 19 | 20 | if [ "$DATA_PATH" = "" ]; then 21 | DATA_PATH="$sbin/../data" 22 | mkdir -p $DATA_PATH 23 | fi 24 | 25 | mkdir -p $LOG_PATH 26 | $sbin/../build/bin/confluod --address=$BIND_ADDRESS --port $BIND_PORT \ 27 | --data-path $DATA_PATH 2>$LOG_PATH/confluo.stderr 1>$LOG_PATH/confluo.stdout & 28 | -------------------------------------------------------------------------------- /sbin/stop-confluo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | killall confluod 4 | -------------------------------------------------------------------------------- /thrift/genall.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | sbin="`dirname "$0"`" 4 | sbin="`cd "$sbin"; pwd`" 5 | 6 | # Genrate cpp files 7 | thrift -r --gen cpp:templates,pure_enums,no_skeleton,include_prefix $sbin/rpc.thrift 8 | # Modify cpp extensions 9 | cpp_files=gen-cpp/*.cpp 10 | for file in $cpp_files; do 11 | mv "$file" ${file/%.cpp/.cc} 12 | done 13 | 14 | # Move to destination 15 | mv gen-cpp/*.cc $sbin/../librpc/src/ 16 | mv gen-cpp/* $sbin/../librpc/rpc/ 17 | 18 | # Generate java files 19 | thrift -r --gen java:private-members,fullcamel,reuse-objects $sbin/rpc.thrift 20 | 21 | # Move to destination 22 | rsync -r --remove-source-files gen-java/confluo $sbin/../javaclient/src/main/java 23 | 24 | # Generate python files 25 | thrift -r --gen py $sbin/rpc.thrift 26 | 27 | # Move to destination 28 | rsync -r --remove-source-files gen-py/confluo $sbin/../pyclient/ 29 | --------------------------------------------------------------------------------